From 86cf6a3a17376dd83a9222f77521d1619b528d66 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 08:42:03 -0400 Subject: [PATCH 001/263] Remove references to unstable identifiers from MSC3440. (#12382) Removes references to unstable thread relation, unstable identifiers for filtering parameters, and the experimental config flag. --- changelog.d/12382.removal | 1 + synapse/api/constants.py | 2 - synapse/api/filtering.py | 12 ---- synapse/config/experimental.py | 3 - synapse/events/utils.py | 6 -- synapse/handlers/message.py | 5 +- synapse/rest/client/versions.py | 1 - synapse/server.py | 2 +- synapse/storage/databases/main/events.py | 5 +- synapse/storage/databases/main/relations.py | 78 +++++---------------- tests/api/test_filtering.py | 4 +- 11 files changed, 21 insertions(+), 98 deletions(-) create mode 100644 changelog.d/12382.removal diff --git a/changelog.d/12382.removal b/changelog.d/12382.removal new file mode 100644 index 000000000000..eb91186340f7 --- /dev/null +++ b/changelog.d/12382.removal @@ -0,0 +1 @@ +Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 92907415e651..0172eb60b8dc 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -179,8 +179,6 @@ class RelationTypes: REPLACE: Final = "m.replace" REFERENCE: Final = "m.reference" THREAD: Final = "m.thread" - # TODO Remove this in Synapse >= v1.57.0. - UNSTABLE_THREAD: Final = "io.element.thread" class LimitBlockingTypes: diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 27e97d6f372d..4a808e33fee1 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -89,9 +89,7 @@ "org.matrix.not_labels": {"type": "array", "items": {"type": "string"}}, # MSC3440, filtering by event relations. "related_by_senders": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_senders": {"type": "array", "items": {"type": "string"}}, "related_by_rel_types": {"type": "array", "items": {"type": "string"}}, - "io.element.relation_types": {"type": "array", "items": {"type": "string"}}, }, } @@ -323,16 +321,6 @@ def __init__(self, hs: "HomeServer", filter_json: JsonDict): self.related_by_senders = self.filter_json.get("related_by_senders", None) self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None) - # Fallback to the unstable prefix if the stable version is not given. - if hs.config.experimental.msc3440_enabled: - self.related_by_senders = self.related_by_senders or self.filter_json.get( - "io.element.relation_senders", None - ) - self.related_by_rel_types = ( - self.related_by_rel_types - or self.filter_json.get("io.element.relation_types", None) - ) - def filters_all_types(self) -> bool: return "*" in self.not_types diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 447476fbfac5..0772dce4117c 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -26,9 +26,6 @@ class ExperimentalConfig(Config): def read_config(self, config: JsonDict, **kwargs: Any) -> None: experimental = config.get("experimental_features") or {} - # MSC3440 (thread relation) - self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False) - # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 918e87ed9cf1..43c3241fb0d1 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -39,7 +39,6 @@ if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations - from synapse.server import HomeServer # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' @@ -396,9 +395,6 @@ class EventClientSerializer: clients. """ - def __init__(self, hs: "HomeServer"): - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - def serialize_event( self, event: Union[JsonDict, EventBase], @@ -525,8 +521,6 @@ def _inject_bundled_aggregations( "current_user_participated": thread.current_user_participated, } serialized_aggregations[RelationTypes.THREAD] = thread_summary - if self._msc3440_enabled: - serialized_aggregations[RelationTypes.UNSTABLE_THREAD] = thread_summary # Include the bundled aggregations in the event. if serialized_aggregations: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7db6905c6165..47a63005a984 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1102,10 +1102,7 @@ async def _validate_event_relation(self, event: EventBase) -> None: raise SynapseError(400, "Can't send same reaction twice") # Don't attempt to start a thread if the parent event is a relation. - elif ( - relation_type == RelationTypes.THREAD - or relation_type == RelationTypes.UNSTABLE_THREAD - ): + elif relation_type == RelationTypes.THREAD: if await self.store.event_includes_relation(relates_to): raise SynapseError( 400, "Cannot start threads from an event with a relation" diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 9a65aa484360..7ee6b5505b20 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -100,7 +100,6 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030 "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440": self.config.experimental.msc3440_enabled, "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above }, }, diff --git a/synapse/server.py b/synapse/server.py index 380369db923e..37c72bd83a9f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -758,7 +758,7 @@ def get_oidc_handler(self) -> "OidcHandler": @cache_in_self def get_event_client_serializer(self) -> EventClientSerializer: - return EventClientSerializer(self) + return EventClientSerializer() @cache_in_self def get_password_policy_handler(self) -> PasswordPolicyHandler: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 3fcd5f5b99de..e3be537cee35 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1819,10 +1819,7 @@ def _handle_event_relations( if rel_type == RelationTypes.REPLACE: txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,)) - if ( - rel_type == RelationTypes.THREAD - or rel_type == RelationTypes.UNSTABLE_THREAD - ): + if rel_type == RelationTypes.THREAD: txn.call_after(self.store.get_thread_summary.invalidate, (parent_id,)) # It should be safe to only invalidate the cache if the user has not # previously participated in the thread, but that's difficult (and diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 407158ceeed1..a5c31f6787d9 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -14,7 +14,6 @@ import logging from typing import ( - TYPE_CHECKING, Collection, Dict, FrozenSet, @@ -32,20 +31,12 @@ from synapse.api.constants import RelationTypes from synapse.events import EventBase from synapse.storage._base import SQLBaseStore -from synapse.storage.database import ( - DatabasePool, - LoggingDatabaseConnection, - LoggingTransaction, - make_in_list_sql_clause, -) +from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause from synapse.storage.databases.main.stream import generate_pagination_where_clause from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomStreamToken, StreamToken from synapse.util.caches.descriptors import cached, cachedList -if TYPE_CHECKING: - from synapse.server import HomeServer - logger = logging.getLogger(__name__) @@ -63,16 +54,6 @@ class _RelatedEvent: class RelationsWorkerStore(SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._msc3440_enabled = hs.config.experimental.msc3440_enabled - @cached(uncached_args=("event",), tree=True) async def get_relations_for_event( self, @@ -497,7 +478,7 @@ def _get_thread_summaries_txn( AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? ORDER BY parent.event_id, child.topological_ordering DESC, child.stream_ordering DESC """ else: @@ -512,22 +493,16 @@ def _get_thread_summaries_txn( AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? ORDER BY child.topological_ordering DESC, child.stream_ordering DESC """ clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", event_ids ) + args.append(RelationTypes.THREAD) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) latest_event_ids = {} for parent_event_id, child_event_id in txn: # Only consider the latest threaded reply (by topological ordering). @@ -547,7 +522,7 @@ def _get_thread_summaries_txn( AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? GROUP BY parent.event_id """ @@ -556,15 +531,9 @@ def _get_thread_summaries_txn( clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", latest_event_ids.keys() ) + args.append(RelationTypes.THREAD) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) counts = dict(cast(List[Tuple[str, int]], txn.fetchall())) return counts, latest_event_ids @@ -622,7 +591,7 @@ async def get_threaded_messages_per_user( parent.event_id = relates_to_id AND parent.room_id = child.room_id WHERE - %s + relation_type = ? AND %s AND %s GROUP BY parent.event_id, child.sender @@ -638,16 +607,9 @@ def _get_threaded_messages_per_user_txn( txn.database_engine, "relates_to_id", event_ids ) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - relations_args = [RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD] - else: - relations_clause = "relation_type = ?" - relations_args = [RelationTypes.THREAD] - txn.execute( - sql % (users_sql, events_clause, relations_clause), - users_args + events_args + relations_args, + sql % (users_sql, events_clause), + [RelationTypes.THREAD] + users_args + events_args, ) return {(row[0], row[1]): row[2] for row in txn} @@ -677,7 +639,7 @@ async def get_threads_participated( user participated in that event's thread, otherwise false. """ - def _get_thread_summary_txn(txn: LoggingTransaction) -> Set[str]: + def _get_threads_participated_txn(txn: LoggingTransaction) -> Set[str]: # Fetch whether the requester has participated or not. sql = """ SELECT DISTINCT relates_to_id @@ -688,28 +650,20 @@ def _get_thread_summary_txn(txn: LoggingTransaction) -> Set[str]: AND parent.room_id = child.room_id WHERE %s - AND %s + AND relation_type = ? AND child.sender = ? """ clause, args = make_in_list_sql_clause( txn.database_engine, "relates_to_id", event_ids ) + args.extend([RelationTypes.THREAD, user_id]) - if self._msc3440_enabled: - relations_clause = "(relation_type = ? OR relation_type = ?)" - args.extend((RelationTypes.THREAD, RelationTypes.UNSTABLE_THREAD)) - else: - relations_clause = "relation_type = ?" - args.append(RelationTypes.THREAD) - - args.append(user_id) - - txn.execute(sql % (clause, relations_clause), args) + txn.execute(sql % (clause,), args) return {row[0] for row in txn.fetchall()} participated_threads = await self.db_pool.runInteraction( - "get_thread_summary", _get_thread_summary_txn + "get_threads_participated", _get_threads_participated_txn ) return {event_id: event_id in participated_threads for event_id in event_ids} diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 8c3354ce3c79..985d6e397d36 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -481,9 +481,7 @@ def test_filter_relations(self): # events). This is a bit cheeky, but tests the logic of _check_event_relations. # Filter for a particular sender. - definition = { - "io.element.relation_senders": ["@foo:bar"], - } + definition = {"related_by_senders": ["@foo:bar"]} async def events_have_relations(*args, **kwargs): return ["$with_relation"] From 320186319ac4f1d16f8f964d92db8921a4b1073e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:23:43 +0100 Subject: [PATCH 002/263] Resync state after partial-state join (#12394) We work through all the events with partial state, updating the state at each of them. Once it's done, we recalculate the state for the whole room, and then mark the room as having complete state. --- changelog.d/12394.misc | 1 + synapse/handlers/federation.py | 75 +++++++++++++++++++ synapse/handlers/federation_event.py | 39 ++++++++++ synapse/storage/databases/main/events.py | 15 ++++ .../storage/databases/main/events_worker.py | 24 ++++++ synapse/storage/databases/main/room.py | 31 ++++++++ synapse/storage/databases/main/state.py | 48 ++++++++++++ synapse/storage/persist_events.py | 56 ++++++++++++++ 8 files changed, 289 insertions(+) create mode 100644 changelog.d/12394.misc diff --git a/changelog.d/12394.misc b/changelog.d/12394.misc new file mode 100644 index 000000000000..69109fcc37d3 --- /dev/null +++ b/changelog.d/12394.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 78d149905f52..1434e99056cb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -466,6 +466,8 @@ async def do_invite_join( ) if ret.partial_state: + # TODO(faster_joins): roll this back if we don't manage to start the + # background resync (eg process_remote_join fails) await self.store.store_partial_state_room(room_id, ret.servers_in_room) max_stream_id = await self._federation_event_handler.process_remote_join( @@ -478,6 +480,18 @@ async def do_invite_join( partial_state=ret.partial_state, ) + if ret.partial_state: + # Kick off the process of asynchronously fetching the state for this + # room. + # + # TODO(faster_joins): pick this up again on restart + run_as_background_process( + desc="sync_partial_state_room", + func=self._sync_partial_state_room, + destination=origin, + room_id=room_id, + ) + # We wait here until this instance has seen the events come down # replication (if we're using replication) as the below uses caches. await self._replication.wait_for_stream_position( @@ -1370,3 +1384,64 @@ async def get_room_complexity( # We fell off the bottom, couldn't get the complexity from anyone. Oh # well. return None + + async def _sync_partial_state_room( + self, + destination: str, + room_id: str, + ) -> None: + """Background process to resync the state of a partial-state room + + Args: + destination: homeserver to pull the state from + room_id: room to be resynced + """ + + # TODO(faster_joins): do we need to lock to avoid races? What happens if other + # worker processes kick off a resync in parallel? Perhaps we should just elect + # a single worker to do the resync. + # + # TODO(faster_joins): what happens if we leave the room during a resync? if we + # really leave, that might mean we have difficulty getting the room state over + # federation. + # + # TODO(faster_joins): try other destinations if the one we have fails + + logger.info("Syncing state for room %s via %s", room_id, destination) + + # we work through the queue in order of increasing stream ordering. + while True: + batch = await self.store.get_partial_state_events_batch(room_id) + if not batch: + # all the events are updated, so we can update current state and + # clear the lazy-loading flag. + logger.info("Updating current state for %s", room_id) + assert ( + self.storage.persistence is not None + ), "TODO(faster_joins): support for workers" + await self.storage.persistence.update_current_state(room_id) + + logger.info("Clearing partial-state flag for %s", room_id) + success = await self.store.clear_partial_state_room(room_id) + if success: + logger.info("State resync complete for %s", room_id) + + # TODO(faster_joins) update room stats and user directory? + return + + # we raced against more events arriving with partial state. Go round + # the loop again. We've already logged a warning, so no need for more. + # TODO(faster_joins): there is still a race here, whereby incoming events which raced + # with us will fail to be persisted after the call to `clear_partial_state_room` due to + # having partial state. + continue + + events = await self.store.get_events_as_list( + batch, + redact_behaviour=EventRedactBehaviour.AS_IS, + allow_rejected=True, + ) + for event in events: + await self._federation_event_handler.update_state_for_partial_state_event( + destination, event + ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 03c1197c997f..32bf02818c54 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -477,6 +477,45 @@ async def process_remote_join( return await self.persist_events_and_notify(room_id, [(event, context)]) + async def update_state_for_partial_state_event( + self, destination: str, event: EventBase + ) -> None: + """Recalculate the state at an event as part of a de-partial-stating process + + Args: + destination: server to request full state from + event: partial-state event to be de-partial-stated + """ + logger.info("Updating state for %s", event.event_id) + with nested_logging_context(suffix=event.event_id): + # if we have all the event's prev_events, then we can work out the + # state based on their states. Otherwise, we request it from the destination + # server. + # + # This is the same operation as we do when we receive a regular event + # over federation. + state = await self._resolve_state_at_missing_prevs(destination, event) + + # build a new state group for it if need be + context = await self._state_handler.compute_event_context( + event, + old_state=state, + ) + if context.partial_state: + # this can happen if some or all of the event's prev_events still have + # partial state - ie, an event has an earlier stream_ordering than one + # or more of its prev_events, so we de-partial-state it before its + # prev_events. + # + # TODO(faster_joins): we probably need to be more intelligent, and + # exclude partial-state prev_events from consideration + logger.warning( + "%s still has partial state: can't de-partial-state it yet", + event.event_id, + ) + return + await self._store.update_state_for_partial_state_event(event, context) + async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] ) -> None: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index e3be537cee35..2a1e567ce08e 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -963,6 +963,21 @@ def _persist_transaction_ids_txn( values=to_insert, ) + async def update_current_state( + self, + room_id: str, + state_delta: DeltaState, + stream_id: int, + ) -> None: + """Update the current state stored in the datatabase for the given room""" + + await self.db_pool.runInteraction( + "update_current_state", + self._update_current_state_txn, + state_delta_by_room={room_id: state_delta}, + stream_id=stream_id, + ) + def _update_current_state_txn( self, txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index a60e3f4fdde0..5288cdba035d 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1979,3 +1979,27 @@ async def is_partial_state_event(self, event_id: str) -> bool: desc="is_partial_state_event", ) return result is not None + + async def get_partial_state_events_batch(self, room_id: str) -> List[str]: + """Get a list of events in the given room that have partial state""" + return await self.db_pool.runInteraction( + "get_partial_state_events_batch", + self._get_partial_state_events_batch_txn, + room_id, + ) + + @staticmethod + def _get_partial_state_events_batch_txn( + txn: LoggingTransaction, room_id: str + ) -> List[str]: + txn.execute( + """ + SELECT event_id FROM partial_state_events AS pse + JOIN events USING (event_id) + WHERE pse.room_id = ? + ORDER BY events.stream_ordering + LIMIT 100 + """, + (room_id,), + ) + return [row[0] for row in txn] diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 18b1acd9e113..87e9482c6054 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1077,6 +1077,37 @@ def get_rooms_for_retention_period_in_range_txn( get_rooms_for_retention_period_in_range_txn, ) + async def clear_partial_state_room(self, room_id: str) -> bool: + # this can race with incoming events, so we watch out for FK errors. + # TODO(faster_joins): this still doesn't completely fix the race, since the persist process + # is not atomic. I fear we need an application-level lock. + try: + await self.db_pool.runInteraction( + "clear_partial_state_room", self._clear_partial_state_room_txn, room_id + ) + return True + except self.db_pool.engine.module.DatabaseError as e: + # TODO(faster_joins): how do we distinguish between FK errors and other errors? + logger.warning( + "Exception while clearing lazy partial-state-room %s, retrying: %s", + room_id, + e, + ) + return False + + @staticmethod + def _clear_partial_state_room_txn(txn: LoggingTransaction, room_id: str) -> None: + DatabasePool.simple_delete_txn( + txn, + table="partial_state_rooms_servers", + keyvalues={"room_id": room_id}, + ) + DatabasePool.simple_delete_one_txn( + txn, + table="partial_state_rooms", + keyvalues={"room_id": room_id}, + ) + class _BackgroundUpdates: REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory" diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index ecdc1fdc4c13..eba35f37000d 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -21,6 +21,7 @@ from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.events import EventBase +from synapse.events.snapshot import EventContext from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -354,6 +355,53 @@ async def get_referenced_state_groups( return {row["state_group"] for row in rows} + async def update_state_for_partial_state_event( + self, + event: EventBase, + context: EventContext, + ) -> None: + """Update the state group for a partial state event""" + await self.db_pool.runInteraction( + "update_state_for_partial_state_event", + self._update_state_for_partial_state_event_txn, + event, + context, + ) + + def _update_state_for_partial_state_event_txn( + self, + txn, + event: EventBase, + context: EventContext, + ): + # we shouldn't have any outliers here + assert not event.internal_metadata.is_outlier() + + # anything that was rejected should have the same state as its + # predecessor. + if context.rejected: + assert context.state_group == context.state_group_before_event + + self.db_pool.simple_update_txn( + txn, + table="event_to_state_groups", + keyvalues={"event_id": event.event_id}, + updatevalues={"state_group": context.state_group}, + ) + + self.db_pool.simple_delete_one_txn( + txn, + table="partial_state_events", + keyvalues={"event_id": event.event_id}, + ) + + # TODO(faster_joins): need to do something about workers here + txn.call_after( + self._get_state_group_for_event.prefill, + (event.event_id,), + context.state_group, + ) + class MainStateBackgroundUpdateStore(RoomMemberWorkerStore): diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index b40292281767..e496ba7bed6e 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -376,6 +376,62 @@ async def persist_event( pos = PersistedEventPosition(self._instance_name, event_stream_id) return event, pos, self.main_store.get_room_max_token() + async def update_current_state(self, room_id: str) -> None: + """Recalculate the current state for a room, and persist it""" + state = await self._calculate_current_state(room_id) + delta = await self._calculate_state_delta(room_id, state) + + # TODO(faster_joins): get a real stream ordering, to make this work correctly + # across workers. + # + # TODO(faster_joins): this can race against event persistence, in which case we + # will end up with incorrect state. Perhaps we should make this a job we + # farm out to the event persister, somehow. + stream_id = self.main_store.get_room_max_stream_ordering() + await self.persist_events_store.update_current_state(room_id, delta, stream_id) + + async def _calculate_current_state(self, room_id: str) -> StateMap[str]: + """Calculate the current state of a room, based on the forward extremities + + Args: + room_id: room for which to calculate current state + + Returns: + map from (type, state_key) to event id for the current state in the room + """ + latest_event_ids = await self.main_store.get_latest_event_ids_in_room(room_id) + state_groups = set( + ( + await self.main_store._get_state_group_for_events(latest_event_ids) + ).values() + ) + + state_maps_by_state_group = await self.state_store._get_state_for_groups( + state_groups + ) + + if len(state_groups) == 1: + # If there is only one state group, then we know what the current + # state is. + return state_maps_by_state_group[state_groups.pop()] + + # Ok, we need to defer to the state handler to resolve our state sets. + logger.debug("calling resolve_state_groups from preserve_events") + + # Avoid a circular import. + from synapse.state import StateResolutionStore + + room_version = await self.main_store.get_room_version_id(room_id) + res = await self._state_resolution_handler.resolve_state_groups( + room_id, + room_version, + state_maps_by_state_group, + event_map=None, + state_res_store=StateResolutionStore(self.main_store), + ) + + return res.state + async def _persist_event_batch( self, events_and_contexts: List[Tuple[EventBase, EventContext]], From 9535fd0f9c4227be14452271e163c2ddbc3a0a19 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 10:20:46 -0400 Subject: [PATCH 003/263] Disable groups/communities by default. (#12344) This disables the endpoints (and sync response fields) for groups/communities by default. --- changelog.d/12344.removal | 1 + docs/upgrade.md | 7 +++++++ synapse/config/experimental.py | 2 +- tests/rest/admin/test_admin.py | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12344.removal diff --git a/changelog.d/12344.removal b/changelog.d/12344.removal new file mode 100644 index 000000000000..ecefa76d8ea5 --- /dev/null +++ b/changelog.d/12344.removal @@ -0,0 +1 @@ +The groups/communities feature in Synapse has been disabled by default. diff --git a/docs/upgrade.md b/docs/upgrade.md index f0c0a3998d98..3d518c81c3b0 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -85,6 +85,13 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.58.0 + +## Groups/communities feature has been disabled by default + +The non-standard groups/communities feature in Synapse has been disabled by default +and will be removed in Synapse v1.61.0. + # Upgrading to v1.57.0 ## Changes to database schema for application services diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 0772dce4117c..979059e72386 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -74,7 +74,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False) # The deprecated groups feature. - self.groups_enabled: bool = experimental.get("groups_enabled", True) + self.groups_enabled: bool = experimental.get("groups_enabled", False) # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 849d00ab4d94..40571b753a9a 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -63,6 +63,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.other_user = self.register_user("user", "pass") self.other_user_token = self.login("user", "pass") + @unittest.override_config({"experimental_features": {"groups_enabled": True}}) def test_delete_group(self) -> None: # Create a new group channel = self.make_request( From ba1588461beadff7d5525d05308731ba286c425f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 16:12:57 +0100 Subject: [PATCH 004/263] Fix typos in release script docs (#12450) --- changelog.d/12450.misc | 1 + scripts-dev/release.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12450.misc diff --git a/changelog.d/12450.misc b/changelog.d/12450.misc new file mode 100644 index 000000000000..4b1c8cba875f --- /dev/null +++ b/changelog.d/12450.misc @@ -0,0 +1 @@ +Fix typo in the release script help string. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 685fa32b03f4..518eaf417cfe 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -69,11 +69,12 @@ def cli(): # ... wait for assets to build ... ./scripts-dev/release.py publish + ./scripts-dev/release.py upload # Optional: generate some nice links for the announcement - ./scripts-dev/release.py upload + ./scripts-dev/release.py announce If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. From 4bdbebccb98bcc0e8862b1e8b532d9a491a90dac Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 12 Apr 2022 11:27:45 -0400 Subject: [PATCH 005/263] Remove the unstable event field for `/send_join` per MSC3083. (#12395) This was missed when initially stabilising room version 8 and was left in as a compatibility shim. Most homeservers have upgraded to a version which expects the proper field name, and the failure mode is reasonable (a user on an older server may have to attempt joining the room twice with an obscure error message the first time). --- changelog.d/12395.misc | 1 + synapse/federation/federation_server.py | 2 -- synapse/federation/transport/client.py | 10 ---------- 3 files changed, 1 insertion(+), 12 deletions(-) create mode 100644 changelog.d/12395.misc diff --git a/changelog.d/12395.misc b/changelog.d/12395.misc new file mode 100644 index 000000000000..0a2123b2942a --- /dev/null +++ b/changelog.d/12395.misc @@ -0,0 +1 @@ +Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 69d833585f2a..e67af6463fc7 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -687,8 +687,6 @@ async def on_send_join_request( time_now = self._clock.time_msec() event_json = event.get_pdu_json(time_now) resp = { - # TODO Remove the unstable prefix when servers have updated. - "org.matrix.msc3083.v2.event": event_json, "event": event_json, "state": [p.get_pdu_json(time_now) for p in state_events], "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events], diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 01dc5ca94f99..1421050b9a53 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -1380,16 +1380,6 @@ def __init__(self, room_version: RoomVersion, v1_api: bool): prefix + "auth_chain.item", use_float=True, ), - # TODO Remove the unstable prefix when servers have updated. - # - # By re-using the same event dictionary this will cause the parsing of - # org.matrix.msc3083.v2.event and event to stomp over each other. - # Generally this should be fine. - ijson.kvitems_coro( - _event_parser(self._response.event_dict), - prefix + "org.matrix.msc3083.v2.event", - use_float=True, - ), ijson.kvitems_coro( _event_parser(self._response.event_dict), prefix + "event", From aa2811026402394b4013033f075d8f509cdc1257 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Apr 2022 16:50:40 +0100 Subject: [PATCH 006/263] Process device list updates asynchronously (#12365) --- changelog.d/12365.feature | 1 + synapse/config/server.py | 8 --- synapse/handlers/device.py | 28 ---------- synapse/storage/databases/main/devices.py | 61 ++++------------------ synapse/storage/schema/__init__.py | 6 +-- tests/federation/test_federation_sender.py | 8 --- tests/storage/test_devices.py | 47 ++++++++++------- 7 files changed, 40 insertions(+), 119 deletions(-) create mode 100644 changelog.d/12365.feature diff --git a/changelog.d/12365.feature b/changelog.d/12365.feature new file mode 100644 index 000000000000..642dea966c44 --- /dev/null +++ b/changelog.d/12365.feature @@ -0,0 +1 @@ +Enable processing of device list updates asynchronously. diff --git a/synapse/config/server.py b/synapse/config/server.py index 415279d269d6..d771045b522a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -680,14 +680,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("use_account_validity_in_account_status") or False ) - # This is a temporary option that enables fully using the new - # `device_lists_changes_in_room` without the backwards compat code. This - # is primarily for testing. If enabled the server should *not* be - # downgraded, as it may lead to missing device list updates. - self.use_new_device_lists_changes_in_room = ( - config.get("use_new_device_lists_changes_in_room") or False - ) - self.rooms_to_exclude_from_sync: List[str] = ( config.get("exclude_rooms_from_sync") or [] ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index ffa28b2a3077..958599e7b816 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -291,12 +291,6 @@ def __init__(self, hs: "HomeServer"): # On start up check if there are any updates pending. hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - # Used to decide if we calculate outbound pokes up front or not. By - # default we do to allow safely downgrading Synapse. - self.use_new_device_lists_changes_in_room = ( - hs.config.server.use_new_device_lists_changes_in_room - ) - def _check_device_name_length(self, name: Optional[str]) -> None: """ Checks whether a device name is longer than the maximum allowed length. @@ -490,23 +484,9 @@ async def notify_device_update( room_ids = await self.store.get_rooms_for_user(user_id) - hosts: Optional[Set[str]] = None - if not self.use_new_device_lists_changes_in_room: - hosts = set() - - if self.hs.is_mine_id(user_id): - for room_id in room_ids: - joined_users = await self.store.get_users_in_room(room_id) - hosts.update(get_domain_from_id(u) for u in joined_users) - - set_tag("target_hosts", hosts) - - hosts.discard(self.server_name) - position = await self.store.add_device_change_to_streams( user_id, device_ids, - hosts=hosts, room_ids=room_ids, ) @@ -528,14 +508,6 @@ async def notify_device_update( # We may need to do some processing asynchronously. self._handle_new_device_update_async() - if hosts: - logger.info( - "Sending device list update notif for %r to: %r", user_id, hosts - ) - for host in hosts: - self.federation_sender.send_device_messages(host, immediate=False) - log_kv({"message": "sent device update to host", "host": host}) - async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] ) -> None: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index dc8009b23df5..74e4e2122a20 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1582,7 +1582,6 @@ async def add_device_change_to_streams( self, user_id: str, device_ids: Collection[str], - hosts: Optional[Collection[str]], room_ids: Collection[str], ) -> Optional[int]: """Persist that a user's devices have been updated, and which hosts @@ -1592,9 +1591,6 @@ async def add_device_change_to_streams( user_id: The ID of the user whose device changed. device_ids: The IDs of any changed devices. If empty, this function will return None. - hosts: The remote destinations that should be notified of the change. If - None then the set of hosts have *not* been calculated, and will be - calculated later by a background task. room_ids: The rooms that the user is in Returns: @@ -1606,14 +1602,12 @@ async def add_device_change_to_streams( context = get_active_span_text_map() - def add_device_changes_txn( - txn, stream_ids_for_device_change, stream_ids_for_outbound_pokes - ): + def add_device_changes_txn(txn, stream_ids): self._add_device_change_to_stream_txn( txn, user_id, device_ids, - stream_ids_for_device_change, + stream_ids, ) self._add_device_outbound_room_poke_txn( @@ -1621,43 +1615,17 @@ def add_device_changes_txn( user_id, device_ids, room_ids, - stream_ids_for_device_change, - context, - hosts_have_been_calculated=hosts is not None, - ) - - # If the set of hosts to send to has not been calculated yet (and so - # `hosts` is None) or there are no `hosts` to send to, then skip - # trying to persist them to the DB. - if not hosts: - return - - self._add_device_outbound_poke_to_stream_txn( - txn, - user_id, - device_ids, - hosts, - stream_ids_for_outbound_pokes, + stream_ids, context, ) - # `device_lists_stream` wants a stream ID per device update. - num_stream_ids = len(device_ids) - - if hosts: - # `device_lists_outbound_pokes` wants a different stream ID for - # each row, which is a row per host per device update. - num_stream_ids += len(hosts) * len(device_ids) - - async with self._device_list_id_gen.get_next_mult(num_stream_ids) as stream_ids: - stream_ids_for_device_change = stream_ids[: len(device_ids)] - stream_ids_for_outbound_pokes = stream_ids[len(device_ids) :] - + async with self._device_list_id_gen.get_next_mult( + len(device_ids) + ) as stream_ids: await self.db_pool.runInteraction( "add_device_change_to_stream", add_device_changes_txn, - stream_ids_for_device_change, - stream_ids_for_outbound_pokes, + stream_ids, ) return stream_ids[-1] @@ -1752,19 +1720,8 @@ def _add_device_outbound_room_poke_txn( room_ids: Collection[str], stream_ids: List[str], context: Dict[str, str], - hosts_have_been_calculated: bool, ) -> None: - """Record the user in the room has updated their device. - - Args: - hosts_have_been_calculated: True if `device_lists_outbound_pokes` - has been updated already with the updates. - """ - - # We only need to convert to outbound pokes if they are our user. - converted_to_destinations = ( - hosts_have_been_calculated or not self.hs.is_mine_id(user_id) - ) + """Record the user in the room has updated their device.""" encoded_context = json_encoder.encode(context) @@ -1789,7 +1746,7 @@ def _add_device_outbound_room_poke_txn( device_id, room_id, stream_id, - converted_to_destinations, + False, encoded_context, ) for room_id in room_ids diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 151f2aa9bbfe..871d4ace123c 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -66,9 +66,9 @@ SCHEMA_COMPAT_VERSION = ( - # we now have `state_key` columns in both `events` and `state_events`, so - # now incompatible with synapses wth SCHEMA_VERSION < 66. - 66 + # We now assume that `device_lists_changes_in_room` has been filled out for + # recent device_list_updates. + 69 ) """Limit on how far the synapse codebase can be rolled back without breaking db compat diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index a6e91956af2f..63ea4f9ee475 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -14,7 +14,6 @@ from typing import Optional from unittest.mock import Mock -from parameterized import parameterized_class from signedjson import key, sign from signedjson.types import BaseKey, SigningKey @@ -155,12 +154,6 @@ def test_send_receipts_with_backoff(self): ) -@parameterized_class( - [ - {"enable_room_poke_code_path": False}, - {"enable_room_poke_code_path": True}, - ] -) class FederationSenderDevicesTestCases(HomeserverTestCase): servlets = [ admin.register_servlets, @@ -175,7 +168,6 @@ def make_homeserver(self, reactor, clock): def default_config(self): c = super().default_config() c["send_federation"] = True - c["use_new_device_lists_changes_in_room"] = self.enable_room_poke_code_path return c def prepare(self, reactor, clock, hs): diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index d1227dd4ac02..5491fbf6da7b 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -21,6 +21,29 @@ class DeviceStoreTestCase(HomeserverTestCase): def prepare(self, reactor, clock, hs): self.store = hs.get_datastores().main + def add_device_change(self, user_id, device_ids, host): + """Add a device list change for the given device to + `device_lists_outbound_pokes` table. + """ + + for device_id in device_ids: + stream_id = self.get_success( + self.store.add_device_change_to_streams( + "user_id", [device_id], ["!some:room"] + ) + ) + + self.get_success( + self.store.add_device_list_outbound_pokes( + user_id=user_id, + device_id=device_id, + room_id="!some:room", + stream_id=stream_id, + hosts=[host], + context={}, + ) + ) + def test_store_new_device(self): self.get_success( self.store.store_device("user_id", "device_id", "display_name") @@ -95,11 +118,7 @@ def test_get_device_updates_by_remote(self): device_ids = ["device_id1", "device_id2"] # Add two device updates with sequential `stream_id`s - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get all device updates ever meant for this remote now_stream_id, device_updates = self.get_success( @@ -123,11 +142,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): "device_id4", "device_id5", ] - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( @@ -147,11 +162,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): # Add some more device updates to ensure it still resumes properly device_ids = ["device_id6", "device_id7"] - self.get_success( - self.store.add_device_change_to_streams( - "user_id", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("user_id", device_ids, "somehost") # Get the next batch of device updates next_stream_id, device_updates = self.get_success( @@ -224,11 +235,7 @@ def test_get_device_updates_by_remote_cross_signing_key_updates( "fakeSelfSigning", ] - self.get_success( - self.store.add_device_change_to_streams( - "@user_id:test", device_ids, ["somehost"], ["!some:room"] - ) - ) + self.add_device_change("@user_id:test", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( From 58c657322a18cadf8036819ef87fce9889d65fb0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:35:48 +0100 Subject: [PATCH 007/263] Run lints under poetry in CI; remove lint tox jobs (#12434) Co-authored-by: Dan Callahan --- .github/workflows/tests.yml | 20 +++++++------------- changelog.d/12434.misc | 1 + poetry.lock | 6 +++--- tox.ini | 31 +------------------------------ 4 files changed, 12 insertions(+), 46 deletions(-) create mode 100644 changelog.d/12434.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a98f619328f..c300d6547b28 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,19 +20,13 @@ jobs: - run: scripts-dev/config-lint.sh lint: - runs-on: ubuntu-latest - strategy: - matrix: - toxenv: - - "check_codestyle" - - "check_isort" - - "mypy" - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install tox - - run: tox -e ${{ matrix.toxenv }} + # This does a vanilla `poetry install` - no extras. I'm slightly anxious + # that we might skip some typechecks on code that uses extras. However, + # I think the right way to fix this is to mark any extras needed for + # typechecking as development dependencies. To detect this, we ought to + # turn up mypy's strictness: disallow unknown imports and be accept fewer + # uses of `Any`. + uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" lint-crlf: runs-on: ubuntu-latest diff --git a/changelog.d/12434.misc b/changelog.d/12434.misc new file mode 100644 index 000000000000..88dab428d245 --- /dev/null +++ b/changelog.d/12434.misc @@ -0,0 +1 @@ +Run lints under poetry in CI, and remove corresponding tox lint jobs. diff --git a/poetry.lock b/poetry.lock index bbe8eba96d40..761ee03abc48 100644 --- a/poetry.lock +++ b/poetry.lock @@ -717,7 +717,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock [[package]] name = "prometheus-client" -version = "0.13.1" +version = "0.14.0" description = "Python client for the Prometheus monitoring system." category = "main" optional = false @@ -2225,8 +2225,8 @@ platformdirs = [ {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, ] prometheus-client = [ - {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"}, - {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"}, + {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"}, + {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"}, ] psycopg2 = [ {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"}, diff --git a/tox.ini b/tox.ini index b4ce400edfc1..4cd9dfb966c7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, check_codestyle, check_isort +envlist = py37, py38, py39, py310 # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208 minversion = 2.3.2 @@ -32,20 +32,6 @@ deps = # install the "enum34" dependency of cryptography. pip>=10 -# directories/files we run the linters on. -# TODO: this is now out of date; we will remove as part of poetry migration. -lint_targets = - setup.py - synapse - tests - # annoyingly, black doesn't find these so we have to list them - scripts-dev - stubs - contrib - synmark - .ci - docker - # default settings for all tox environments [testenv] deps = @@ -116,18 +102,3 @@ setenv = commands = python -m synmark {posargs:} -[testenv:check_codestyle] -extras = lint -commands = - python -m black --check --diff {[base]lint_targets} - flake8 {[base]lint_targets} {env:PEP8SUFFIX:} - -[testenv:check_isort] -extras = lint -commands = isort -c --df {[base]lint_targets} - -[testenv:mypy] -deps = - {[base]deps} -extras = all,mypy -commands = mypy From 5a275a237700a606fde5c03e4a0f116a1c9aeda8 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:41:21 +0100 Subject: [PATCH 008/263] Run "main" trial tests under poetry (#12438) * Run "main" trial tests under poetry Olddeps and twisted trunk tests are handled in separate PRs. The PyPy config is a best-effort only; it's completely untested. Pulled out from #12337. * Changelog --- .github/workflows/tests.yml | 26 +++++++++++++------------- changelog.d/12438.misc | 1 + 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 changelog.d/12438.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c300d6547b28..7946b76dba8c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,23 +65,23 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] database: ["sqlite"] - toxenv: ["py"] + extras: ["all"] include: # Newest Python without optional deps - python-version: "3.10" - toxenv: "py-noextras" + extras: "" # Oldest Python with PostgreSQL - python-version: "3.7" database: "postgres" postgres-version: "10" - toxenv: "py" + extras: "all" # Newest Python with newest PostgreSQL - python-version: "3.10" database: "postgres" postgres-version: "14" - toxenv: "py" + extras: "all" steps: - uses: actions/checkout@v2 @@ -93,17 +93,16 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox + extras: ${{ matrix.extras }} - name: Await PostgreSQL if: ${{ matrix.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done - - run: tox -e ${{ matrix.toxenv }} + - run: poetry run trial --jobs=2 tests env: - TRIAL_FLAGS: "--jobs=2" SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} SYNAPSE_POSTGRES_HOST: localhost SYNAPSE_POSTGRES_USER: postgres @@ -150,23 +149,24 @@ jobs: trial-pypy: # Very slow; only run if the branch name includes 'pypy' + # Note: sqlite only; no postgres. Completely untested since poetry move. if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} needs: linting-done runs-on: ubuntu-latest strategy: matrix: python-version: ["pypy-3.7"] + extras: ["all"] steps: - uses: actions/checkout@v2 + # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + extras: ${{ matrix.extras }} + - run: poetry run trial --jobs=2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/changelog.d/12438.misc b/changelog.d/12438.misc new file mode 100644 index 000000000000..f2c07a56da14 --- /dev/null +++ b/changelog.d/12438.misc @@ -0,0 +1 @@ +Run "main" trial tests under `poetry`. From 32545d2e2611743633d9f98a40d5f62791e19a6f Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 12 Apr 2022 13:46:55 -0700 Subject: [PATCH 009/263] Bump twisted version to the latest in lockfile (#12441) --- changelog.d/12441.misc | 1 + poetry.lock | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelog.d/12441.misc diff --git a/changelog.d/12441.misc b/changelog.d/12441.misc new file mode 100644 index 000000000000..c2619f16540b --- /dev/null +++ b/changelog.d/12441.misc @@ -0,0 +1 @@ +Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). diff --git a/poetry.lock b/poetry.lock index 761ee03abc48..a9f3e61015c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1285,7 +1285,7 @@ urllib3 = ">=1.26.0" [[package]] name = "twisted" -version = "22.2.0" +version = "22.4.0" description = "An asynchronous networking framework written in Python" category = "main" optional = false @@ -1305,19 +1305,20 @@ typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] +conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"] contextvars = ["contextvars (>=2.4,<3)"] dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"] dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"] -http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] +macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] +osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"] tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"] -windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] [[package]] name = "twisted-iocpsupport" @@ -2592,8 +2593,8 @@ twine = [ {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"}, ] twisted = [ - {file = "Twisted-22.2.0-py3-none-any.whl", hash = "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"}, - {file = "Twisted-22.2.0.tar.gz", hash = "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2"}, + {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"}, + {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"}, ] twisted-iocpsupport = [ {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"}, From 36d8b83888aaa30dd2738f551ecab4f7eb5c206e Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 13 Apr 2022 11:32:44 +0200 Subject: [PATCH 010/263] Rename Mutual Rooms `unstable_features` flag to match MSC (#12445) Signed-off-by: Jonathan de Jong --- changelog.d/12445.misc | 1 + synapse/rest/client/versions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12445.misc diff --git a/changelog.d/12445.misc b/changelog.d/12445.misc new file mode 100644 index 000000000000..954248115a0d --- /dev/null +++ b/changelog.d/12445.misc @@ -0,0 +1 @@ +Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. \ No newline at end of file diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7ee6b5505b20..7b29026381ed 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -86,7 +86,7 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666": True, + "uk.half-shot.msc2666.mutual_rooms": True, # Whether new rooms will be set to encrypted or not (based on presets). "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, From d24cd17820e9f3bd654cfd923b27ca981100c691 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 13 Apr 2022 11:26:53 +0100 Subject: [PATCH 011/263] Use poetry lockfile in twisted trunk CI job (#12425) Fixes #12458 Co-authored-by: Sean Quah Co-authored-by: Dan Callahan --- .ci/patch_for_twisted_trunk.sh | 8 ----- .github/workflows/twisted_trunk.yml | 46 +++++++++++++++++++++-------- changelog.d/12425.misc | 1 + 3 files changed, 35 insertions(+), 20 deletions(-) delete mode 100755 .ci/patch_for_twisted_trunk.sh create mode 100644 changelog.d/12425.misc diff --git a/.ci/patch_for_twisted_trunk.sh b/.ci/patch_for_twisted_trunk.sh deleted file mode 100755 index f524581986a9..000000000000 --- a/.ci/patch_for_twisted_trunk.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# replaces the dependency on Twisted in `python_dependencies` with trunk. - -set -e -cd "$(dirname "$0")"/.. - -sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index fb9d46b7bfdd..8fc1affb7746 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -6,16 +6,25 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e mypy + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + extras: "all" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run mypy trial: runs-on: ubuntu-latest @@ -23,14 +32,15 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - uses: actions/setup-python@v2 + - uses: matrix-org/setup-python-poetry@v1 with: - python-version: 3.7 - - run: .ci/patch_for_twisted_trunk.sh - - run: pip install tox - - run: tox -e py - env: - TRIAL_FLAGS: "--jobs=2" + python-version: "3.x" + extras: "all test" + - run: | + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" + - run: poetry run trial --jobs 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. @@ -55,11 +65,23 @@ jobs: steps: - uses: actions/checkout@v2 - name: Patch dependencies - run: .ci/patch_for_twisted_trunk.sh + # Note: The poetry commands want to create a virtualenv in /src/.venv/, + # but the sytest-synapse container expects it to be in /venv/. + # We symlink it before running poetry so that poetry actually + # ends up installing to `/venv`. + run: | + ln -s -T /venv /src/.venv + poetry remove twisted + poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry install --no-interaction --extras "all test" working-directory: /src - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src + env: + # Use offline mode to avoid reinstalling the pinned version of + # twisted. + OFFLINE: 1 - name: Summarise results.tap if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc new file mode 100644 index 000000000000..3b076be0bd52 --- /dev/null +++ b/changelog.d/12425.misc @@ -0,0 +1 @@ +Run twisted trunk CI job in the locked poetry environment. From e3a49f4784d5c915355ac9306e60b09433db60b5 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 13 Apr 2022 11:38:35 +0100 Subject: [PATCH 012/263] Fix missing sync events during historical batch imports (#12319) Discovered after much in-depth investigation in #12281. Closes: #12281 Closes: #3305 Signed off by: Nick Mills-Barrett nick@beeper.com --- changelog.d/12319.bugfix | 1 + synapse/handlers/message.py | 14 +-- synapse/handlers/sync.py | 15 ++- synapse/storage/databases/main/stream.py | 26 +++++ tests/rest/client/test_room_batch.py | 125 ++++++++++++++++++++++- 5 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 changelog.d/12319.bugfix diff --git a/changelog.d/12319.bugfix b/changelog.d/12319.bugfix new file mode 100644 index 000000000000..a50191feaaaf --- /dev/null +++ b/changelog.d/12319.bugfix @@ -0,0 +1 @@ +Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 47a63005a984..1b092e900eb5 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -175,17 +175,13 @@ async def get_state_events( state_filter = state_filter or StateFilter.all() if at_token: - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=at_token.room_key, limit=1 + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=at_token.room_key, ) - if not last_events: + if not last_event: raise NotFoundError("Can't find event for token %s" % (at_token,)) - last_event = last_events[0] # check whether the user is in the room at that time to determine # whether they should be treated as peeking. @@ -204,7 +200,7 @@ async def get_state_events( visible_events = await filter_events_for_client( self.storage, user_id, - last_events, + [last_event], filter_send_to_client=False, is_peeking=is_peeking, ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 6c8b17c4205d..5125126a807c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -661,16 +661,15 @@ async def get_state_at( stream_position: point at which to get state state_filter: The state filter used to fetch state from the database. """ - # FIXME this claims to get the state at a stream position, but - # get_recent_events_for_room operates by topo ordering. This therefore - # does not reliably give you the state at the given stream position. - # (https://github.com/matrix-org/synapse/issues/3305) - last_events, _ = await self.store.get_recent_events_for_room( - room_id, end_token=stream_position.room_key, limit=1 + # FIXME: This gets the state at the latest event before the stream ordering, + # which might not be the same as the "current state" of the room at the time + # of the stream token if there were multiple forward extremities at the time. + last_event = await self.store.get_last_event_in_room_before_stream_ordering( + room_id, + end_token=stream_position.room_key, ) - if last_events: - last_event = last_events[-1] + if last_event: state = await self.get_state_after_event( last_event, state_filter=state_filter or StateFilter.all() ) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 6d45a8a9f6cd..793e906630e8 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -758,6 +758,32 @@ def _f(txn: LoggingTransaction) -> Optional[Tuple[int, int, str]]: "get_room_event_before_stream_ordering", _f ) + async def get_last_event_in_room_before_stream_ordering( + self, + room_id: str, + end_token: RoomStreamToken, + ) -> Optional[EventBase]: + """Returns the last event in a room at or before a stream ordering + + Args: + room_id + end_token: The token used to stream from + + Returns: + The most recent event. + """ + + last_row = await self.get_room_event_before_stream_ordering( + room_id=room_id, + stream_ordering=end_token.stream, + ) + if last_row: + _, _, event_id = last_row + event = await self.get_event(event_id, get_prev_content=True) + return event + + return None + async def get_current_room_stream_token_for_room_id( self, room_id: Optional[str] = None ) -> RoomStreamToken: diff --git a/tests/rest/client/test_room_batch.py b/tests/rest/client/test_room_batch.py index 44f333a0ee77..41a1bf6d890e 100644 --- a/tests/rest/client/test_room_batch.py +++ b/tests/rest/client/test_room_batch.py @@ -7,9 +7,9 @@ from synapse.api.constants import EventContentFields, EventTypes from synapse.appservice import ApplicationService from synapse.rest import admin -from synapse.rest.client import login, register, room, room_batch +from synapse.rest.client import login, register, room, room_batch, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, RoomStreamToken from synapse.util import Clock from tests import unittest @@ -63,6 +63,7 @@ class RoomBatchTestCase(unittest.HomeserverTestCase): room.register_servlets, register.register_servlets, login.register_servlets, + sync.register_servlets, ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: @@ -178,3 +179,123 @@ def test_same_state_groups_for_whole_historical_batch(self) -> None: "Expected a single state_group to be returned by saw state_groups=%s" % (state_group_map.keys(),), ) + + @unittest.override_config({"experimental_features": {"msc2716_enabled": True}}) + def test_sync_while_batch_importing(self) -> None: + """ + Make sure that /sync correctly returns full room state when a user joins + during ongoing batch backfilling. + See: https://github.com/matrix-org/synapse/issues/12281 + """ + # Create user who will be invited & join room + user_id = self.register_user("beep", "test") + user_tok = self.login("beep", "test") + + time_before_room = int(self.clock.time_msec()) + + # Create a room with some events + room_id, _, _, _ = self._create_test_room() + # Invite the user + self.helper.invite( + room_id, src=self.appservice.sender, tok=self.appservice.token, targ=user_id + ) + + # Create another room, send a bunch of events to advance the stream token + other_room_id = self.helper.create_room_as( + self.appservice.sender, tok=self.appservice.token + ) + for _ in range(5): + self.helper.send_event( + room_id=other_room_id, + type=EventTypes.Message, + content={"msgtype": "m.text", "body": "C"}, + tok=self.appservice.token, + ) + + # Join the room as the normal user + self.helper.join(room_id, user_id, tok=user_tok) + + # Create an event to hang the historical batch from - In order to see + # the failure case originally reported in #12281, the historical batch + # must be hung from the most recent event in the room so the base + # insertion event ends up with the highest `topogological_ordering` + # (`depth`) in the room but will have a negative `stream_ordering` + # because it's a `historical` event. Previously, when assembling the + # `state` for the `/sync` response, the bugged logic would sort by + # `topological_ordering` descending and pick up the base insertion + # event because it has a negative `stream_ordering` below the given + # pagination token. Now we properly sort by `stream_ordering` + # descending which puts `historical` events with a negative + # `stream_ordering` way at the bottom and aren't selected as expected. + response = self.helper.send_event( + room_id=room_id, + type=EventTypes.Message, + content={ + "msgtype": "m.text", + "body": "C", + }, + tok=self.appservice.token, + ) + event_to_hang_id = response["event_id"] + + channel = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2716/rooms/%s/batch_send?prev_event_id=%s" + % (room_id, event_to_hang_id), + content={ + "events": _create_message_events_for_batch_send_request( + self.virtual_user_id, time_before_room, 3 + ), + "state_events_at_start": _create_join_state_events_for_batch_send_request( + [self.virtual_user_id], time_before_room + ), + }, + access_token=self.appservice.token, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Now we need to find the invite + join events stream tokens so we can sync between + main_store = self.hs.get_datastores().main + events, next_key = self.get_success( + main_store.get_recent_events_for_room( + room_id, + 50, + end_token=main_store.get_room_max_token(), + ), + ) + invite_event_position = None + for event in events: + if ( + event.type == "m.room.member" + and event.content["membership"] == "invite" + ): + invite_event_position = self.get_success( + main_store.get_topological_token_for_event(event.event_id) + ) + break + + assert invite_event_position is not None, "No invite event found" + + # Remove the topological order from the token by re-creating w/stream only + invite_event_position = RoomStreamToken(None, invite_event_position.stream) + + # Sync everything after this token + since_token = self.get_success(invite_event_position.to_string(main_store)) + sync_response = self.make_request( + "GET", + f"/sync?since={since_token}", + access_token=user_tok, + ) + + # Assert that, for this room, the user was considered to have joined and thus + # receives the full state history + state_event_types = [ + event["type"] + for event in sync_response.json_body["rooms"]["join"][room_id]["state"][ + "events" + ] + ] + + assert ( + "m.room.create" in state_event_types + ), "Missing room full state in sync response" From 73d8ded0b030a81e828c07bb134c08db67569e5d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Apr 2022 16:21:07 +0100 Subject: [PATCH 013/263] Prevent a sync request from removing a user's busy presence status (#12213) In trying to use the MSC3026 busy presence status, the user's status would be set back to 'online' next time they synced. This change makes it so that syncing does not affect a user's presence status if it is currently set to 'busy': it must be removed through the presence API. The MSC defers to implementations on the behaviour of busy presence, so this ought to remain compatible with the MSC. --- changelog.d/12213.bugfix | 1 + synapse/handlers/events.py | 6 ++- synapse/handlers/presence.py | 56 ++++++++++++++++++----- synapse/rest/client/sync.py | 9 ++-- tests/handlers/test_presence.py | 79 +++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 changelog.d/12213.bugfix diff --git a/changelog.d/12213.bugfix b/changelog.d/12213.bugfix new file mode 100644 index 000000000000..9278e3a9c163 --- /dev/null +++ b/changelog.d/12213.bugfix @@ -0,0 +1 @@ +Prevent a sync request from removing a user's busy presence status. diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index d2ccb5c5d311..e89c4df31466 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -16,7 +16,7 @@ import random from typing import TYPE_CHECKING, Iterable, List, Optional -from synapse.api.constants import EduTypes, EventTypes, Membership +from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig @@ -67,7 +67,9 @@ async def get_stream( presence_handler = self.hs.get_presence_handler() context = await presence_handler.user_syncing( - auth_user_id, affect_presence=affect_presence + auth_user_id, + affect_presence=affect_presence, + presence_state=PresenceState.ONLINE, ) with context: if timeout: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 209a4b0e521f..d078162c2938 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -151,7 +151,7 @@ def __init__(self, hs: "HomeServer"): @abc.abstractmethod async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -165,6 +165,7 @@ async def user_syncing( affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ @abc.abstractmethod @@ -228,6 +229,11 @@ async def current_state_for_users( return states + async def current_state_for_user(self, user_id: str) -> UserPresenceState: + """Get the current presence state for a user.""" + res = await self.current_state_for_users([user_id]) + return res[user_id] + @abc.abstractmethod async def set_state( self, @@ -461,7 +467,7 @@ def send_stop_syncing(self) -> None: self.send_user_sync(user_id, False, last_sync_ms) async def user_syncing( - self, user_id: str, affect_presence: bool + self, user_id: str, affect_presence: bool, presence_state: str ) -> ContextManager[None]: """Record that a user is syncing. @@ -471,6 +477,17 @@ async def user_syncing( if not affect_presence or not self._presence_enabled: return _NullContextManager() + prev_state = await self.current_state_for_user(user_id) + if prev_state != PresenceState.BUSY: + # We set state here but pass ignore_status_msg = True as we don't want to + # cause the status message to be cleared. + # Note that this causes last_active_ts to be incremented which is not + # what the spec wants: see comment in the BasePresenceHandler version + # of this function. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + curr_sync = self._user_to_num_current_syncs.get(user_id, 0) self._user_to_num_current_syncs[user_id] = curr_sync + 1 @@ -942,7 +959,10 @@ async def bump_presence_active_time(self, user: UserID) -> None: await self._update_states([prev_state.copy_and_replace(**new_fields)]) async def user_syncing( - self, user_id: str, affect_presence: bool = True + self, + user_id: str, + affect_presence: bool = True, + presence_state: str = PresenceState.ONLINE, ) -> ContextManager[None]: """Returns a context manager that should surround any stream requests from the user. @@ -956,6 +976,7 @@ async def user_syncing( affect_presence: If false this function will be a no-op. Useful for streams that are not associated with an actual client that is being used by a user. + presence_state: The presence state indicated in the sync request """ # Override if it should affect the user's presence, if presence is # disabled. @@ -967,9 +988,25 @@ async def user_syncing( self.user_to_num_current_syncs[user_id] = curr_sync + 1 prev_state = await self.current_state_for_user(user_id) + + # If they're busy then they don't stop being busy just by syncing, + # so just update the last sync time. + if prev_state.state != PresenceState.BUSY: + # XXX: We set_state separately here and just update the last_active_ts above + # This keeps the logic as similar as possible between the worker and single + # process modes. Using set_state will actually cause last_active_ts to be + # updated always, which is not what the spec calls for, but synapse has done + # this for... forever, I think. + await self.set_state( + UserID.from_string(user_id), {"presence": presence_state}, True + ) + # Retrieve the new state for the logic below. This should come from the + # in-memory cache. + prev_state = await self.current_state_for_user(user_id) + + # To keep the single process behaviour consistent with worker mode, run the + # same logic as `update_external_syncs_row`, even though it looks weird. if prev_state.state == PresenceState.OFFLINE: - # If they're currently offline then bring them online, otherwise - # just update the last sync times. await self._update_states( [ prev_state.copy_and_replace( @@ -979,6 +1016,10 @@ async def user_syncing( ) ] ) + # otherwise, set the new presence state & update the last sync time, + # but don't update last_active_ts as this isn't an indication that + # they've been active (even though it's probably been updated by + # set_state above) else: await self._update_states( [ @@ -1086,11 +1127,6 @@ async def update_external_syncs_clear(self, process_id: str) -> None: ) self.external_process_last_updated_ms.pop(process_id, None) - async def current_state_for_user(self, user_id: str) -> UserPresenceState: - """Get the current presence state for a user.""" - res = await self.current_state_for_users([user_id]) - return res[user_id] - async def _persist_and_notify(self, states: List[UserPresenceState]) -> None: """Persist states in the database, poke the notifier and send to interested remote servers diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 2e25e8638b9f..e8772f86e72f 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -180,13 +180,10 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: affect_presence = set_presence != PresenceState.OFFLINE - if affect_presence: - await self.presence_handler.set_state( - user, {"presence": set_presence}, True - ) - context = await self.presence_handler.user_syncing( - user.to_string(), affect_presence=affect_presence + user.to_string(), + affect_presence=affect_presence, + presence_state=set_presence, ) with context: sync_result = await self.sync_handler.wait_for_sync_for_user( diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index b2ed9cbe3775..c96dc6caf2d5 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -657,6 +657,85 @@ def test_set_presence_with_status_msg_none(self): # Mark user as online and `status_msg = None` self._set_presencestate_with_status_msg(user_id, PresenceState.ONLINE, None) + def test_set_presence_from_syncing_not_set(self): + """Test that presence is not set by syncing if affect_presence is false""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, False, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should still be unavailable + self.assertEqual(state.state, PresenceState.UNAVAILABLE) + # and status message should still be the same + self.assertEqual(state.status_msg, status_msg) + + def test_set_presence_from_syncing_is_set(self): + """Test that presence is set by syncing if affect_presence is true""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should now be online + self.assertEqual(state.state, PresenceState.ONLINE) + + def test_set_presence_from_syncing_keeps_status(self): + """Test that presence set by syncing retains status message""" + user_id = "@test:server" + status_msg = "I'm here!" + + self._set_presencestate_with_status_msg( + user_id, PresenceState.UNAVAILABLE, status_msg + ) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # our status message should be the same as it was before + self.assertEqual(state.status_msg, status_msg) + + def test_set_presence_from_syncing_keeps_busy(self): + """Test that presence set by syncing doesn't affect busy status""" + # while this isn't the default + self.presence_handler._busy_presence_enabled = True + + user_id = "@test:server" + status_msg = "I'm busy!" + + self._set_presencestate_with_status_msg(user_id, PresenceState.BUSY, status_msg) + + self.get_success( + self.presence_handler.user_syncing(user_id, True, PresenceState.ONLINE) + ) + + state = self.get_success( + self.presence_handler.get_state(UserID.from_string(user_id)) + ) + # we should still be busy + self.assertEqual(state.state, PresenceState.BUSY) + def _set_presencestate_with_status_msg( self, user_id: str, state: str, status_msg: Optional[str] ): From 0922462fc7df951e88c8ec0fb35e53e3cd801b76 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 13 Apr 2022 08:27:51 -0700 Subject: [PATCH 014/263] docs: Don't render the table of contents on the print page (#12340) --- changelog.d/12340.doc | 1 + docs/website_files/table-of-contents.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 changelog.d/12340.doc diff --git a/changelog.d/12340.doc b/changelog.d/12340.doc new file mode 100644 index 000000000000..8354f2259e0c --- /dev/null +++ b/changelog.d/12340.doc @@ -0,0 +1 @@ +Fix rendering of the documentation site when using the 'print' feature. diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js index 0de5960b22b1..772da97fb9de 100644 --- a/docs/website_files/table-of-contents.js +++ b/docs/website_files/table-of-contents.js @@ -75,6 +75,20 @@ function setTocEntry() { * Populate sidebar on load */ window.addEventListener('load', () => { + // Prevent rendering the table of contents of the "print book" page, as it + // will end up being rendered into the output (in a broken-looking way) + + // Get the name of the current page (i.e. 'print.html') + const pageNameExtension = window.location.pathname.split('/').pop(); + + // Split off the extension (as '.../print' is also a valid page name), which + // should result in 'print' + const pageName = pageNameExtension.split('.')[0]; + if (pageName === "print") { + // Don't render the table of contents on this page + return; + } + // Only create table of contents if there is more than one header on the page if (headers.length <= 1) { return; From 8e2759f2d86d68fa621ba51ae73171e25fe9510d Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 13 Apr 2022 10:04:01 -0700 Subject: [PATCH 015/263] Limit `device_id` size to 512B (#12454) * --- changelog.d/12454.misc | 1 + synapse/rest/client/login.py | 9 +++++++++ tests/rest/client/test_login.py | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12454.misc diff --git a/changelog.d/12454.misc b/changelog.d/12454.misc new file mode 100644 index 000000000000..cb7ff74b4c8f --- /dev/null +++ b/changelog.d/12454.misc @@ -0,0 +1 @@ +Limit length of device_id to less than 512 characters. diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index c9d44c5964ce..4a4dbe75de6b 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -342,6 +342,15 @@ async def _complete_login( user_id = canonical_uid device_id = login_submission.get("device_id") + + # If device_id is present, check that device_id is not longer than a reasonable 512 characters + if device_id and len(device_id) > 512: + raise LoginError( + 400, + "device_id cannot be longer than 512 characters.", + errcode=Codes.INVALID_PARAM, + ) + initial_display_name = login_submission.get("initial_device_display_name") ( device_id, diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index 090d2d0a2927..0a3d017dc9b9 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import json import time import urllib.parse from typing import Any, Dict, List, Optional, Union @@ -384,6 +384,31 @@ def test_session_can_hard_logout_all_sessions_after_being_soft_logged_out( channel = self.make_request(b"POST", "/logout/all", access_token=access_token) self.assertEqual(channel.result["code"], b"200", channel.result) + def test_login_with_overly_long_device_id_fails(self) -> None: + self.register_user("mickey", "cheese") + + # create a device_id longer than 512 characters + device_id = "yolo" * 512 + + body = { + "type": "m.login.password", + "user": "mickey", + "password": "cheese", + "device_id": device_id, + } + + # make a login request with the bad device_id + channel = self.make_request( + "POST", + "/_matrix/client/v3/login", + json.dumps(body).encode("utf8"), + custom_headers=None, + ) + + # test that the login fails with the correct error code + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + @skip_unless(has_saml2 and HAS_OIDC, "Requires SAML2 and OIDC") class MultiSSOTestCase(unittest.HomeserverTestCase): From 8af8a9bce5319ebf99573433f9f89d94767e173c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 13 Apr 2022 20:50:08 +0100 Subject: [PATCH 016/263] Dockerfile-workers: give the master its own log config (#12466) When we run a worker-mode synapse under docker, everything gets logged to stdout. Currently, output from the workers is tacked with a worker name, for example: ``` 2022-04-13 15:27:56,810 - worker:frontend_proxy1 - synapse.util.caches.lrucache - 154 - INFO - LruCache._expire_old_entries-0 - Dropped 0 items from caches ``` - note `worker:frontend_proxy1`. No such tag is applied to log lines from the master, which makes somewhat confusing reading. To fix this, we generate a dedicated log config file for the master in the same way that we do for the workers, and use that. --- changelog.d/12466.misc | 1 + docker/configure_workers_and_start.py | 48 +++++++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 changelog.d/12466.misc diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc new file mode 100644 index 000000000000..b0c2c950fe55 --- /dev/null +++ b/changelog.d/12466.misc @@ -0,0 +1 @@ +Dockerfile-workers: give the master its own log config. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3e91024e8c92..8f1e61930e65 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Set +from typing import Any, Dict, Mapping, Set import jinja2 import yaml @@ -341,7 +341,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # base shared worker jinja2 template. # # This config file will be passed to all workers, included Synapse's main process. - shared_config = {"listeners": listeners} + shared_config: Dict[str, Any] = {"listeners": listeners} # The supervisord config. The contents of which will be inserted into the # base supervisord jinja2 template. @@ -446,21 +446,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Write out the worker's logging config file - # Check whether we should write worker logs to disk, in addition to the console - extra_log_template_args = {} - if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): - extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( - dir=data_dir, name=worker_name - ) - - # Render and write the file - log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) - convert( - "/conf/log.config", - log_config_filepath, - worker_name=worker_name, - **extra_log_template_args, - ) + log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir) # Then a worker config file convert( @@ -496,6 +482,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Finally, we'll write out the config files. + # log config for the master process + master_log_config = generate_worker_log_config(environ, "master", data_dir) + shared_config["log_config"] = master_log_config + # Shared homeserver config convert( "/conf/shared.yaml.j2", @@ -532,6 +522,30 @@ def generate_worker_files(environ, config_path: str, data_dir: str): os.mkdir(log_dir) +def generate_worker_log_config( + environ: Mapping[str, str], worker_name: str, data_dir: str +) -> str: + """Generate a log.config file for the given worker. + + Returns: the path to the generated file + """ + # Check whether we should write worker logs to disk, in addition to the console + extra_log_template_args = {} + if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): + extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( + dir=data_dir, name=worker_name + ) + # Render and write the file + log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) + convert( + "/conf/log.config", + log_config_filepath, + worker_name=worker_name, + **extra_log_template_args, + ) + return log_config_filepath + + def start_supervisord(): """Starts up supervisord which then starts and monitors all other necessary processes From 6b3e0ea6bdff15aa24a42f1f5938785460a323a7 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Apr 2022 11:03:24 +0100 Subject: [PATCH 017/263] Use `poetry` to manage the virtualenv in debian packages (#12449) * Use `poetry` to build venv in debian packages Co-authored-by: Dan Callahan Co-authored-by: Shay * Changelog * Only pull in from requirements.txt Addresses the same problem as #12439. * Include `test` and `all` extras `poetry export` helpfully silently ignores an unknown extra Haven't seen this before because it's the only place we export `all` and `test`. I could have __sworm__ that the syntax `--extra "all test"` worked for `poetry install`... * Clean up requirements file on subsequence builds * Fix shell syntax Co-authored-by: Dan Callahan Co-authored-by: Shay --- changelog.d/12449.misc | 1 + debian/build_virtualenv | 20 ++++++++++++++++---- debian/changelog | 7 +++++++ debian/clean | 1 + 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12449.misc create mode 100644 debian/clean diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc new file mode 100644 index 000000000000..03e08aace427 --- /dev/null +++ b/changelog.d/12449.misc @@ -0,0 +1 @@ +Use `poetry` to manage the virtualenv in debian packages. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index e6911636192c..b068792592d6 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in ;; esac -# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather -# than the 2/3 compatible `virtualenv`. - +# Manually install Poetry and export a pip-compatible `requirements.txt` +# We need a Poetry pre-release as the export command is buggy in < 1.2 +TEMP_VENV="$(mktemp -d)" +python3 -m venv "$TEMP_VENV" +source "$TEMP_VENV/bin/activate" +pip install -U pip +pip install poetry==1.2.0b1 +poetry export --extras all --extras test -o exported_requirements.txt +deactivate +rm -rf "$TEMP_VENV" + +# Use --no-deps to only install pinned versions in exported_requirements.txt, +# and to avoid https://github.com/pypa/pip/issues/9644 dh_virtualenv \ --install-suffix "matrix-synapse" \ --builtin-venv \ @@ -41,9 +51,11 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ + --extra-pip-arg="--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ - --extras="all,systemd,test" + --extras="all,systemd,test" \ + --requirements="exported_requirements.txt" PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" diff --git a/debian/changelog b/debian/changelog index 67215aa0806b..fdbc92df6bb4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium + + * Non-maintainer upload. + * Use poetry to manage the bundled virtualenv included with this package. + + -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium * New synapse release 1.57.0~rc1. diff --git a/debian/clean b/debian/clean new file mode 100644 index 000000000000..d488f298d587 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +exported_requirements.txt From 535a689cfcec896dd4ce74ef567bd30c11ade9e3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 14 Apr 2022 11:33:06 +0100 Subject: [PATCH 018/263] Reintroduce the lint targets in the linter script (#12455) --- changelog.d/12455.misc | 1 + docs/code_style.md | 54 ++++++++++++------------------------------ scripts-dev/lint.sh | 16 +++++++++++-- 3 files changed, 30 insertions(+), 41 deletions(-) create mode 100644 changelog.d/12455.misc diff --git a/changelog.d/12455.misc b/changelog.d/12455.misc new file mode 100644 index 000000000000..9b19945673e4 --- /dev/null +++ b/changelog.d/12455.misc @@ -0,0 +1 @@ +Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. diff --git a/docs/code_style.md b/docs/code_style.md index ebda6dcc85f4..db7edcd76b69 100644 --- a/docs/code_style.md +++ b/docs/code_style.md @@ -6,60 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to quickly and automatically check for formatting (and sometimes logical) errors in code. -The necessary tools are detailed below. +The necessary tools are: -First install them with: +- [black](https://black.readthedocs.io/en/stable/), a source code formatter; +- [isort](https://pycqa.github.io/isort/), which organises each file's imports; +- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and +- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker. + +Install them with: ```sh pip install -e ".[lint,mypy]" ``` -- **black** - - The Synapse codebase uses [black](https://pypi.org/project/black/) - as an opinionated code formatter, ensuring all comitted code is - properly formatted. - - Have `black` auto-format your code (it shouldn't change any - functionality) with: - - ```sh - black . - ``` - -- **flake8** - - `flake8` is a code checking tool. We require code to pass `flake8` - before being merged into the codebase. - - Check all application and test code with: +The easiest way to run the lints is to invoke the linter script as follows. - ```sh - flake8 . - ``` - -- **isort** - - `isort` ensures imports are nicely formatted, and can suggest and - auto-fix issues such as double-importing. - - Auto-fix imports with: - - ```sh - isort . - ``` +```sh +scripts-dev/lint.sh +``` It's worth noting that modern IDEs and text editors can run these tools automatically on save. It may be worth looking into whether this functionality is supported in your editor for a more convenient -development workflow. It is not, however, recommended to run `flake8` on -save as it takes a while and is very resource intensive. +development workflow. It is not, however, recommended to run `flake8` or `mypy` +on save as they take a while and can be very resource intensive. ## General rules - **Naming**: - - Use camel case for class and type names - - Use underscores for functions and variables. + - Use `CamelCase` for class and type names + - Use underscores for `function_names` and `variable_names`. - **Docstrings**: should follow the [google code style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). See the diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 4698d2d5be32..91a704d98247 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -79,8 +79,20 @@ else # If we were not asked to lint changed files, and no paths were found as a result, # then lint everything! if [[ -z ${files+x} ]]; then - # Lint all source code files and directories - files=( "." ) + # CI runs each linter on the entire checkout, e.g. `black .`. So don't + # rely on this list to *find* lint targets if that misses a file; instead; + # use it to exclude files from linters when this can't be done by config. + # + # To check which files the linters examine, use: + # black --verbose . 2>&1 | \grep -v ignored + # isort --show-files . + # flake8 --verbose . # This isn't a great option + # mypy has explicit config in mypy.ini; there is also mypy --verbose + files=( + "synapse" "docker" "tests" + "scripts-dev" + "contrib" "setup.py" "synmark" "stubs" ".ci" + ) fi fi From 0b014eb25e8c0766d2552fae2ba366492dc16d4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Apr 2022 13:05:31 +0100 Subject: [PATCH 019/263] Only send out device list updates for our own users (#12465) Broke in #12365 --- changelog.d/12465.feature | 1 + synapse/handlers/device.py | 10 +++-- synapse/storage/databases/main/devices.py | 4 +- tests/federation/test_federation_sender.py | 43 +++++++++++++++++++++- tests/storage/test_devices.py | 6 +-- 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 changelog.d/12465.feature diff --git a/changelog.d/12465.feature b/changelog.d/12465.feature new file mode 100644 index 000000000000..642dea966c44 --- /dev/null +++ b/changelog.d/12465.feature @@ -0,0 +1 @@ +Enable processing of device list updates asynchronously. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 958599e7b816..3c0fc756d46b 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -649,9 +649,13 @@ async def _handle_new_device_update_async(self) -> None: return for user_id, device_id, room_id, stream_id, opentracing_context in rows: - joined_user_ids = await self.store.get_users_in_room(room_id) - hosts = {get_domain_from_id(u) for u in joined_user_ids} - hosts.discard(self.server_name) + hosts = set() + + # Ignore any users that aren't ours + if self.hs.is_mine_id(user_id): + joined_user_ids = await self.store.get_users_in_room(room_id) + hosts = {get_domain_from_id(u) for u in joined_user_ids} + hosts.discard(self.server_name) # Check if we've already sent this update to some hosts if current_stream_id == stream_id: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 74e4e2122a20..318e4df376b2 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1703,7 +1703,9 @@ def _add_device_outbound_poke_to_stream_txn( next(stream_id_iterator), user_id, device_id, - False, + not self.hs.is_mine_id( + user_id + ), # We only need to send out update for *our* users now, encoded_context if whitelisted_homeserver(destination) else "{}", ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 63ea4f9ee475..91f982518e69 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -162,7 +162,9 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): def make_homeserver(self, reactor, clock): return self.setup_test_homeserver( - federation_transport_client=Mock(spec=["send_transaction"]), + federation_transport_client=Mock( + spec=["send_transaction", "query_user_devices"] + ), ) def default_config(self): @@ -218,6 +220,45 @@ def test_send_device_updates(self): self.assertEqual(len(self.edus), 1) self.check_device_update_edu(self.edus.pop(0), u1, "D2", stream_id) + def test_dont_send_device_updates_for_remote_users(self): + """Check that we don't send device updates for remote users""" + + # Send the server a device list EDU for the other user, this will cause + # it to try and resync the device lists. + self.hs.get_federation_transport_client().query_user_devices.return_value = ( + defer.succeed( + { + "stream_id": "1", + "user_id": "@user2:host2", + "devices": [{"device_id": "D1"}], + } + ) + ) + + self.get_success( + self.hs.get_device_handler().device_list_updater.incoming_device_list_update( + "host2", + { + "user_id": "@user2:host2", + "device_id": "D1", + "stream_id": "1", + "prev_ids": [], + }, + ) + ) + + self.reactor.advance(1) + + # We shouldn't see an EDU for that update + self.assertEqual(self.edus, []) + + # Check that we did successfully process the inbound EDU (otherwise this + # test would pass if we failed to process the EDU) + devices = self.get_success( + self.hs.get_datastores().main.get_cached_devices_for_user("@user2:host2") + ) + self.assertIn("D1", devices) + def test_upload_signatures(self): """Uploading signatures on some devices should produce updates for that user""" diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index 5491fbf6da7b..ccc3893869db 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -118,7 +118,7 @@ def test_get_device_updates_by_remote(self): device_ids = ["device_id1", "device_id2"] # Add two device updates with sequential `stream_id`s - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get all device updates ever meant for this remote now_stream_id, device_updates = self.get_success( @@ -142,7 +142,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): "device_id4", "device_id5", ] - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get device updates meant for this remote next_stream_id, device_updates = self.get_success( @@ -162,7 +162,7 @@ def test_get_device_updates_by_remote_can_limit_properly(self): # Add some more device updates to ensure it still resumes properly device_ids = ["device_id6", "device_id7"] - self.add_device_change("user_id", device_ids, "somehost") + self.add_device_change("@user_id:test", device_ids, "somehost") # Get the next batch of device updates next_stream_id, device_updates = self.get_success( From a743f7d33ebbd383fbe9bf70e024db79525d891c Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 14 Apr 2022 14:09:07 +0200 Subject: [PATCH 020/263] Replace `federation_reader` with `generic_worker` in docs (#12457) --- changelog.d/12457.doc | 1 + docs/systemd-with-workers/README.md | 10 +++++----- .../workers/federation_reader.yaml | 13 ------------- .../workers/generic_worker.yaml | 13 +++++++++++++ docs/workers.md | 10 ++++------ 5 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12457.doc delete mode 100644 docs/systemd-with-workers/workers/federation_reader.yaml create mode 100644 docs/systemd-with-workers/workers/generic_worker.yaml diff --git a/changelog.d/12457.doc b/changelog.d/12457.doc new file mode 100644 index 000000000000..a4871622cf76 --- /dev/null +++ b/changelog.d/12457.doc @@ -0,0 +1 @@ +Update worker documentation and replace old `federation_reader` with `generic_worker`. \ No newline at end of file diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md index b160d9352840..d516501085bf 100644 --- a/docs/systemd-with-workers/README.md +++ b/docs/systemd-with-workers/README.md @@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/ for the systemd unit files. The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/) -contains an example configuration for the `federation_reader` worker. +contains an example configuration for the `generic_worker` worker. ## Synapse configuration files See [the worker documentation](../workers.md) for information on how to set up the configuration files and reverse-proxy correctly. -Below is a sample `federation_reader` worker configuration file. +Below is a sample `generic_worker` worker configuration file. ```yaml -{{#include workers/federation_reader.yaml}} +{{#include workers/generic_worker.yaml}} ``` Systemd manages daemonization itself, so ensure that none of the configuration @@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target # Restart the master alone systemctl start matrix-synapse.service -# Restart a specific worker (eg. federation_reader); the master is +# Restart a specific worker (eg. generic_worker); the master is # unaffected by this. -systemctl restart matrix-synapse-worker@federation_reader.service +systemctl restart matrix-synapse-worker@generic_worker.service # Add a new worker (assuming all configs are set up already) systemctl enable matrix-synapse-worker@federation_writer.service diff --git a/docs/systemd-with-workers/workers/federation_reader.yaml b/docs/systemd-with-workers/workers/federation_reader.yaml deleted file mode 100644 index 13e69e62c9db..000000000000 --- a/docs/systemd-with-workers/workers/federation_reader.yaml +++ /dev/null @@ -1,13 +0,0 @@ -worker_app: synapse.app.federation_reader -worker_name: federation_reader1 - -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8011 - resources: - - names: [federation] - -worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml new file mode 100644 index 000000000000..8561e2cda59c --- /dev/null +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -0,0 +1,13 @@ +worker_app: synapse.app.generic_worker +worker_name: generic_worker1 + +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8011 + resources: + - names: [client, federation] + +worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml diff --git a/docs/workers.md b/docs/workers.md index caef44b614de..446b7e5064e2 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -146,12 +146,10 @@ worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - - type: http - port: 8083 - resources: - - names: - - client - - federation + - type: http + port: 8083 + resources: + - names: [client, federation] worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml ``` From 960b4fb40927d890b2db32d4ef83a47a52baaf23 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 14 Apr 2022 14:56:10 +0100 Subject: [PATCH 021/263] complement-synapse-workers: factor out separate entry point script (#12467) ... with a bit more verbosity. --- changelog.d/12467.misc | 1 + docker/complement/SynapseWorkers.Dockerfile | 40 +++------------ .../start-complement-synapse-workers.sh | 50 +++++++++++++++++++ 3 files changed, 58 insertions(+), 33 deletions(-) create mode 100644 changelog.d/12467.misc create mode 100755 docker/complement/conf-workers/start-complement-synapse-workers.sh diff --git a/changelog.d/12467.misc b/changelog.d/12467.misc new file mode 100644 index 000000000000..fbf415f7072b --- /dev/null +++ b/changelog.d/12467.misc @@ -0,0 +1 @@ +complement-synapse-workers: factor out separate entry point script. diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 982219a91e7e..65df2d114d58 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -13,8 +13,8 @@ RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/cadd tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root # Install postgresql -RUN apt-get update -RUN apt-get install -y postgresql +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13 # Configure a user and create a database for Synapse RUN pg_ctlcluster 13 main start && su postgres -c "echo \ @@ -34,40 +34,14 @@ WORKDIR /data # Copy the caddy config COPY conf-workers/caddy.complement.json /root/caddy.json +# Copy the entrypoint +COPY conf-workers/start-complement-synapse-workers.sh / + # Expose caddy's listener ports EXPOSE 8008 8448 -ENTRYPOINT \ - # Replace the server name in the caddy config - sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \ - # Start postgres - pg_ctlcluster 13 main start 2>&1 && \ - # Start caddy - /root/caddy start --config /root/caddy.json 2>&1 && \ - # Set the server name of the homeserver - SYNAPSE_SERVER_NAME=${SERVER_NAME} \ - # No need to report stats here - SYNAPSE_REPORT_STATS=no \ - # Set postgres authentication details which will be placed in the homeserver config file - POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ - # Specify the workers to test with - SYNAPSE_WORKER_TYPES="\ - event_persister, \ - event_persister, \ - background_worker, \ - frontend_proxy, \ - event_creator, \ - user_dir, \ - media_repository, \ - federation_inbound, \ - federation_reader, \ - federation_sender, \ - synchrotron, \ - appservice, \ - pusher" \ - # Run the script that writes the necessary config files and starts supervisord, which in turn - # starts everything else - /configure_workers_and_start.py +ENTRYPOINT /start-complement-synapse-workers.sh +# Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh new file mode 100755 index 000000000000..2c1e05bd6221 --- /dev/null +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement + +set -e + +function log { + d=$(date +"%Y-%m-%d %H:%M:%S,%3N") + echo "$d $@" +} + +# Replace the server name in the caddy config +sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json + +log "starting postgres" +pg_ctlcluster 13 main start + +log "starting caddy" +/root/caddy start --config /root/caddy.json + +# Set the server name of the homeserver +export SYNAPSE_SERVER_NAME=${SERVER_NAME} + +# No need to report stats here +export SYNAPSE_REPORT_STATS=no + +# Set postgres authentication details which will be placed in the homeserver config file +export POSTGRES_PASSWORD=somesecret +export POSTGRES_USER=postgres +export POSTGRES_HOST=localhost + +# Specify the workers to test with +export SYNAPSE_WORKER_TYPES="\ + event_persister, \ + event_persister, \ + background_worker, \ + frontend_proxy, \ + event_creator, \ + user_dir, \ + media_repository, \ + federation_inbound, \ + federation_reader, \ + federation_sender, \ + synchrotron, \ + appservice, \ + pusher" + +# Run the script that writes the necessary config files and starts supervisord, which in turn +# starts everything else +exec /configure_workers_and_start.py From 7efddbebefbadce3e5f0938ea123e7e37ad2503c Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 14 Apr 2022 15:25:22 +0100 Subject: [PATCH 022/263] Update documentation to reflect that `run_background_tasks_on` is no longer experimental. (#12451) * Background workers aren't experimental anymore * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) * Stream writers aren't experimental either --- changelog.d/12451.doc | 1 + docs/workers.md | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12451.doc diff --git a/changelog.d/12451.doc b/changelog.d/12451.doc new file mode 100644 index 000000000000..c8b23c128539 --- /dev/null +++ b/changelog.d/12451.doc @@ -0,0 +1 @@ +Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. diff --git a/docs/workers.md b/docs/workers.md index 446b7e5064e2..858411b15ec3 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -341,9 +341,9 @@ effects of bursts of events from that bridge on events sent by normal users. #### Stream writers -Additionally, there is *experimental* support for moving writing of specific -streams (such as events) off of the main process to a particular worker. (This -is only supported with Redis-based replication.) +Additionally, the writing of specific streams (such as events) can be moved off +of the main process to a particular worker. +(This is only supported with Redis-based replication.) To enable this, the worker must have a HTTP replication listener configured, have a `worker_name` and be listed in the `instance_map` config. The same worker @@ -420,7 +420,7 @@ the stream writer for the `presence` stream: #### Background tasks -There is also *experimental* support for moving background tasks to a separate +There is also support for moving background tasks to a separate worker. Background tasks are run periodically or started via replication. Exactly which tasks are configured to run depends on your Synapse configuration (e.g. if stats is enabled). From aaaff98202fc1be4e3794291c4b670b8fc62cee5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 14 Apr 2022 15:36:49 +0100 Subject: [PATCH 023/263] Dockerfile-workers: reduce the amount we install (#12464) This is an attempt to reduce the rebuild time. In short, we reduce the amount of stuff that the dockerfile installs, so as to give a faster startup. --- changelog.d/12464.misc | 1 + docker/Dockerfile-workers | 17 ++++++++++++++--- docker/conf-workers/supervisord.conf.j2 | 3 +++ docker/configure_workers_and_start.py | 19 +++++++++---------- 4 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 changelog.d/12464.misc diff --git a/changelog.d/12464.misc b/changelog.d/12464.misc new file mode 100644 index 000000000000..7a8cc6ba512c --- /dev/null +++ b/changelog.d/12464.misc @@ -0,0 +1 @@ +Dockerfile-workers: reduce the amount we install in the image. diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 6fb1cdbfb020..9ccb2b22a750 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -2,10 +2,19 @@ FROM matrixdotorg/synapse # Install deps -RUN apt-get update -RUN apt-get install -y supervisor redis nginx +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + redis-server nginx-light -# Remove the default nginx sites +# Install supervisord with pip instead of apt, to avoid installing a second +# copy of python. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install supervisor~=4.2 + +# Disable the default nginx sites RUN rm /etc/nginx/sites-enabled/default # Copy Synapse worker, nginx and supervisord configuration template files @@ -19,5 +28,7 @@ EXPOSE 8080/tcp COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py ENTRYPOINT ["/configure_workers_and_start.py"] +# Replace the healthcheck with one which checks *all* the workers. The script +# is generated by configure_workers_and_start.py. HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ CMD /bin/sh /healthcheck.sh diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2 index 0de2c6143b5a..408ef727879d 100644 --- a/docker/conf-workers/supervisord.conf.j2 +++ b/docker/conf-workers/supervisord.conf.j2 @@ -5,6 +5,9 @@ nodaemon=true user=root +[include] +files = /etc/supervisor/conf.d/*.conf + [program:nginx] command=/usr/sbin/nginx -g "daemon off;" priority=500 diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 8f1e61930e65..23cac18e8dbd 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -502,9 +502,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): ) # Supervisord config + os.makedirs("/etc/supervisor", exist_ok=True) convert( "/conf/supervisord.conf.j2", - "/etc/supervisor/conf.d/supervisord.conf", + "/etc/supervisor/supervisord.conf", main_config_path=config_path, worker_config=supervisord_config, ) @@ -546,14 +547,6 @@ def generate_worker_log_config( return log_config_filepath -def start_supervisord(): - """Starts up supervisord which then starts and monitors all other necessary processes - - Raises: CalledProcessError if calling start.py return a non-zero exit code. - """ - subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE) - - def main(args, environ): config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") @@ -581,7 +574,13 @@ def main(args, environ): # Start supervisord, which will start Synapse, all of the configured worker # processes, redis, nginx etc. according to the config we created above. - start_supervisord() + log("Starting supervisord") + os.execl( + "/usr/local/bin/supervisord", + "supervisord", + "-c", + "/etc/supervisor/supervisord.conf", + ) if __name__ == "__main__": From 3c758d9808ddc00ee96c5426dda3caea356c8745 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 Apr 2022 11:32:30 -0700 Subject: [PATCH 024/263] Add a manual documenting config file options (#12368) --- changelog.d/12368.doc | 1 + docs/SUMMARY.md | 1 + .../configuration/config_documentation.md | 3412 +++++++++++++++++ 3 files changed, 3414 insertions(+) create mode 100644 changelog.d/12368.doc create mode 100644 docs/usage/configuration/config_documentation.md diff --git a/changelog.d/12368.doc b/changelog.d/12368.doc new file mode 100644 index 000000000000..62e4cb2c7ef5 --- /dev/null +++ b/changelog.d/12368.doc @@ -0,0 +1 @@ +Add a manual documenting config file options. \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6aa48e191999..65570cefbe1b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,6 +17,7 @@ # Usage - [Federation](federate.md) - [Configuration](usage/configuration/README.md) + - [Configuration Manual](usage/configuration/config_documentation.md) - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md) - [Logging Sample Config File](usage/configuration/logging_sample_config.md) - [Structured Logging](structured_logging.md) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md new file mode 100644 index 000000000000..9c864af6ec32 --- /dev/null +++ b/docs/usage/configuration/config_documentation.md @@ -0,0 +1,3412 @@ +# Configuring Synapse + +This is intended as a guide to the Synapse configuration. The behavior of a Synapse instance can be modified +through the many configuration settings documented here — each config option is explained, +including what the default is, how to change the default and what sort of behaviour the setting governs. +Also included is an example configuration for each setting. If you don't want to spend a lot of time +thinking about options, the config as generated sets sensible defaults for all values. Do note however that the +database defaults to SQLite, which is not recommended for production usage. You can read more on this subject +[here](../../setup/installation.md#using-postgresql). + +## Config Conventions + +Configuration options that take a time period can be set using a number +followed by a letter. Letters have the following meanings: + +* `s` = second +* `m` = minute +* `h` = hour +* `d` = day +* `w` = week +* `y` = year + +For example, setting `redaction_retention_period: 5m` would remove redacted +messages from the database after 5 minutes, rather than 5 months. + +### YAML +The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules +apply if you want your config file to be read properly. A few helpful things to know: +* `#` before any option in the config will comment out that setting and either a default (if available) will + be applied or Synapse will ignore the setting. Thus, in example #1 below, the setting will be read and + applied, but in example #2 the setting will not be read and a default will be applied. + + Example #1: + ```yaml + pid_file: DATADIR/homeserver.pid + ``` + Example #2: + ```yaml + #pid_file: DATADIR/homeserver.pid + ``` +* Indentation matters! The indentation before a setting + will determine whether a given setting is read as part of another + setting, or considered on its own. Thus, in example #1, the `enabled` setting + is read as a sub-option of the `presence` setting, and will be properly applied. + + However, the lack of indentation before the `enabled` setting in example #2 means + that when reading the config, Synapse will consider both `presence` and `enabled` as + different settings. In this case, `presence` has no value, and thus a default applied, and `enabled` + is an option that Synapse doesn't recognize and thus ignores. + + Example #1: + ```yaml + presence: + enabled: false + ``` + Example #2: + ```yaml + presence: + enabled: false + ``` + In this manual, all top-level settings (ones with no indentation) are identified + at the beginning of their section (i.e. "Config option: `example_setting`") and + the sub-options, if any, are identified and listed in the body of the section. + In addition, each setting has an example of its usage, with the proper indentation + shown. + + +## Modules + +Server admins can expand Synapse's functionality with external modules. + +See [here](../../modules/index.md) for more +documentation on how to configure or create custom modules for Synapse. + + +--- +Config option: `modules` + +Use the `module` sub-option to add modules under this option to extend functionality. +The `module` setting then has a sub-option, `config`, which can be used to define some configuration +for the `module`. + +Defaults to none. + +Example configuration: +```yaml +modules: + - module: my_super_module.MySuperClass + config: + do_thing: true + - module: my_other_super_module.SomeClass + config: {} +``` +--- +## Server ## + +Define your homeserver name and other base options. + +--- +Config option: `server_name` + +This sets the public-facing domain of the server. + +The `server_name` name will appear at the end of usernames and room addresses +created on your server. For example if the `server_name` was example.com, +usernames on your server would be in the format `@user:example.com` + +In most cases you should avoid using a matrix specific subdomain such as +matrix.example.com or synapse.example.com as the `server_name` for the same +reasons you wouldn't use user@email.example.com as your email address. +See [here](../../delegate.md) +for information on how to host Synapse on a subdomain while preserving +a clean `server_name`. + +The `server_name` cannot be changed later so it is important to +configure this correctly before you start Synapse. It should be all +lowercase and may contain an explicit port. + +There is no default for this option. + +Example configuration #1: +```yaml +server_name: matrix.org +``` +Example configuration #2: +```yaml +server_name: localhost:8080 +``` +--- +Config option: `pid_file` + +When running Synapse as a daemon, the file to store the pid in. Defaults to none. + +Example configuration: +```yaml +pid_file: DATADIR/homeserver.pid +``` +--- +Config option: `web_client_location` + +The absolute URL to the web client which `/` will redirect to. Defaults to none. + +Example configuration: +```yaml +web_client_location: https://riot.example.com/ +``` +--- +Config option: `public_baseurl` + +The public-facing base URL that clients use to access this Homeserver (not +including _matrix/...). This is the same URL a user might enter into the +'Custom Homeserver URL' field on their client. If you use Synapse with a +reverse proxy, this should be the URL to reach Synapse via the proxy. +Otherwise, it should be the URL to reach Synapse's client HTTP listener (see +'listeners' below). + +Defaults to `https:///`. + +Example configuration: +```yaml +public_baseurl: https://example.com/ +``` +--- +Config option: `serve_server_wellknown` + +By default, other servers will try to reach our server on port 8448, which can +be inconvenient in some environments. + +Provided `https:///` on port 443 is routed to Synapse, this +option configures Synapse to serve a file at `https:///.well-known/matrix/server`. +This will tell other servers to send traffic to port 443 instead. + +This option currently defaults to false. + +See https://matrix-org.github.io/synapse/latest/delegate.html for more +information. + +Example configuration: +```yaml +serve_server_wellknown: true +``` +--- +Config option: `soft_file_limit` + +Set the soft limit on the number of file descriptors synapse can use. +Zero is used to indicate synapse should set the soft limit to the hard limit. +Defaults to 0. + +Example configuration: +```yaml +soft_file_limit: 3 +``` +--- +Config option: `presence` + +Presence tracking allows users to see the state (e.g online/offline) +of other local and remote users. Set the `enabled` sub-option to false to +disable presence tracking on this homeserver. Defaults to true. +This option replaces the previous top-level 'use_presence' option. + +Example configuration: +```yaml +presence: + enabled: false +``` +--- +Config option: `require_auth_for_profile_requests` + +Whether to require authentication to retrieve profile data (avatars, display names) of other +users through the client API. Defaults to false. Note that profile data is also available +via the federation API, unless `allow_profile_lookup_over_federation` is set to false. + +Example configuration: +```yaml +require_auth_for_profile_requests: true +``` +--- +Config option: `limit_profile_requests_to_users_who_share_rooms` + +Use this option to require a user to share a room with another user in order +to retrieve their profile information. Only checked on Client-Server +requests. Profile requests from other servers should be checked by the +requesting server. Defaults to false. + +Example configuration: +```yaml +limit_profile_requests_to_users_who_share_rooms: true +``` +--- +Config option: `include_profile_data_on_invite` + +Use this option to prevent a user's profile data from being retrieved and +displayed in a room until they have joined it. By default, a user's +profile data is included in an invite event, regardless of the values +of the above two settings, and whether or not the users share a server. +Defaults to true. + +Example configuration: +```yaml +include_profile_data_on_invite: false +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, removes the need for authentication to access the server's +public rooms directory through the client API, meaning that anyone can +query the room directory. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_without_auth: true +``` +--- +Config option: `allow_public_rooms_without_auth` + +If set to true, allows any other homeserver to fetch the server's public +rooms directory via federation. Defaults to false. + +Example configuration: +```yaml +allow_public_rooms_over_federation: true +``` +--- +Config option: `default_room_version` + +The default room version for newly created rooms on this server. + +Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions) + +For example, for room version 1, `default_room_version` should be set +to "1". + +Currently defaults to "9". + +Example configuration: +```yaml +default_room_version: "8" +``` +--- +Config option: `gc_thresholds` + +The garbage collection threshold parameters to pass to `gc.set_threshold`, if defined. +Defaults to none. + +Example configuration: +```yaml +gc_thresholds: [700, 10, 10] +``` +--- +Config option: `gc_min_interval` + +The minimum time in seconds between each GC for a generation, regardless of +the GC thresholds. This ensures that we don't do GC too frequently. A value of `[1s, 10s, 30s]` +indicates that a second must pass between consecutive generation 0 GCs, etc. + +Defaults to `[1s, 10s, 30s]`. + +Example configuration: +```yaml +gc_min_interval: [0.5s, 30s, 1m] +``` +--- +Config option: `filter_timeline_limit` + +Set the limit on the returned events in the timeline in the get +and sync operations. Defaults to 100. A value of -1 means no upper limit. + + +Example configuration: +```yaml +filter_timeline_limit: 5000 +``` +--- +Config option: `block_non_admin_invites` + +Whether room invites to users on this server should be blocked +(except those sent by local server admins). Defaults to false. + +Example configuration: +```yaml +block_non_admin_invites: true +``` +--- +Config option: `enable_search` + +If set to false, new messages will not be indexed for searching and users +will receive errors when searching for messages. Defaults to true. + +Example configuration: +```yaml +enable_search: false +``` +--- +Config option: `ip_range_blacklist` + +This option prevents outgoing requests from being sent to the specified blacklisted IP address +CIDR ranges. If this option is not specified then it defaults to private IP +address ranges (see the example below). + +The blacklist applies to the outbound requests for federation, identity servers, +push servers, and for checking key validity for third-party invite events. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This option replaces `federation_ip_range_blacklist` in Synapse v1.25.0. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +--- +Config option: `ip_range_whitelist` + +List of IP address CIDR ranges that should be allowed for federation, +identity servers, push servers, and for checking key validity for +third-party invite events. This is useful for specifying exceptions to +wide-ranging blacklisted target IP ranges - e.g. for communication with +a push server only visible in your network. + +This whitelist overrides `ip_range_blacklist` and defaults to an empty +list. + +Example configuration: +```yaml +ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `listeners` + +List of ports that Synapse should listen on, their purpose and their +configuration. + +Sub-options for each listener include: + +* `port`: the TCP port to bind to. + +* `bind_addresses`: a list of local addresses to listen on. The default is + 'all local interfaces'. + +* `type`: the type of listener. Normally `http`, but other valid options are: + + * `manhole`: (see the docs [here](../../manhole.md)), + + * `metrics`: (see the docs [here](../../metrics-howto.md)), + + * `replication`: (see the docs [here](../../workers.md)). + +* `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. + +* `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is + behind a reverse-proxy. + +* `resources`: Only valid for an 'http' listener. A list of resources to host + on this port. Sub-options for each resource are: + + * `names`: a list of names of HTTP resources. See below for a list of valid resource names. + + * `compress`: set to true to enable HTTP compression for this resource. + +* `additional_resources`: Only valid for an 'http' listener. A map of + additional endpoints which should be loaded via dynamic modules. + +Valid resource names are: + +* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies 'media' and 'static'. + +* `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. + +* `federation`: the server-server API (/_matrix/federation). Also implies `media`, `keys`, `openid` + +* `keys`: the key discovery API (/_matrix/keys). + +* `media`: the media API (/_matrix/media). + +* `metrics`: the metrics interface. See [here](../../metrics-howto.md). + +* `openid`: OpenID authentication. See [here](../../openid.md). + +* `replication`: the HTTP replication API (/_synapse/replication). See [here](../../workers.md). + +* `static`: static resources under synapse/static (/_matrix/static). (Mostly useful for 'fallback authentication'.) + +Example configuration #1: +```yaml +listeners: + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # (Note that you will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` +Example configuration #2: +```yaml +listeners: + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://matrix-org.github.io/synapse/latest/reverse_proxy.html. + # + - port: 8008 + tls: false + type: http + x_forwarded: true + bind_addresses: ['::1', '127.0.0.1'] + + resources: + - names: [client, federation] + compress: false + + # example additional_resources: + additional_resources: + "/_matrix/my/custom/endpoint": + module: my_module.CustomRequestHandler + config: {} + + # Turn on the twisted ssh manhole service on localhost on the given + # port. + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` +--- +Config option: `manhole_settings` + +Connection settings for the manhole. You can find more information +on the manhole [here](../../manhole.md). Manhole sub-options include: +* `username` : the username for the manhole. This defaults to 'matrix'. +* `password`: The password for the manhole. This defaults to 'rabbithole'. +* `ssh_priv_key_path` and `ssh_pub_key_path`: The private and public SSH key pair used to encrypt the manhole traffic. + If these are left unset, then hardcoded and non-secret keys are used, + which could allow traffic to be intercepted if sent over a public network. + +Example configuration: +```yaml +manhole_settings: + username: manhole + password: mypassword + ssh_priv_key_path: CONFDIR/id_rsa + ssh_pub_key_path: CONFDIR/id_rsa.pub +``` +--- +Config option: `dummy_events_threshold` + +Forward extremities can build up in a room due to networking delays between +homeservers. Once this happens in a large room, calculation of the state of +that room can become quite expensive. To mitigate this, once the number of +forward extremities reaches a given threshold, Synapse will send an +`org.matrix.dummy_event` event, which will reduce the forward extremities +in the room. + +This setting defines the threshold (i.e. number of forward extremities in the room) at which dummy events are sent. +The default value is 10. + +Example configuration: +```yaml +dummy_events_threshold: 5 +``` +--- +## Homeserver blocking ## +Useful options for Synapse admins. + +--- + +Config option: `admin_contact` + +How to reach the server admin, used in `ResourceLimitError`. Defaults to none. + +Example configuration: +```yaml +admin_contact: 'mailto:admin@server.com' +``` +--- +Config option: `hs_disabled` and `hs_disabled_message` + +Blocks users from connecting to the homeserver and provides a human-readable reason +why the connection was blocked. Defaults to false. + +Example configuration: +```yaml +hs_disabled: true +hs_disabled_message: 'Reason for why the HS is blocked' +``` +--- +Config option: `limit_usage_by_mau` + +This option disables/enables monthly active user blocking. Used in cases where the admin or +server owner wants to limit to the number of monthly active users. When enabled and a limit is +reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. +Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. + +Example configuration: +```yaml +limit_usage_by_mau: true +``` +--- +Config option: `max_mau_value` + +This option sets the hard limit of monthly active users above which the server will start +blocking user actions if `limit_usage_by_mau` is enabled. Defaults to 0. + +Example configuration: +```yaml +max_mau_value: 50 +``` +--- +Config option: `mau_trial_days` + +The option `mau_trial_days` is a means to add a grace period for active users. It +means that users must be active for the specified number of days before they +can be considered active and guards against the case where lots of users +sign up in a short space of time never to return after their initial +session. Defaults to 0. + +Example configuration: +```yaml +mau_trial_days: 5 +``` +--- +Config option: `mau_limit_alerting` + +The option `mau_limit_alerting` is a means of limiting client-side alerting +should the mau limit be reached. This is useful for small instances +where the admin has 5 mau seats (say) for 5 specific people and no +interest increasing the mau limit further. Defaults to true, which +means that alerting is enabled. + +Example configuration: +```yaml +mau_limit_alerting: false +``` +--- +Config option: `mau_stats_only` + +If enabled, the metrics for the number of monthly active users will +be populated, however no one will be limited based on these numbers. If `limit_usage_by_mau` +is true, this is implied to be true. Defaults to false. + +Example configuration: +```yaml +mau_stats_only: true +``` +--- +Config option: `mau_limit_reserved_threepids` + +Sometimes the server admin will want to ensure certain accounts are +never blocked by mau checking. These accounts are specified by this option. +Defaults to none. Add accounts by specifying the `medium` and `address` of the +reserved threepid (3rd party identifier). + +Example configuration: +```yaml +mau_limit_reserved_threepids: + - medium: 'email' + address: 'reserved_user@example.com' +``` +--- +Config option: `server_context` + +This option is used by phonehome stats to group together related servers. +Defaults to none. + +Example configuration: +```yaml +server_context: context +``` +--- +Config option: `limit_remote_rooms` + +When this option is enabled, the room "complexity" will be checked before a user +joins a new remote room. If it is above the complexity limit, the server will +disallow joining, or will instantly leave. This is useful for homeservers that are +resource-constrained. Options for this setting include: +* `enabled`: whether this check is enabled. Defaults to false. +* `complexity`: the limit above which rooms cannot be joined. The default is 1.0. +* `complexity_error`: override the error which is returned when the room is too complex with a + custom message. +* `admins_can_join`: allow server admins to join complex rooms. Default is false. + +Room complexity is an arbitrary measure based on factors such as the number of +users in the room. + +Example configuration: +```yaml +limit_remote_rooms: + enabled: true + complexity: 0.5 + complexity_error: "I can't let you do that, Dave." + admins_can_join: true +``` +--- +Config option: `require_membership_for_aliases` + +Whether to require a user to be in the room to add an alias to it. +Defaults to true. + +Example configuration: +```yaml +require_membership_for_aliases: false +``` +--- +Config option: `allow_per_room_profiles` + +Whether to allow per-room membership profiles through the sending of membership +events with profile information that differs from the target's global profile. +Defaults to true. + +Example configuration: +```yaml +allow_per_room_profiles: false +``` +--- +Config option: `max_avatar_size` + +The largest permissible file size in bytes for a user avatar. Defaults to no restriction. +Use M for MB and K for KB. + +Note that user avatar changes will not work if this is set without using Synapse's media repository. + +Example configuration: +```yaml +max_avatar_size: 10M +``` +--- +Config option: `allowed_avatar_mimetypes` + +The MIME types allowed for user avatars. Defaults to no restriction. + +Note that user avatar changes will not work if this is set without +using Synapse's media repository. + +Example configuration: +```yaml +allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"] +``` +--- +Config option: `redaction_retention_period` + +How long to keep redacted events in unredacted form in the database. After +this period redacted events get replaced with their redacted form in the DB. + +Defaults to `7d`. Set to `null` to disable. + +Example configuration: +```yaml +redaction_retention_period: 28d +``` +--- +Config option: `user_ips_max_age` + +How long to track users' last seen time and IPs in the database. + +Defaults to `28d`. Set to `null` to disable clearing out of old rows. + +Example configuration: +```yaml +user_ips_max_age: 14d +``` +--- +Config option: `request_token_inhibit_3pid_errors` + +Inhibits the `/requestToken` endpoints from returning an error that might leak +information about whether an e-mail address is in use or not on this +homeserver. Defaults to false. +Note that for some endpoints the error situation is the e-mail already being +used, and for others the error is entering the e-mail being unused. +If this option is enabled, instead of returning an error, these endpoints will +act as if no error happened and return a fake session ID ('sid') to clients. + +Example configuration: +```yaml +request_token_inhibit_3pid_errors: true +``` +--- +Config option: `next_link_domain_whitelist` + +A list of domains that the domain portion of `next_link` parameters +must match. + +This parameter is optionally provided by clients while requesting +validation of an email or phone number, and maps to a link that +users will be automatically redirected to after validation +succeeds. Clients can make use this parameter to aid the validation +process. + +The whitelist is applied whether the homeserver or an identity server is handling validation. + +The default value is no whitelist functionality; all domains are +allowed. Setting this value to an empty list will instead disallow +all domains. + +Example configuration: +```yaml +next_link_domain_whitelist: ["matrix.org"] +``` +--- +Config option: `templates` and `custom_template_directory` + +These options define templates to use when generating email or HTML page contents. +The `custom_template_directory` determines which directory Synapse will try to +find template files in to use to generate email or HTML page contents. +If not set, or a file is not found within the template directory, a default +template from within the Synapse package will be used. + +See [here](../../templates.md) for more +information about using custom templates. + +Example configuration: +```yaml +templates: + custom_template_directory: /path/to/custom/templates/ +``` +--- +Config option: `retention` + +This option and the associated options determine message retention policy at the +server level. + +Room admins and mods can define a retention period for their rooms using the +`m.room.retention` state event, and server admins can cap this period by setting +the `allowed_lifetime_min` and `allowed_lifetime_max` config options. + +If this feature is enabled, Synapse will regularly look for and purge events +which are older than the room's maximum retention period. Synapse will also +filter events received over federation so that events that should have been +purged are ignored and not stored again. + +The message retention policies feature is disabled by default. + +This setting has the following sub-options: +* `default_policy`: Default retention policy. If set, Synapse will apply it to rooms that lack the + 'm.room.retention' state event. This option is further specified by the + `min_lifetime` and `max_lifetime` sub-options associated with it. Note that the + value of `min_lifetime` doesn't matter much because Synapse doesn't take it into account yet. + +* `allowed_lifetime_min` and `allowed_lifetime_max`: Retention policy limits. If + set, and the state of a room contains a `m.room.retention` event in its state + which contains a `min_lifetime` or a `max_lifetime` that's out of these bounds, + Synapse will cap the room's policy to these limits when running purge jobs. + +* `purge_jobs` and the associated `shortest_max_lifetime` and `longest_max_lifetime` sub-options: + Server admins can define the settings of the background jobs purging the + events whose lifetime has expired under the `purge_jobs` section. + + If no configuration is provided for this option, a single job will be set up to delete + expired events in every room daily. + + Each job's configuration defines which range of message lifetimes the job + takes care of. For example, if `shortest_max_lifetime` is '2d' and + `longest_max_lifetime` is '3d', the job will handle purging expired events in + rooms whose state defines a `max_lifetime` that's both higher than 2 days, and + lower than or equal to 3 days. Both the minimum and the maximum value of a + range are optional, e.g. a job with no `shortest_max_lifetime` and a + `longest_max_lifetime` of '3d' will handle every room with a retention policy + whose `max_lifetime` is lower than or equal to three days. + + The rationale for this per-job configuration is that some rooms might have a + retention policy with a low `max_lifetime`, where history needs to be purged + of outdated messages on a more frequent basis than for the rest of the rooms + (e.g. every 12h), but not want that purge to be performed by a job that's + iterating over every room it knows, which could be heavy on the server. + + If any purge job is configured, it is strongly recommended to have at least + a single job with neither `shortest_max_lifetime` nor `longest_max_lifetime` + set, or one job without `shortest_max_lifetime` and one job without + `longest_max_lifetime` set. Otherwise some rooms might be ignored, even if + `allowed_lifetime_min` and `allowed_lifetime_max` are set, because capping a + room's policy to these values is done after the policies are retrieved from + Synapse's database (which is done using the range specified in a purge job's + configuration). + +Example configuration: +```yaml +retention: + enabled: true + default_policy: + min_lifetime: 1d + max_lifetime: 1y + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + interval: 1d +``` +--- +## TLS ## + +Options related to TLS. + +--- +Config option: `tls_certificate_path` + +This option specifies a PEM-encoded X509 certificate for TLS. +This certificate, as of Synapse 1.0, will need to be a valid and verifiable +certificate, signed by a recognised Certificate Authority. Defaults to none. + +Be sure to use a `.pem` file that includes the full certificate chain including +any intermediate certificates (for instance, if using certbot, use +`fullchain.pem` as your certificate, not `cert.pem`). + +Example configuration: +```yaml +tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt" +``` +--- +Config option: `tls_private_key_path` + +PEM-encoded private key for TLS. Defaults to none. + +Example configuration: +```yaml +tls_private_key_path: "CONFDIR/SERVERNAME.tls.key" +``` +--- +Config option: `federation_verify_certificates` +Whether to verify TLS server certificates for outbound federation requests. + +Defaults to true. To disable certificate verification, set the option to false. + +Example configuration: +```yaml +federation_verify_certificates: false +``` +--- +Config option: `federation_client_minimum_tls_version` + +The minimum TLS version that will be used for outbound federation requests. + +Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note +that setting this value higher than `1.2` will prevent federation to most +of the public Matrix network: only configure it to `1.3` if you have an +entirely private federation setup and you can ensure TLS 1.3 support. + +Example configuration: +```yaml +federation_client_minimum_tls_version: 1.2 +``` +--- +Config option: `federation_certificate_verification_whitelist` + +Skip federation certificate verification on a given whitelist +of domains. + +This setting should only be used in very specific cases, such as +federation over Tor hidden services and similar. For private networks +of homeservers, you likely want to use a private CA instead. + +Only effective if `federation_verify_certicates` is `true`. + +Example configuration: +```yaml +federation_certificate_verification_whitelist: + - lon.example.com + - "*.domain.com" + - "*.onion" +``` +--- +Config option: `federation_custom_ca_list` + +List of custom certificate authorities for federation traffic. + +This setting should only normally be used within a private network of +homeservers. + +Note that this list will replace those that are provided by your +operating environment. Certificates must be in PEM format. + +Example configuration: +```yaml +federation_custom_ca_list: + - myCA1.pem + - myCA2.pem + - myCA3.pem +``` +--- +## Federation ## + +Options related to federation. + +--- +Config option: `federation_domain_whitelist` + +Restrict federation to the given whitelist of domains. +N.B. we recommend also firewalling your federation listener to limit +inbound federation traffic as early as possible, rather than relying +purely on this application-layer restriction. If not specified, the +default is to whitelist everything. + +Example configuration: +```yaml +federation_domain_whitelist: + - lon.example.com + - nyc.example.com + - syd.example.com +``` +--- +Config option: `federation_metrics_domains` + +Report prometheus metrics on the age of PDUs being sent to and received from +the given domains. This can be used to give an idea of "delay" on inbound +and outbound federation, though be aware that any delay can be due to problems +at either end or with the intermediate network. + +By default, no domains are monitored in this way. + +Example configuration: +```yaml +federation_metrics_domains: + - matrix.org + - example.com +``` +--- +Config option: `allow_profile_lookup_over_federation` + +Set to false to disable profile lookup over federation. By default, the +Federation API allows other homeservers to obtain profile data of any user +on this homeserver. + +Example configuration: +```yaml +allow_profile_lookup_over_federation: false +``` +--- +Config option: `allow_device_name_lookup_over_federation` + +Set this option to false to disable device display name lookup over federation. By default, the +Federation API allows other homeservers to obtain device display names of any user +on this homeserver. + +Example configuration: +```yaml +allow_device_name_lookup_over_federation: false +``` +--- +## Caching ## + +Options related to caching + +--- +Config option: `event_cache_size` + +The number of events to cache in memory. Not affected by +`caches.global_factor`. Defaults to 10K. + +Example configuration: +```yaml +event_cache_size: 15K +``` +--- +Config option: `cache` and associated values + +A cache 'factor' is a multiplier that can be applied to each of +Synapse's caches in order to increase or decrease the maximum +number of entries that can be stored. + +Caching can be configured through the following sub-options: + +* `global_factor`: Controls the global cache factor, which is the default cache factor + for all caches if a specific factor for that cache is not otherwise + set. + + This can also be set by the `SYNAPSE_CACHE_FACTOR` environment + variable. Setting by environment variable takes priority over + setting through the config file. + + Defaults to 0.5, which will halve the size of all caches. + +* `per_cache_factors`: A dictionary of cache name to cache factor for that individual + cache. Overrides the global cache factor for a given cache. + + These can also be set through environment variables comprised + of `SYNAPSE_CACHE_FACTOR_` + the name of the cache in capital + letters and underscores. Setting by environment variable + takes priority over setting through the config file. + Ex. `SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0` + + Some caches have '*' and other characters that are not + alphanumeric or underscores. These caches can be named with or + without the special characters stripped. For example, to specify + the cache factor for `*stateGroupCache*` via an environment + variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`. + +* `expire_caches`: Controls whether cache entries are evicted after a specified time + period. Defaults to true. Set to false to disable this feature. Note that never expiring + caches may result in excessive memory usage. + +* `cache_entry_ttl`: If `expire_caches` is enabled, this flag controls how long an entry can + be in a cache without having been accessed before being evicted. + Defaults to 30m. + +* `sync_response_cache_duration`: Controls how long the results of a /sync request are + cached for after a successful response is returned. A higher duration can help clients + with intermittent connections, at the cost of higher memory usage. + By default, this is zero, which means that sync responses are not cached + at all. + + +Example configuration: +```yaml +caches: + global_factor: 1.0 + per_cache_factors: + get_users_who_share_room_with_user: 2.0 + expire_caches: false + sync_response_cache_duration: 2m +``` +--- +## Database ## +Config options related to database settings. + +--- +Config option: `database` + +The `database` setting defines the database that synapse uses to store all of +its data. + +Associated sub-options: + +* `name`: this option specifies the database engine to use: either `sqlite3` (for SQLite) + or `psycopg2` (for PostgreSQL). If no name is specified Synapse will default to SQLite. + +* `txn_limit` gives the maximum number of transactions to run per connection + before reconnecting. Defaults to 0, which means no limit. + +* `allow_unsafe_locale` is an option specific to Postgres. Under the default behavior, Synapse will refuse to + start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended) + by setting `allow_unsafe_locale` to true. Note that doing so may corrupt your database. You can find more information + [here](../../postgres.md#fixing-incorrect-collate-or-ctype) and [here](https://wiki.postgresql.org/wiki/Locale_data_changes). + +* `args` gives options which are passed through to the database engine, + except for options starting with `cp_`, which are used to configure the Twisted + connection pool. For a reference to valid arguments, see: + * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect) + * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) + * for [the connection pool](https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__) + +For more information on using Synapse with Postgres, +see [here](../../postgres.md). + +Example SQLite configuration: +``` +database: + name: sqlite3 + args: + database: /path/to/homeserver.db +``` + +Example Postgres configuration: +``` +database: + name: psycopg2 + txn_limit: 10000 + args: + user: synapse_user + password: secretpassword + database: synapse + host: localhost + port: 5432 + cp_min: 5 + cp_max: 10 +``` +--- +## Logging ## +Config options related to logging. + +--- +Config option: `log_config` + +This option specifies a yaml python logging config file as described [here](https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema). + +Example configuration: +```yaml +log_config: "CONFDIR/SERVERNAME.log.config" +``` +--- +## Ratelimiting ## +Options related to ratelimiting in Synapse. + +Each ratelimiting configuration is made of two parameters: + - `per_second`: number of requests a client can send per second. + - `burst_count`: number of requests a client can send before being throttled. +--- +Config option: `rc_message` + + +Ratelimiting settings for client messaging. + +This is a ratelimiting option for messages that ratelimits sending based on the account the client +is using. It defaults to: `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_message: + per_second: 0.5 + burst_count: 15 +``` +--- +Config option: `rc_registration` + +This option ratelimits registration requests based on the client's IP address. +It defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_registration: + per_second: 0.15 + burst_count: 2 +``` +--- +Config option: `rc_registration_token_validity` + +This option checks the validity of registration tokens that ratelimits requests based on +the client's IP address. +Defaults to `per_second: 0.1`, `burst_count: 5`. + +Example configuration: +```yaml +rc_registration_token_validity: + per_second: 0.3 + burst_count: 6 +``` +--- +Config option: `rc_login` + +This option specifies several limits for login: +* `address` ratelimits login requests based on the client's IP + address. Defaults to `per_second: 0.17`, `burst_count: 3`. + +* `account` ratelimits login requests based on the account the + client is attempting to log into. Defaults to `per_second: 0.17`, + `burst_count: 3`. + +* `failted_attempts` ratelimits login requests based on the account the + client is attempting to log into, based on the amount of failed login + attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. + +Example configuration: +```yaml +rc_login: + address: + per_second: 0.15 + burst_count: 5 + account: + per_second: 0.18 + burst_count: 4 + failed_attempts: + per_second: 0.19 + burst_count: 7 +``` +--- +Config option: `rc_admin_redaction` + +This option sets ratelimiting redactions by room admins. If this is not explicitly +set then it uses the same ratelimiting as per `rc_message`. This is useful +to allow room admins to deal with abuse quickly. + +Example configuration: +```yaml +rc_admin_redaction: + per_second: 1 + burst_count: 50 +``` +--- +Config option: `rc_joins` + +This option allows for ratelimiting number of rooms a user can join. This setting has the following sub-options: + +* `local`: ratelimits when users are joining rooms the server is already in. + Defaults to `per_second: 0.1`, `burst_count: 10`. + +* `remote`: ratelimits when users are trying to join rooms not on the server (which + can be more computationally expensive than restricting locally). Defaults to + `per_second: 0.01`, `burst_count: 10` + +Example configuration: +```yaml +rc_joins: + local: + per_second: 0.2 + burst_count: 15 + remote: + per_second: 0.03 + burst_count: 12 +``` +--- +Config option: `rc_3pid_validation` + +This option ratelimits how often a user or IP can attempt to validate a 3PID. +Defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_3pid_validation: + per_second: 0.003 + burst_count: 5 +``` +--- +Config option: `rc_invites` + +This option sets ratelimiting how often invites can be sent in a room or to a +specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10` and +`per_user` defaults to `per_second: 0.003`, `burst_count: 5`. + +Example configuration: +```yaml +rc_invites: + per_room: + per_second: 0.5 + burst_count: 5 + per_user: + per_second: 0.004 + burst_count: 3 +``` +--- +Config option: `rc_third_party_invite` + +This option ratelimits 3PID invites (i.e. invites sent to a third-party ID +such as an email address or a phone number) based on the account that's +sending the invite. Defaults to `per_second: 0.2`, `burst_count: 10`. + +Example configuration: +```yaml +rc_third_party_invite: + per_second: 0.2 + burst_count: 10 +``` +--- +Config option: `rc_federation` + +Defines limits on federation requests. + +The `rc_federation` configuration has the following sub-options: +* `window_size`: window size in milliseconds. Defaults to 1000. +* `sleep_limit`: number of federation requests from a single server in + a window before the server will delay processing the request. Defaults to 10. +* `sleep_delay`: duration in milliseconds to delay processing events + from remote servers by if they go over the sleep limit. Defaults to 500. +* `reject_limit`: maximum number of concurrent federation requests + allowed from a single server. Defaults to 50. +* `concurrent`: number of federation requests to concurrently process + from a single server. Defaults to 3. + +Example configuration: +```yaml +rc_federation: + window_size: 750 + sleep_limit: 15 + sleep_delay: 400 + reject_limit: 40 + concurrent: 5 +``` +--- +Config option: `federation_rr_transactions_per_room_per_second` + +Sets outgoing federation transaction frequency for sending read-receipts, +per-room. + +If we end up trying to send out more read-receipts, they will get buffered up +into fewer transactions. Defaults to 50. + +Example configuration: +```yaml +federation_rr_transactions_per_room_per_second: 40 +``` +--- +## Media Store ## +Config options relating to Synapse media store. + +--- +Config option: `enable_media_repo` + +Enable the media store service in the Synapse master. Defaults to true. +Set to false if you are using a separate media store worker. + +Example configuration: +```yaml +enable_media_repo: false +``` +--- +Config option: `media_store_path` + +Directory where uploaded images and attachments are stored. + +Example configuration: +```yaml +media_store_path: "DATADIR/media_store" +``` +--- +Config option: `media_storage_providers` + +Media storage providers allow media to be stored in different +locations. Defaults to none. Associated sub-options are: +* `module`: type of resource, e.g. `file_system`. +* `store_local`: whether to store newly uploaded local files +* `store_remote`: whether to store newly downloaded local files +* `store_synchronous`: whether to wait for successful storage for local uploads +* `config`: sets a path to the resource through the `directory` option + +Example configuration: +```yaml +media_storage_providers: + - module: file_system + store_local: false + store_remote: false + store_synchronous: false + config: + directory: /mnt/some/other/directory +``` +--- +Config option: `max_upload_size` + +The largest allowed upload size in bytes. + +If you are using a reverse proxy you may also need to set this value in +your reverse proxy's config. Defaults to 50M. Notably Nginx has a small max body size by default. +See [here](../../reverse_proxy.md) for more on using a reverse proxy with Synapse. + +Example configuration: +```yaml +max_upload_size: 60M +``` +--- +Config option: `max_image_pixels` + +Maximum number of pixels that will be thumbnailed. Defaults to 32M. + +Example configuration: +```yaml +max_image_pixels: 35M +``` +--- +Config option: `dynamic_thumbnails` + +Whether to generate new thumbnails on the fly to precisely match +the resolution requested by the client. If true then whenever +a new resolution is requested by the client the server will +generate a new thumbnail. If false the server will pick a thumbnail +from a precalculated list. Defaults to false. + +Example configuration: +```yaml +dynamic_thumbnails: true +``` +--- +Config option: `thumbnail_sizes` + +List of thumbnails to precalculate when an image is uploaded. Associated sub-options are: +* `width` +* `height` +* `method`: i.e. `crop`, `scale`, etc. + +Example configuration: +```yaml +thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale +``` +Config option: `url_preview_enabled` + +This setting determines whether the preview URL API is enabled. +It is disabled by default. Set to true to enable. If enabled you must specify a +`url_preview_ip_range_blacklist` blacklist. + +Example configuration: +```yaml +url_preview_enabled: true +``` +--- +Config option: `url_preview_ip_range_blacklist` + +List of IP address CIDR ranges that the URL preview spider is denied +from accessing. There are no defaults: you must explicitly +specify a list for URL previewing to work. You should specify any +internal services in your network that you do not want synapse to try +to connect to, otherwise anyone in any Matrix room could cause your +synapse to issue arbitrary GET requests to your internal services, +causing serious security issues. + +(0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +listed here, since they correspond to unroutable addresses.) + +This must be specified if `url_preview_enabled` is set. It is recommended that +you use the following example list as a starting point. + +Note: The value is ignored when an HTTP proxy is in use. + +Example configuration: +```yaml +url_preview_ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '192.88.99.0/24' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + - '2001:db8::/32' + - 'ff00::/8' + - 'fec0::/10' +``` +---- +Config option: `url_preview_ip_range_whitelist` + +This option sets a list of IP address CIDR ranges that the URL preview spider is allowed +to access even if they are specified in `url_preview_ip_range_blacklist`. +This is useful for specifying exceptions to wide-ranging blacklisted +target IP ranges - e.g. for enabling URL previews for a specific private +website only visible in your network. Defaults to none. + +Example configuration: +```yaml +url_preview_ip_range_whitelist: + - '192.168.1.1' +``` +--- +Config option: `url_preview_url_blacklist` + +Optional list of URL matches that the URL preview spider is +denied from accessing. You should use `url_preview_ip_range_blacklist` +in preference to this, otherwise someone could define a public DNS +entry that points to a private IP address and circumvent the blacklist. +This is more useful if you know there is an entire shape of URL that +you know that will never want synapse to try to spider. + +Each list entry is a dictionary of url component attributes as returned +by urlparse.urlsplit as applied to the absolute form of the URL. See +[here](https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit) for more +information. Some examples are: + +* `username` +* `netloc` +* `scheme` +* `path` + +The values of the dictionary are treated as a filename match pattern +applied to that component of URLs, unless they start with a ^ in which +case they are treated as a regular expression match. If all the +specified component matches for a given list item succeed, the URL is +blacklisted. + +Example configuration: +```yaml +url_preview_url_blacklist: + # blacklist any URL with a username in its URI + - username: '*' + + # blacklist all *.google.com URLs + - netloc: 'google.com' + - netloc: '*.google.com' + + # blacklist all plain HTTP URLs + - scheme: 'http' + + # blacklist http(s)://www.acme.com/foo + - netloc: 'www.acme.com' + path: '/foo' + + # blacklist any URL with a literal IPv4 address + - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' +``` +--- +Config option: `max_spider_size` + +The largest allowed URL preview spidering size in bytes. Defaults to 10M. + +Example configuration: +```yaml +max_spider_size: 8M +``` +--- +Config option: `url_preview_language` + +A list of values for the Accept-Language HTTP header used when +downloading webpages during URL preview generation. This allows +Synapse to specify the preferred languages that URL previews should +be in when communicating with remote servers. + +Each value is a IETF language tag; a 2-3 letter identifier for a +language, optionally followed by subtags separated by '-', specifying +a country or region variant. + +Multiple values can be provided, and a weight can be added to each by +using quality value syntax (;q=). '*' translates to any language. + +Defaults to "en". + +Example configuration: +```yaml + url_preview_accept_language: + - en-UK + - en-US;q=0.9 + - fr;q=0.8 + - *;q=0.7 +``` +---- +Config option: `oembed` + +oEmbed allows for easier embedding content from a website. It can be +used for generating URLs previews of services which support it. A default list of oEmbed providers +is included with Synapse. Set `disable_default_providers` to true to disable using +these default oEmbed URLs. Use `additional_providers` to specify additional files with oEmbed configuration (each +should be in the form of providers.json). By default this list is empty. + +Example configuration: +```yaml +oembed: + disable_default_providers: true + additional_providers: + - oembed/my_providers.json +``` +--- +## Captcha ## + +See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. + +--- +Config option: `recaptcha_public_key` + +This homeserver's ReCAPTCHA public key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_public_key: "YOUR_PUBLIC_KEY" +``` +--- +Config option: `recaptcha_private_key` + +This homeserver's ReCAPTCHA private key. Must be specified if `enable_registration_captcha` is +enabled. + +Example configuration: +```yaml +recaptcha_private_key: "YOUR_PRIVATE_KEY" +``` +--- +Config option: `enable_registration_captcha` + +Set to true to enable ReCaptcha checks when registering, preventing signup +unless a captcha is answered. Requires a valid ReCaptcha public/private key. +Defaults to false. + +Example configuration: +```yaml +enable_registration_captcha: true +``` +--- +Config option: `recaptcha_siteverify_api` + +The API endpoint to use for verifying `m.login.recaptcha` responses. +Defaults to `https://www.recaptcha.net/recaptcha/api/siteverify`. + +Example configuration: +```yaml +recaptcha_siteverify_api: "https://my.recaptcha.site" +``` +--- +## TURN ## +Options related to adding a TURN server to Synapse. + +--- +Config option: `turn_uris` + +The public URIs of the TURN server to give to clients. + +Example configuration: +```yaml +turn_uris: [turn:example.org] +``` +--- +Config option: `turn_shared_secret` + +The shared secret used to compute passwords for the TURN server. + +Example configuration: +```yaml +turn_shared_secret: "YOUR_SHARED_SECRET" +``` +---- +Config options: `turn_username` and `turn_password` + +The Username and password if the TURN server needs them and does not use a token. + +Example configuration: +```yaml +turn_username: "TURNSERVER_USERNAME" +turn_password: "TURNSERVER_PASSWORD" +``` +--- +Config option: `turn_user_lifetime` + +How long generated TURN credentials last. Defaults to 1h. + +Example configuration: +```yaml +turn_user_lifetime: 2h +``` +--- +Config option: `turn_allow_guests` + +Whether guests should be allowed to use the TURN server. This defaults to true, otherwise +VoIP will be unreliable for guests. However, it does introduce a slight security risk as +it allows users to connect to arbitrary endpoints without having first signed up for a valid account (e.g. by passing a CAPTCHA). + +Example configuration: +```yaml +turn_allow_guests: false +``` +--- +## Registration ## + +Registration can be rate-limited using the parameters in the [Ratelimiting](#ratelimiting) section of this manual. + +--- +Config option: `enable_registration` + +Enable registration for new users. Defaults to false. It is highly recommended that if you enable registration, +you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration +without any verification, you must also set `enable_registration_without_verification` to true. + +Example configuration: +```yaml +enable_registration: true +``` +--- +Config option: `enable_registration_without_verification` +Enable registration without email or captcha verification. Note: this option is *not* recommended, +as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect +unless `enable_registration` is also enabled. + +Example configuration: +```yaml +enable_registration_without_verification: true +``` +--- +Config option: `session_lifetime` + +Time that a user's session remains valid for, after they log in. + +Note that this is not currently compatible with guest logins. + +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already +logged in. + +By default, this is infinite. + +Example configuration: +```yaml +session_lifetime: 24h +``` +---- +Config option: `refresh_access_token_lifetime` + +Time that an access token remains valid for, if the session is using refresh tokens. + +For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). + +Note that this only applies to clients which advertise support for refresh tokens. + +Note also that this is calculated at login time and refresh time: changes are not applied to +existing sessions until they are refreshed. + +By default, this is 5 minutes. + +Example configuration: +```yaml +refreshable_access_token_lifetime: 10m +``` +--- +Config option: `refresh_token_lifetime: 24h` + +Time that a refresh token remains valid for (provided that it is not +exchanged for another one first). +This option can be used to automatically log-out inactive sessions. +Please see the manual for more information. + +Note also that this is calculated at login time and refresh time: +changes are not applied to existing sessions until they are refreshed. + +By default, this is infinite. + +Example configuration: +```yaml +refresh_token_lifetime: 24h +``` +--- +Config option: `nonrefreshable_access_token_lifetime` + +Time that an access token remains valid for, if the session is NOT +using refresh tokens. + +Please note that not all clients support refresh tokens, so setting +this to a short value may be inconvenient for some users who will +then be logged out frequently. + +Note also that this is calculated at login time: changes are not applied +retrospectively to existing sessions for users that have already logged in. + +By default, this is infinite. + +Example configuration: +```yaml +nonrefreshable_access_token_lifetime: 24h +``` +--- +Config option: `registrations_require_3pid` + +If this is set, the user must provide all of the specified types of 3PID when registering. + +Example configuration: +```yaml +registrations_require_3pid: + - email + - msisdn +``` +--- +Config option: `disable_msisdn_registration` + +Explicitly disable asking for MSISDNs from the registration +flow (overrides `registrations_require_3pid` if MSISDNs are set as required). + +Example configuration: +```yaml +disable_msisdn_registration: true +``` +--- +Config option: `allowed_local_3pids` + +Mandate that users are only allowed to associate certain formats of +3PIDs with accounts on this server, as specified by the `medium` and `pattern` sub-options. + +Example configuration: +```yaml +allowed_local_3pids: + - medium: email + pattern: '^[^@]+@matrix\.org$' + - medium: email + pattern: '^[^@]+@vector\.im$' + - medium: msisdn + pattern: '\+44' +``` +--- +Config option: `enable_3pid_lookup` + +Enable 3PIDs lookup requests to identity servers from this server. Defaults to true. + +Example configuration: +```yaml +enable_3pid_lookup: false +``` +--- +Config option: `registration_requires_token` + +Require users to submit a token during registration. +Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). +Note that `enable_registration` must be set to true. +Disabling this option will not delete any tokens previously generated. +Defaults to false. Set to true to enable. + +Example configuration: +```yaml +registration_requires_token: true +``` +--- +Config option: `registration_shared_secret` + +If set, allows registration of standard or admin accounts by anyone who +has the shared secret, even if registration is otherwise disabled. + +Example configuration: +```yaml +registration_shared_secret: +``` +--- +Config option: `bcrypt_rounds` + +Set the number of bcrypt rounds used to generate password hash. +Larger numbers increase the work factor needed to generate the hash. +The default number is 12 (which equates to 2^12 rounds). +N.B. that increasing this will exponentially increase the time required +to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. +Example configuration: +```yaml +bcrypt_rounds: 14 +``` +--- +Config option: `allow_guest_access` + +Allows users to register as guests without a password/email/etc, and +participate in rooms hosted on this server which have been made +accessible to anonymous users. Defaults to false. + +Example configuration: +```yaml +allow_guest_access: true +``` +--- +Config option: `default_identity_server` + +The identity server which we suggest that clients should use when users log +in on this server. + +(By default, no suggestion is made, so it is left up to the client. +This setting is ignored unless `public_baseurl` is also explicitly set.) + +Example configuration: +```yaml +default_identity_server: https://matrix.org +``` +--- +Config option: `account_threepid_delegates` + +Handle threepid (email/phone etc) registration and password resets through a set of +*trusted* identity servers. Note that this allows the configured identity server to +reset passwords for accounts! + +Be aware that if `email` is not set, and SMTP options have not been +configured in the email config block, registration and user password resets via +email will be globally disabled. + +Additionally, if `msisdn` is not set, registration and password resets via msisdn +will be disabled regardless, and users will not be able to associate an msisdn +identifier to their account. This is due to Synapse currently not supporting +any method of sending SMS messages on its own. + +To enable using an identity server for operations regarding a particular third-party +identifier type, set the value to the URL of that identity server as shown in the +examples below. + +Servers handling the these requests must answer the `/requestToken` endpoints defined +by the Matrix Identity Service API [specification](https://matrix.org/docs/spec/identity_service/latest). + +Example configuration: +```yaml +account_threepid_delegates: + email: https://example.com # Delegate email sending to example.com + msisdn: http://localhost:8090 # Delegate SMS sending to this local process +``` +--- +Config option: `enable_set_displayname` + +Whether users are allowed to change their displayname after it has +been initially set. Useful when provisioning users based on the +contents of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_displayname: false +``` +--- +Config option: `enable_set_avatar_url` + +Whether users are allowed to change their avatar after it has been +initially set. Useful when provisioning users based on the contents +of a third-party directory. + +Does not apply to server administrators. Defaults to true. + +Example configuration: +```yaml +enable_set_avatar_url: false +``` +--- +Config option: `enable_3pid_changes` + +Whether users can change the third-party IDs associated with their accounts +(email address and msisdn). + +Defaults to true. + +Example configuration: +```yaml +enable_3pid_changes: false +``` +--- +Config option: `auto_join_rooms` + +Users who register on this homeserver will automatically be joined +to the rooms listed under this option. + +By default, any room aliases included in this list will be created +as a publicly joinable room when the first user registers for the +homeserver. If the room already exists, make certain it is a publicly joinable +room, i.e. the join rule of the room must be set to 'public'. You can find more options +relating to auto-joining rooms below. + +Example configuration: +```yaml +auto_join_rooms: + - "#exampleroom:example.com" + - "#anotherexampleroom:example.com" +``` +--- +Config option: `autocreate_auto_join_rooms` + +Where `auto_join_rooms` are specified, setting this flag ensures that +the rooms exist by creating them when the first user on the +homeserver registers. + +By default the auto-created rooms are publicly joinable from any federated +server. Use the `autocreate_auto_join_rooms_federated` and +`autocreate_auto_join_room_preset` settings to customise this behaviour. + +Setting to false means that if the rooms are not manually created, +users cannot be auto-joined since they do not exist. + +Defaults to true. + +Example configuration: +```yaml +autocreate_auto_join_rooms: false +``` +--- +Config option: `autocreate_auto_join_rooms_federated` + +Whether the rooms listen in `auto_join_rooms` that are auto-created are available +via federation. Only has an effect if `autocreate_auto_join_rooms` is true. + +Note that whether a room is federated cannot be modified after +creation. + +Defaults to true: the room will be joinable from other servers. +Set to false to prevent users from other homeservers from +joining these rooms. + +Example configuration: +```yaml +autocreate_auto_join_rooms_federated: false +``` +--- +Config option: `autocreate_auto_join_room_preset` + +The room preset to use when auto-creating one of `auto_join_rooms`. Only has an +effect if `autocreate_auto_join_rooms` is true. + +Possible values for this option are: +* "public_chat": the room is joinable by anyone, including + federated servers if `autocreate_auto_join_rooms_federated` is true (the default). +* "private_chat": an invitation is required to join these rooms. +* "trusted_private_chat": an invitation is required to join this room and the invitee is + assigned a power level of 100 upon joining the room. + +If a value of "private_chat" or "trusted_private_chat" is used then +`auto_join_mxid_localpart` must also be configured. + +Defaults to "public_chat". + +Example configuration: +```yaml +autocreate_auto_join_room_preset: private_chat +``` +--- +Config option: `auto_join_mxid_localpart` + +The local part of the user id which is used to create `auto_join_rooms` if +`autocreate_auto_join_rooms` is true. If this is not provided then the +initial user account that registers will be used to create the rooms. + +The user id is also used to invite new users to any auto-join rooms which +are set to invite-only. + +It *must* be configured if `autocreate_auto_join_room_preset` is set to +"private_chat" or "trusted_private_chat". + +Note that this must be specified in order for new users to be correctly +invited to any auto-join rooms which have been set to invite-only (either +at the time of creation or subsequently). + +Note that, if the room already exists, this user must be joined and +have the appropriate permissions to invite new members. + +Example configuration: +```yaml +auto_join_mxid_localpart: system +``` +--- +Config option: `auto_join_rooms_for_guests` + +When `auto_join_rooms` is specified, setting this flag to false prevents +guest accounts from being automatically joined to the rooms. + +Defaults to true. + +Example configuration: +```yaml +auto_join_rooms_for_guests: false +``` +--- +Config option: `inhibit_user_in_use_error` + +Whether to inhibit errors raised when registering a new account if the user ID +already exists. If turned on, requests to `/register/available` will always +show a user ID as available, and Synapse won't raise an error when starting +a registration with a user ID that already exists. However, Synapse will still +raise an error if the registration completes and the username conflicts. + +Defaults to false. + +Example configuration: +```yaml +inhibit_user_in_use_error: true +``` +--- +## Metrics ### +Config options related to metrics. + +--- +Config option: `enable_metrics` + +Set to true to enable collection and rendering of performance metrics. +Defaults to false. + +Example configuration: +```yaml +enable_metrics: true +``` +--- +Config option: `sentry` + +Use this option to enable sentry integration. Provide the DSN assigned to you by sentry +with the `dsn` setting. + +NOTE: While attempts are made to ensure that the logs don't contain +any sensitive information, this cannot be guaranteed. By enabling +this option the sentry server may therefore receive sensitive +information, and it in turn may then disseminate sensitive information +through insecure notification channels if so configured. + +Example configuration: +```yaml +sentry: + dsn: "..." +``` +--- +Config option: `metrics_flags` + +Flags to enable Prometheus metrics which are not suitable to be +enabled by default, either for performance reasons or limited use. +Currently the only option is `known_servers`, which publishes +`synapse_federation_known_servers`, a gauge of the number of +servers this homeserver knows about, including itself. May cause +performance problems on large homeservers. + +Example configuration: +```yaml +metrics_flags: + known_servers: true +``` +--- +Config option: `report_stats` + +Whether or not to report anonymized homeserver usage statistics. This is originally +set when generating the config. Set this option to true or false to change the current +behavior. + +Example configuration: +```yaml +report_stats: true +``` +--- +Config option: `report_stats_endpoint` + +The endpoint to report the anonymized homeserver usage statistics to. +Defaults to https://matrix.org/report-usage-stats/push + +Example configuration: +```yaml +report_stats_endpoint: https://example.com/report-usage-stats/push +``` +--- +## API Configuration ## +Config settings related to the client/server API + +--- +Config option: `room_prejoin_state:` + +Controls for the state that is shared with users who receive an invite +to a room. By default, the following state event types are shared with users who +receive invites to the room: +- m.room.join_rules +- m.room.canonical_alias +- m.room.avatar +- m.room.encryption +- m.room.name +- m.room.create +- m.room.topic + +To change the default behavior, use the following sub-options: +* `disable_default_event_types`: set to true to disable the above defaults. If this + is enabled, only the event types listed in `additional_event_types` are shared. + Defaults to false. +* `additional_event_types`: Additional state event types to share with users when they are invited + to a room. By default, this list is empty (so only the default event types are shared). + +Example configuration: +```yaml +room_prejoin_state: + disable_default_event_types: true + additional_event_types: + - org.example.custom.event.type + - m.room.join_rules +``` +--- +Config option: `track_puppeted_user_ips` + +We record the IP address of clients used to access the API for various +reasons, including displaying it to the user in the "Where you're signed in" +dialog. + +By default, when puppeting another user via the admin API, the client IP +address is recorded against the user who created the access token (ie, the +admin user), and *not* the puppeted user. + +Set this option to true to also record the IP address against the puppeted +user. (This also means that the puppeted user will count as an "active" user +for the purpose of monthly active user tracking - see `limit_usage_by_mau` etc +above.) + +Example configuration: +```yaml +track_puppeted_user_ips: true +``` +--- +Config option: `app_service_config_files` + +A list of application service config files to use. + +Example configuration: +```yaml +app_service_config_files: + - app_service_1.yaml + - app_service_2.yaml +``` +--- +Config option: `track_appservice_user_ips` + +Defaults to false. Set to true to enable tracking of application service IP addresses. +Implicitly enables MAU tracking for application service users. + +Example configuration: +```yaml +track_appservice_user_ips: true +``` +--- +Config option: `macaroon_secret_key` + +A secret which is used to sign access tokens. If none is specified, +the `registration_shared_secret` is used, if one is given; otherwise, +a secret key is derived from the signing key. + +Example configuration: +```yaml +macaroon_secret_key: +``` +--- +Config option: `form_secret` + +A secret which is used to calculate HMACs for form values, to stop +falsification of values. Must be specified for the User Consent +forms to work. + +Example configuration: +```yaml +form_secret: +``` +--- +## Signing Keys ## +Config options relating to signing keys + +--- +Config option: `signing_key_path` + +Path to the signing key to sign messages with. + +Example configuration: +```yaml +signing_key_path: "CONFDIR/SERVERNAME.signing.key" +``` +--- +Config option: `old_signing_keys` + +The keys that the server used to sign messages with but won't use +to sign new messages. For each key, `key` should be the base64-encoded public key, and +`expired_ts`should be the time (in milliseconds since the unix epoch) that +it was last used. + +It is possible to build an entry from an old `signing.key` file using the +`export_signing_key` script which is provided with synapse. + +Example configuration: +```yaml +old_signing_keys: + "ed25519:id": { key: "base64string", expired_ts: 123456789123 } +``` +--- +Config option: `key_refresh_interval` + +How long key response published by this server is valid for. +Used to set the `valid_until_ts` in `/key/v2` APIs. +Determines how quickly servers will query to check which keys +are still valid. Defaults to 1d. + +Example configuration: +```yaml +key_refresh_interval: 2d +``` +--- +Config option: `trusted_key_servers:` + +The trusted servers to download signing keys from. + +When we need to fetch a signing key, each server is tried in parallel. + +Normally, the connection to the key server is validated via TLS certificates. +Additional security can be provided by configuring a `verify key`, which +will make synapse check that the response is signed by that key. + +This setting supercedes an older setting named `perspectives`. The old format +is still supported for backwards-compatibility, but it is deprecated. + +`trusted_key_servers` defaults to matrix.org, but using it will generate a +warning on start-up. To suppress this warning, set +`suppress_key_server_warning` to true. + +Options for each entry in the list include: +* `server_name`: the name of the server. Required. +* `verify_keys`: an optional map from key id to base64-encoded public key. + If specified, we will check that the response is signed by at least + one of the given keys. +* `accept_keys_insecurely`: a boolean. Normally, if `verify_keys` is unset, + and `federation_verify_certificates` is not `true`, synapse will refuse + to start, because this would allow anyone who can spoof DNS responses + to masquerade as the trusted key server. If you know what you are doing + and are sure that your network environment provides a secure connection + to the key server, you can set this to `true` to override this behaviour. + +Example configuration #1: +```yaml +trusted_key_servers: + - server_name: "my_trusted_server.example.com" + verify_keys: + "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" + - server_name: "my_other_trusted_server.example.com" +``` +Example configuration #2: +```yaml +trusted_key_servers: + - server_name: "matrix.org" +``` +--- +Config option: `suppress_key_server_warning` + +Set the following to true to disable the warning that is emitted when the +`trusted_key_servers` include 'matrix.org'. See above. + +Example configuration: +```yaml +suppress_key_server_warning: true +``` +--- +Config option: `key_server_signing_keys_path` + +The signing keys to use when acting as a trusted key server. If not specified +defaults to the server signing key. + +Can contain multiple keys, one per line. + +Example configuration: +```yaml +key_server_signing_keys_path: "key_server_signing_keys.key" +``` +--- +## Single sign-on integration ## + +The following settings can be used to make Synapse use a single sign-on +provider for authentication, instead of its internal password database. + +You will probably also want to set the following options to false to +disable the regular login/registration flows: + * `enable_registration` + * `password_config.enabled` + +You will also want to investigate the settings under the "sso" configuration +section below. + +--- +Config option: `saml2_config` + +Enable SAML2 for registration and login. Uses pysaml2. To learn more about pysaml and +to find a full list options for configuring pysaml, read the docs [here](https://pysaml2.readthedocs.io/en/latest/). + +At least one of `sp_config` or `config_path` must be set in this section to +enable SAML login. You can either put your entire pysaml config inline using the `sp_config` +option, or you can specify a path to a psyaml config file with the sub-option `config_path`. +This setting has the following sub-options: + +* `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config. + Default values will be used for the `entityid` and `service` settings, + so it is not normally necessary to specify them unless you need to + override them. Here are a few useful sub-options for configuring pysaml: + * `metadata`: Point this to the IdP's metadata. You must provide either a local + file via the `local` attribute or (preferably) a URL via the + `remote` attribute. + * `accepted_time_diff: 3`: Allowed clock difference in seconds between the homeserver and IdP. + Defaults to 0. + * `service`: By default, the user has to go to our login page first. If you'd like + to allow IdP-initiated login, set `allow_unsolicited` to true under `sp` in the `service` + section. +* `config_path`: specify a separate pysaml2 configuration file thusly: + `config_path: "CONFDIR/sp_conf.py"` +* `saml_session_lifetime`: The lifetime of a SAML session. This defines how long a user has to + complete the authentication process, if `allow_unsolicited` is unset. The default is 15 minutes. +* `user_mapping_provider`: Using this option, an external module can be provided as a + custom solution to mapping attributes returned from a saml provider onto a matrix user. The + `user_mapping_provider` has the following attributes: + * `module`: The custom module's class. + * `config`: Custom configuration values for the module. Use the values provided in the + example if you are using the built-in user_mapping_provider, or provide your own + config values for a custom class if you are using one. This section will be passed as a Python + dictionary to the module's `parse_config` method. The built-in provider takes the following two + options: + * `mxid_source_attribute`: The SAML attribute (after mapping via the attribute maps) to use + to derive the Matrix ID from. It is 'uid' by default. Note: This used to be configured by the + `saml2_config.mxid_source_attribute option`. If that is still defined, its value will be used instead. + * `mxid_mapping`: The mapping system to use for mapping the saml attribute onto a + matrix ID. Options include: `hexencode` (which maps unpermitted characters to '=xx') + and `dotreplace` (which replaces unpermitted characters with '.'). + The default is `hexencode`. Note: This used to be configured by the + `saml2_config.mxid_mapping option`. If that is still defined, its value will be used instead. +* `grandfathered_mxid_source_attribute`: In previous versions of synapse, the mapping from SAML attribute to + MXID was always calculated dynamically rather than stored in a table. For backwards- compatibility, we will look for `user_ids` + matching such a pattern before creating a new account. This setting controls the SAML attribute which will be used for this + backwards-compatibility lookup. Typically it should be 'uid', but if the attribute maps are changed, it may be necessary to change it. + The default is 'uid'. +* `attribute_requirements`: It is possible to configure Synapse to only allow logins if SAML attributes + match particular values. The requirements can be listed under + `attribute_requirements` as shown in the example. All of the listed attributes must + match for the login to be permitted. +* `idp_entityid`: If the metadata XML contains multiple IdP entities then the `idp_entityid` + option must be set to the entity to redirect users to. + Most deployments only have a single IdP entity and so should omit this option. + + +Once SAML support is enabled, a metadata file will be exposed at +`https://:/_synapse/client/saml2/metadata.xml`, which you may be able to +use to configure your SAML IdP with. Alternatively, you can manually configure +the IdP to use an ACS location of +`https://:/_synapse/client/saml2/authn_response`. + +Example configuration: +```yaml +saml2_config: + sp_config: + metadata: + local: ["saml2/idp.xml"] + remote: + - url: https://our_idp/metadata.xml + accepted_time_diff: 3 + + service: + sp: + allow_unsolicited: true + + # The examples below are just used to generate our metadata xml, and you + # may well not need them, depending on your setup. Alternatively you + # may need a whole lot more detail - see the pysaml2 docs! + description: ["My awesome SP", "en"] + name: ["Test SP", "en"] + + ui_info: + display_name: + - lang: en + text: "Display Name is the descriptive name of your service." + description: + - lang: en + text: "Description should be a short paragraph explaining the purpose of the service." + information_url: + - lang: en + text: "https://example.com/terms-of-service" + privacy_statement_url: + - lang: en + text: "https://example.com/privacy-policy" + keywords: + - lang: en + text: ["Matrix", "Element"] + logo: + - lang: en + text: "https://example.com/logo.svg" + width: "200" + height: "80" + + organization: + name: Example com + display_name: + - ["Example co", "en"] + url: "http://example.com" + + contact_person: + - given_name: Bob + sur_name: "the Sysadmin" + email_address": ["admin@example.com"] + contact_type": technical + + saml_session_lifetime: 5m + + user_mapping_provider: + # Below options are intended for the built-in provider, they should be + # changed if using a custom module. + config: + mxid_source_attribute: displayName + mxid_mapping: dotreplace + + grandfathered_mxid_source_attribute: upn + + attribute_requirements: + - attribute: userGroup + value: "staff" + - attribute: department + value: "sales" + + idp_entityid: 'https://our_idp/entityid' +``` +--- +Config option: `oidc_providers` + +List of OpenID Connect (OIDC) / OAuth 2.0 identity providers, for registration +and login. See [here](../../openid.md) +for information on how to configure these options. + +For backwards compatibility, it is also possible to configure a single OIDC +provider via an `oidc_config` setting. This is now deprecated and admins are +advised to migrate to the `oidc_providers` format. (When doing that migration, +use `oidc` for the `idp_id` to ensure that existing users continue to be +recognised.) + +Options for each entry include: +* `idp_id`: a unique identifier for this identity provider. Used internally + by Synapse; should be a single word such as 'github'. + Note that, if this is changed, users authenticating via that provider + will no longer be recognised as the same user! + (Use "oidc" here if you are migrating from an old `oidc_config` configuration.) + +* `idp_name`: A user-facing name for this identity provider, which is used to + offer the user a choice of login mechanisms. + +* `idp_icon`: An optional icon for this identity provider, which is presented + by clients and Synapse's own IdP picker page. If given, must be an + MXC URI of the format mxc:///. (An easy way to + obtain such an MXC URI is to upload an image to an (unencrypted) room + and then copy the "url" from the source of the event.) + +* `idp_brand`: An optional brand for this identity provider, allowing clients + to style the login flow according to the identity provider in question. + See the [spec](https://spec.matrix.org/latest/) for possible options here. + +* `discover`: set to false to disable the use of the OIDC discovery mechanism + to discover endpoints. Defaults to true. + +* `issuer`: Required. The OIDC issuer. Used to validate tokens and (if discovery + is enabled) to discover the provider's endpoints. + +* `client_id`: Required. oauth2 client id to use. + +* `client_secret`: oauth2 client secret to use. May be omitted if + `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'. + +* `client_secret_jwt_key`: Alternative to client_secret: details of a key used + to create a JSON Web Token to be used as an OAuth2 client secret. If + given, must be a dictionary with the following properties: + + * `key`: a pem-encoded signing key. Must be a suitable key for the + algorithm specified. Required unless `key_file` is given. + + * `key_file`: the path to file containing a pem-encoded signing key file. + Required unless `key` is given. + + * `jwt_header`: a dictionary giving properties to include in the JWT + header. Must include the key `alg`, giving the algorithm used to + sign the JWT, such as "ES256", using the JWA identifiers in + RFC7518. + + * `jwt_payload`: an optional dictionary giving properties to include in + the JWT payload. Normally this should include an `iss` key. + +* `client_auth_method`: auth method to use when exchanging the token. Valid + values are `client_secret_basic` (default), `client_secret_post` and + `none`. + +* `scopes`: list of scopes to request. This should normally include the "openid" + scope. Defaults to ["openid"]. + +* `authorization_endpoint`: the oauth2 authorization endpoint. Required if + provider discovery is disabled. + +* `token_endpoint`: the oauth2 token endpoint. Required if provider discovery is + disabled. + +* `userinfo_endpoint`: the OIDC userinfo endpoint. Required if discovery is + disabled and the 'openid' scope is not requested. + +* `jwks_uri`: URI where to fetch the JWKS. Required if discovery is disabled and + the 'openid' scope is used. + +* `skip_verification`: set to 'true' to skip metadata verification. Use this if + you are connecting to a provider that is not OpenID Connect compliant. + Defaults to false. Avoid this in production. + +* `user_profile_method`: Whether to fetch the user profile from the userinfo + endpoint, or to rely on the data returned in the id_token from the `token_endpoint`. + Valid values are: `auto` or `userinfo_endpoint`. + Defaults to `auto`, which uses the userinfo endpoint if `openid` is + not included in `scopes`. Set to `userinfo_endpoint` to always use the + userinfo endpoint. + +* `allow_existing_users`: set to true to allow a user logging in via OIDC to + match a pre-existing account instead of failing. This could be used if + switching from password logins to OIDC. Defaults to false. + +* `user_mapping_provider`: Configuration for how attributes returned from a OIDC + provider are mapped onto a matrix user. This setting has the following + sub-properties: + + * `module`: The class name of a custom mapping module. Default is + `synapse.handlers.oidc.JinjaOidcMappingProvider`. + See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers + for information on implementing a custom mapping provider. + + * `config`: Configuration for the mapping provider module. This section will + be passed as a Python dictionary to the user mapping provider + module's `parse_config` method. + + For the default provider, the following settings are available: + + * subject_claim: name of the claim containing a unique identifier + for the user. Defaults to 'sub', which OpenID Connect + compliant providers should provide. + + * `localpart_template`: Jinja2 template for the localpart of the MXID. + If this is not set, the user will be prompted to choose their + own username (see the documentation for the `sso_auth_account_details.html` + template). This template can use the `localpart_from_email` filter. + + * `confirm_localpart`: Whether to prompt the user to validate (or + change) the generated localpart (see the documentation for the + 'sso_auth_account_details.html' template), instead of + registering the account right away. + + * `display_name_template`: Jinja2 template for the display name to set + on first login. If unset, no displayname will be set. + + * `email_template`: Jinja2 template for the email address of the user. + If unset, no email address will be added to the account. + + * `extra_attributes`: a map of Jinja2 templates for extra attributes + to send back to the client during login. Note that these are non-standard and clients will ignore them + without modifications. + + When rendering, the Jinja2 templates are given a 'user' variable, + which is set to the claims returned by the UserInfo Endpoint and/or + in the ID Token. + + +It is possible to configure Synapse to only allow logins if certain attributes +match particular values in the OIDC userinfo. The requirements can be listed under +`attribute_requirements` as shown here: +```yaml +attribute_requirements: + - attribute: family_name + value: "Stephensson" + - attribute: groups + value: "admin" +``` +All of the listed attributes must match for the login to be permitted. Additional attributes can be added to +userinfo by expanding the `scopes` section of the OIDC config to retrieve +additional information from the OIDC provider. + +If the OIDC claim is a list, then the attribute must match any value in the list. +Otherwise, it must exactly match the value of the claim. Using the example +above, the `family_name` claim MUST be "Stephensson", but the `groups` +claim MUST contain "admin". + +Example configuration: +```yaml +oidc_providers: + # Generic example + # + - idp_id: my_idp + idp_name: "My OpenID provider" + idp_icon: "mxc://example.com/mediaid" + discover: false + issuer: "https://accounts.example.com/" + client_id: "provided-by-your-issuer" + client_secret: "provided-by-your-issuer" + client_auth_method: client_secret_post + scopes: ["openid", "profile"] + authorization_endpoint: "https://accounts.example.com/oauth2/auth" + token_endpoint: "https://accounts.example.com/oauth2/token" + userinfo_endpoint: "https://accounts.example.com/userinfo" + jwks_uri: "https://accounts.example.com/.well-known/jwks.json" + skip_verification: true + user_mapping_provider: + config: + subject_claim: "id" + localpart_template: "{{ user.login }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" + attribute_requirements: + - attribute: userGroup + value: "synapseUsers" +``` +--- +Config option: `cas_config` + +Enable Central Authentication Service (CAS) for registration and login. +Has the following sub-options: +* `enabled`: Set this to true to enable authorization against a CAS server. + Defaults to false. +* `server_url`: The URL of the CAS authorization endpoint. +* `displayname_attribute`: The attribute of the CAS response to use as the display name. + If no name is given here, no displayname will be set. +* `required_attributes`: It is possible to configure Synapse to only allow logins if CAS attributes + match particular values. All of the keys given below must exist + and the values must match the given value. Alternately if the given value + is `None` then any value is allowed (the attribute just must exist). + All of the listed attributes must match for the login to be permitted. + +Example configuration: +```yaml +cas_config: + enabled: true + server_url: "https://cas-server.com" + displayname_attribute: name + required_attributes: + userGroup: "staff" + department: None +``` +--- +Config option: `sso` + +Additional settings to use with single-sign on systems such as OpenID Connect, +SAML2 and CAS. + +Server admins can configure custom templates for pages related to SSO. See +[here](../../templates.md) for more information. + +Options include: +* `client_whitelist`: A list of client URLs which are whitelisted so that the user does not + have to confirm giving access to their account to the URL. Any client + whose URL starts with an entry in the following list will not be subject + to an additional confirmation step after the SSO login is completed. + WARNING: An entry such as "https://my.client" is insecure, because it + will also match "https://my.client.evil.site", exposing your users to + phishing attacks from evil.site. To avoid this, include a slash after the + hostname: "https://my.client/". + The login fallback page (used by clients that don't natively support the + required login flows) is whitelisted in addition to any URLs in this list. + By default, this list contains only the login fallback page. +* `update_profile_information`: Use this setting to keep a user's profile fields in sync with information from + the identity provider. Currently only syncing the displayname is supported. Fields + are checked on every SSO login, and are updated if necessary. + Note that enabling this option will override user profile information, + regardless of whether users have opted-out of syncing that + information when first signing in. Defaults to false. + + +Example configuration: +```yaml +sso: + client_whitelist: + - https://riot.im/develop + - https://my.custom.client/ + update_profile_information: true +``` +--- +Config option: `jwt_config` + +JSON web token integration. The following settings can be used to make +Synapse JSON web tokens for authentication, instead of its internal +password database. + +Each JSON Web Token needs to contain a "sub" (subject) claim, which is +used as the localpart of the mxid. + +Additionally, the expiration time ("exp"), not before time ("nbf"), +and issued at ("iat") claims are validated if present. + +Note that this is a non-standard login type and client support is +expected to be non-existent. + +See [here](../../jwt.md) for more. + +Additional sub-options for this setting include: +* `enabled`: Set to true to enable authorization using JSON web + tokens. Defaults to false. +* `secret`: This is either the private shared secret or the public key used to + decode the contents of the JSON web token. Required if `enabled` is set to true. +* `algorithm`: The algorithm used to sign the JSON web token. Supported algorithms are listed at + https://pyjwt.readthedocs.io/en/latest/algorithms.html Required if `enabled` is set to true. +* `subject_claim`: Name of the claim containing a unique identifier for the user. + Optional, defaults to `sub`. +* `issuer`: The issuer to validate the "iss" claim against. Optional. If provided the + "iss" claim will be required and validated for all JSON web tokens. +* `audiences`: A list of audiences to validate the "aud" claim against. Optional. + If provided the "aud" claim will be required and validated for all JSON web tokens. + Note that if the "aud" claim is included in a JSON web token then + validation will fail without configuring audiences. + +Example configuration: +```yaml +jwt_config: + enabled: true + secret: "provided-by-your-issuer" + algorithm: "provided-by-your-issuer" + subject_claim: "name_of_claim" + issuer: "provided-by-your-issuer" + audiences: + - "provided-by-your-issuer" +``` +--- +Config option: `password_config` + +Use this setting to enable password-based logins. + +This setting has the following sub-options: +* `enabled`: Defaults to true. +* `localdb_enabled`: Set to false to disable authentication against the local password + database. This is ignored if `enabled` is false, and is only useful + if you have other `password_providers`. Defaults to true. +* `pepper`: Set the value here to a secret random string for extra security. # Uncomment and change to a secret random string for extra security. + DO NOT CHANGE THIS AFTER INITIAL SETUP! +* `policy`: Define and enforce a password policy, such as minimum lengths for passwords, etc. + Each parameter is optional. This is an implementation of MSC2000. Parameters are as follows: + * `enabled`: Defaults to false. Set to true to enable. + * `minimum_length`: Minimum accepted length for a password. Defaults to 0. + * `require_digit`: Whether a password must contain at least one digit. + Defaults to false. + * `require_symbol`: Whether a password must contain at least one symbol. + A symbol is any character that's not a number or a letter. Defaults to false. + * `require_lowercase`: Whether a password must contain at least one lowercase letter. + Defaults to false. + * `require_uppercase`: Whether a password must contain at least one uppercase letter. + Defaults to false. + + +Example configuration: +```yaml +password_config: + enabled: false + localdb_enabled: false + pepper: "EVEN_MORE_SECRET" + + policy: + enabled: true + minimum_length: 15 + require_digit: true + require_symbol: true + require_lowercase: true + require_uppercase: true +``` +--- +Config option: `ui_auth` + +The amount of time to allow a user-interactive authentication session to be active. + +This defaults to 0, meaning the user is queried for their credentials +before every action, but this can be overridden to allow a single +validation to be re-used. This weakens the protections afforded by +the user-interactive authentication process, by allowing for multiple +(and potentially different) operations to use the same validation session. + +This is ignored for potentially "dangerous" operations (including +deactivating an account, modifying an account password, and +adding a 3PID). + +Use the `session_timeout` sub-option here to change the time allowed for credential validation. + +Example configuration: +```yaml +ui_auth: + session_timeout: "15s" +``` +--- +Config option: `email` + +Configuration for sending emails from Synapse. + +Server admins can configure custom templates for email content. See +[here](../../templates.md) for more information. + +This setting has the following sub-options: +* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'. +* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 25. +* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no + authentication is attempted. +* `require_transport_security`: Set to true to require TLS transport security for SMTP. + By default, Synapse will connect over plain text, and will then switch to + TLS via STARTTLS *if the SMTP server supports it*. If this option is set, + Synapse will refuse to connect unless the server supports STARTTLS. +* `enable_tls`: By default, if the server supports TLS, it will be used, and the server + must present a certificate that is valid for 'smtp_host'. If this option + is set to false, TLS will not be used. +* `notif_from`: defines the "From" address to use when sending emails. + It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name, + which is normally set in `app_name`, but may be overridden by the + Matrix client application. Note that the placeholder must be written '%(app)s', including the + trailing 's'. +* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email + subjects. It defaults to 'Matrix'. +* `enable_notifs`: Set to true to enable sending emails for messages that the user + has missed. Disabled by default. +* `notif_for_new_users`: Set to false to disable automatic subscription to email + notifications for new users. Enabled by default. +* `client_base_url`: Custom URL for client links within the email notifications. By default + links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`; + the old name is still supported for backwards-compatibility but is now deprecated.) +* `validation_token_lifetime`: Configures the time that a validation email will expire after sending. + Defaults to 1h. +* `invite_client_location`: The web client location to direct users to during an invite. This is passed + to the identity server as the `org.matrix.web_client_location` key. Defaults + to unset, giving no guidance to the identity server. +* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will + be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application. + In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname + of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the + message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will + can use the '%(server_name)s' placeholder, which will be replaced by the value of the + `server_name` setting in your Synapse configuration. + + Here is a list of subjects for notification emails that can be set: + * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a + room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a + room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..." + * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in + a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..." + * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a + name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..." + * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms. + Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in + multiple rooms. This is similar to the setting above except it's used when + the room in which the notification was triggered has no name. Defaults to + "[%(app)s] You have messages on %(app)s from %(person)s and others..." + * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name. + Defaults to "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a + name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..." + * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset" + * `email_validation`: Subject to use when sending a verification email to assert an address's + ownership. Defaults to "[%(server_name)s] Validate your email" + +Example configuration: +```yaml +email: + smtp_host: mail.server + smtp_port: 587 + smtp_user: "exampleusername" + smtp_pass: "examplepassword" + require_transport_security: true + enable_tls: false + notif_from: "Your Friendly %(app)s homeserver " + app_name: my_branded_matrix_server + enable_notifs: true + notif_for_new_users: false + client_base_url: "http://localhost/riot" + validation_token_lifetime: 15m + invite_client_location: https://app.element.io + + subjects: + message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..." + message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..." + messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..." + messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..." + messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..." + messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..." + invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..." + invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..." + password_reset: "[%(server_name)s] Password reset" + email_validation: "[%(server_name)s] Validate your email" +``` +--- +## Push ## +Configuration settings related to push notifications + +--- +Config option: `push` + +This setting defines options for push notifications. + +This option has a number of sub-options. They are as follows: +* `include_content`: Clients requesting push notifications can either have the body of + the message sent in the notification poke along with other details + like the sender, or just the event ID and room ID (`event_id_only`). + If clients choose the to have the body sent, this option controls whether the + notification request includes the content of the event (other details + like the sender are still included). If `event_id_only` is enabled, it + has no effect. + For modern android devices the notification content will still appear + because it is loaded by the app. iPhone, however will send a + notification saying only that a message arrived and who it came from. + Defaults to true. Set to false to only include the event ID and room ID in push notification payloads. +* `group_unread_count_by_room: false`: When a push notification is received, an unread count is also sent. + This number can either be calculated as the number of unread messages for the user, or the number of *rooms* the + user has unread messages in. Defaults to true, meaning push clients will see the number of + rooms with unread messages in them. Set to false to instead send the number + of unread messages. + +Example configuration: +```yaml +push: + include_content: false + group_unread_count_by_room: false +``` +--- +## Rooms ## +Config options relating to rooms. + +--- +Config option: `encryption_enabled_by_default` + +Controls whether locally-created rooms should be end-to-end encrypted by +default. + +Possible options are "all", "invite", and "off". They are defined as: + +* "all": any locally-created room +* "invite": any room created with the `private_chat` or `trusted_private_chat` + room creation presets +* "off": this option will take no effect + +The default value is "off". + +Note that this option will only affect rooms created after it is set. It +will also not affect rooms created by other servers. + +Example configuration: +```yaml +encryption_enabled_by_default_for_room_type: invite +``` +--- +Config option: `enable_group_creation` + +Set to true to allow non-server-admin users to create groups on this server + +Example configuration: +```yaml +enable_group_creation: true +``` +--- +Config option: `group_creation_prefix` + +If enabled/present, non-server admins can only create groups with local parts +starting with this prefix. + +Example configuration: +```yaml +group_creation_prefix: "unofficial_" +``` +--- +Config option: `user_directory` + +This setting defines options related to the user directory. + +This option has the following sub-options: +* `enabled`: Defines whether users can search the user directory. If false then + empty responses are returned to all queries. Defaults to true. +* `search_all_users`: Defines whether to search all users visible to your HS when searching + the user directory. If false, search results will only contain users + visible in public rooms and users sharing a room with the requester. + Defaults to false. + NB. If you set this to true, and the last time the user_directory search + indexes were (re)built was before Synapse 1.44, you'll have to + rebuild the indexes in order to search through all known users. + These indexes are built the first time Synapse starts; admins can + manually trigger a rebuild via API following the instructions at + https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/background_updates.html#run + Set to true to return search results containing all known users, even if that + user does not share a room with the requester. +* `prefer_local_users`: Defines whether to prefer local users in search query results. + If set to true, local users are more likely to appear above remote users when searching the + user directory. Defaults to false. + +Example configuration: +```yaml +user_directory: + enabled: false + search_all_users: true + prefer_local_users: true +``` +--- +Config option: `user_consent` + +For detailed instructions on user consent configuration, see [here](../../consent_tracking.md). + +Parts of this section are required if enabling the `consent` resource under +`listeners`, in particular `template_dir` and `version`. # TODO: link `listeners` + +* `template_dir`: gives the location of the templates for the HTML forms. + This directory should contain one subdirectory per language (eg, `en`, `fr`), + and each language directory should contain the policy document (named as + .html) and a success page (success.html). + +* `version`: specifies the 'current' version of the policy document. It defines + the version to be served by the consent resource if there is no 'v' + parameter. + +* `server_notice_content`: if enabled, will send a user a "Server Notice" + asking them to consent to the privacy policy. The `server_notices` section ##TODO: link + must also be configured for this to work. Notices will *not* be sent to + guest users unless `send_server_notice_to_guests` is set to true. + +* `block_events_error`, if set, will block any attempts to send events + until the user consents to the privacy policy. The value of the setting is + used as the text of the error. + +* `require_at_registration`, if enabled, will add a step to the registration + process, similar to how captcha works. Users will be required to accept the + policy before their account is created. + +* `policy_name` is the display name of the policy users will see when registering + for an account. Has no effect unless `require_at_registration` is enabled. + Defaults to "Privacy Policy". + +Example configuration: +```yaml +user_consent: + template_dir: res/templates/privacy + version: 1.0 + server_notice_content: + msgtype: m.text + body: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + send_server_notice_to_guests: true + block_events_error: >- + To continue using this homeserver you must review and agree to the + terms and conditions at %(consent_uri)s + require_at_registration: false + policy_name: Privacy Policy +``` +--- +Config option: `stats` + +Settings for local room and user statistics collection. See [here](../../room_and_user_statistics.md) +for more. + +* `enabled`: Set to false to disable room and user statistics. Note that doing + so may cause certain features (such as the room directory) not to work + correctly. Defaults to true. + +Example configuration: +```yaml +stats: + enabled: false +``` +--- +Config option: `server_notices` + +Use this setting to enable a room which can be used to send notices +from the server to users. It is a special room which users cannot leave; notices +in the room come from a special "notices" user id. + +If you use this setting, you *must* define the `system_mxid_localpart` +sub-setting, which defines the id of the user which will be used to send the +notices. + +Sub-options for this setting include: +* `system_mxid_display_name`: set the display name of the "notices" user +* `system_mxid_avatar_url`: set the avatar for the "notices" user +* `room_name`: set the room name of the server notices room + +Example configuration: +```yaml +server_notices: + system_mxid_localpart: notices + system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: "Server Notices" +``` +--- +Config option: `enable_room_list_search` + +Set to false to disable searching the public room list. When disabled +blocks searching local and remote room lists for local and remote +users by always returning an empty list for all queries. Defaults to true. + +Example configuration: +```yaml +enable_room_list_search: false +``` +--- +Config option: `alias_creation` + +The `alias_creation` option controls who is allowed to create aliases +on this server. + +The format of this option is a list of rules that contain globs that +match against user_id, room_id and the new alias (fully qualified with +server name). The action in the first rule that matches is taken, +which can currently either be "allow" or "deny". + +Missing user_id/room_id/alias fields default to "*". + +If no rules match the request is denied. An empty list means no one +can create aliases. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against the alias being created. Defaults to "*". +* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*" +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +alias_creation_rules: + - user_id: "bad_user" + alias: "spammy_alias" + room_id: "*" + action: deny +``` +--- +Config options: `room_list_publication_rules` + +The `room_list_publication_rules` option controls who can publish and +which rooms can be published in the public room list. + +The format of this option is the same as that for +`alias_creation_rules`. + +If the room has one or more aliases associated with it, only one of +the aliases needs to match the alias rule. If there are no aliases +then only rules with `alias: *` match. + +If no rules match the request is denied. An empty list means no one +can publish rooms. + +Options for the rules include: +* `user_id`: Matches against the creator of the alias. Defaults to "*". +* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*". +* `room_id`: Matches against the room ID being published. Defaults to "*". +* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. + +Example configuration: +```yaml +room_list_publication_rules: + - user_id: "*" + alias: "*" + room_id: "*" + action: allow +``` +--- +## Opentracing ## +Configuration options related to Opentracing support. + +--- +Config option: `opentracing` + +These settings enable and configure opentracing, which implements distributed tracing. +This allows you to observe the causal chains of events across servers +including requests, key lookups etc., across any server running +synapse or any other services which support opentracing +(specifically those implemented with Jaeger). + +Sub-options include: +* `enabled`: whether tracing is enabled. Set to true to enable. Disabled by default. +* `homeserver_whitelist`: The list of homeservers we wish to send and receive span contexts and span baggage. + See [here](../../opentracing.md) for more. + This is a list of regexes which are matched against the `server_name` of the homeserver. + By default, it is empty, so no servers are matched. +* `force_tracing_for_users`: # A list of the matrix IDs of users whose requests will always be traced, + even if the tracing system would otherwise drop the traces due to probabilistic sampling. + By default, the list is empty. +* `jaeger_config`: Jaeger can be configured to sample traces at different rates. + All configuration options provided by Jaeger can be set here. Jaeger's configuration is + mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). + +Example configuration: +```yaml +opentracing: + enabled: true + homeserver_whitelist: + - ".*" + force_tracing_for_users: + - "@user1:server_name" + - "@user2:server_name" + + jaeger_config: + sampler: + type: const + param: 1 + logging: + false +``` +--- +## Workers ## +Configuration options related to workers. + +--- +Config option: `send_federation` + +Controls sending of outbound federation transactions on the main process. +Set to false if using a federation sender worker. Defaults to true. + +Example configuration: +```yaml +send_federation: false +``` +--- +Config option: `federation_sender_instances` + +It is possible to run multiple federation sender workers, in which case the +work is balanced across them. Use this setting to list the senders. + +This configuration setting must be shared between all federation sender workers, and if +changed all federation sender workers must be stopped at the same time and then +started, to ensure that all instances are running with the same config (otherwise +events may be dropped). + +Example configuration: +```yaml +federation_sender_instances: + - federation_sender1 +``` +--- +Config option: `instance_map` + +When using workers this should be a map from worker name to the +HTTP replication listener of the worker, if configured. + +Example configuration: +```yaml +instance_map: + worker1: + host: localhost + port: 8034 +``` +--- +Config option: `stream_writers` + +Experimental: When using workers you can define which workers should +handle event persistence and typing notifications. Any worker +specified here must also be in the `instance_map`. + +Example configuration: +```yaml +stream_writers: + events: worker1 + typing: worker1 +``` +--- +Config option: `run_background_task_on` + +The worker that is used to run background tasks (e.g. cleaning up expired +data). If not provided this defaults to the main process. + +Example configuration: +```yaml +run_background_tasks_on: worker1 +``` +--- +Config option: `worker_replication_secret` + +A shared secret used by the replication APIs to authenticate HTTP requests +from workers. + +By default this is unused and traffic is not authenticated. + +Example configuration: +```yaml +worker_replication_secret: "secret_secret" +``` +Config option: `redis` + +Configuration for Redis when using workers. This *must* be enabled when +using workers (unless using old style direct TCP configuration). +This setting has the following sub-options: +* `enabled`: whether to use Redis support. Defaults to false. +* `host` and `port`: Optional host and port to use to connect to redis. Defaults to + localhost and 6379 +* `password`: Optional password if configured on the Redis instance. + +Example configuration: +```yaml +redis: + enabled: true + host: localhost + port: 6379 + password: +``` +## Background Updates ## +Configuration settings related to background updates. + +--- +Config option: `background_updates` + +Background updates are database updates that are run in the background in batches. +The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to +sleep can all be configured. This is helpful to speed up or slow down the updates. +This setting has the following sub-options: +* `background_update_duration_ms`: How long in milliseconds to run a batch of background updates for. Defaults to 100. + Set a different time to change the default. +* `sleep_enabled`: Whether to sleep between updates. Defaults to true. Set to false to change the default. +* `sleep_duration_ms`: If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. + Set a duration to change the default. +* `min_batch_size`: Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. + Set a size to change the default. +* `default_batch_size`: The batch size to use for the first iteration of a new background update. The default is 100. + Set a size to change the default. + +Example configuration: +```yaml +background_updates: + background_update_duration_ms: 500 + sleep_enabled: false + sleep_duration_ms: 300 + min_batch_size: 10 + default_batch_size: 50 +``` \ No newline at end of file From f8d3ee95706100e982f30f4c04fc3df8de4d4212 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 18 Apr 2022 12:41:55 -0600 Subject: [PATCH 025/263] Fix grammatical error in error message (#12483) * Fix grammatical error in error message * changelog --- changelog.d/12483.misc | 1 + synapse/storage/databases/main/state.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12483.misc diff --git a/changelog.d/12483.misc b/changelog.d/12483.misc new file mode 100644 index 000000000000..88c6e3e465c8 --- /dev/null +++ b/changelog.d/12483.misc @@ -0,0 +1 @@ +Fix grammatical error in federation error response when the room version of a room is unknown. diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index eba35f37000d..7a1b013fa3ac 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -130,7 +130,7 @@ def get_room_version_id_txn(self, txn: LoggingTransaction, room_id: str) -> str: ) if room_version is None: - raise NotFoundError("Could not room_version for %s" % (room_id,)) + raise NotFoundError("Could not find room_version for %s" % (room_id,)) return room_version From b121a3ad2b75f94002b085e1945110fc446a560a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 12:17:29 +0100 Subject: [PATCH 026/263] Back out implementation of MSC2314 (#12474) MSC2314 has now been closed, so we're backing out its implementation, which originally happened in #6176. Unfortunately it's not a direct revert, as that PR mixed in a bunch of unrelated changes to tests etc. --- changelog.d/12474.misc | 1 + synapse/federation/federation_server.py | 26 ++++-------- .../federation/transport/server/federation.py | 2 +- sytest-blacklist | 4 -- tests/federation/test_federation_server.py | 41 +------------------ 5 files changed, 13 insertions(+), 61 deletions(-) create mode 100644 changelog.d/12474.misc diff --git a/changelog.d/12474.misc b/changelog.d/12474.misc new file mode 100644 index 000000000000..5292108b3920 --- /dev/null +++ b/changelog.d/12474.misc @@ -0,0 +1 @@ +Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index e67af6463fc7..beab1227b84c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -515,7 +515,7 @@ async def _process_edu(edu_dict: JsonDict) -> None: ) async def on_room_state_request( - self, origin: str, room_id: str, event_id: Optional[str] + self, origin: str, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -530,18 +530,13 @@ async def on_room_state_request( # - but that's non-trivial to get right, and anyway somewhat defeats # the point of the linearizer. async with self._server_linearizer.queue((origin, room_id)): - resp: JsonDict = dict( - await self._state_resp_cache.wrap( - (room_id, event_id), - self._on_context_state_request_compute, - room_id, - event_id, - ) + resp = await self._state_resp_cache.wrap( + (room_id, event_id), + self._on_context_state_request_compute, + room_id, + event_id, ) - room_version = await self.store.get_room_version_id(room_id) - resp["room_version"] = room_version - return 200, resp async def on_state_ids_request( @@ -574,14 +569,11 @@ async def _on_state_ids_request_compute( return {"pdu_ids": state_ids, "auth_chain_ids": list(auth_chain_ids)} async def _on_context_state_request_compute( - self, room_id: str, event_id: Optional[str] + self, room_id: str, event_id: str ) -> Dict[str, list]: pdus: Collection[EventBase] - if event_id: - event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) - pdus = await self.store.get_events_as_list(event_ids) - else: - pdus = (await self.state.get_current_state(room_id)).values() + event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) + pdus = await self.store.get_events_as_list(event_ids) auth_chain = await self.store.get_auth_chain( room_id, [pdu.event_id for pdu in pdus] diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index aed3d5069ca1..6fbc7b5f15a7 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -160,7 +160,7 @@ async def on_GET( return await self.handler.on_room_state_request( origin, room_id, - parse_string_from_args(query, "event_id", None, required=False), + parse_string_from_args(query, "event_id", None, required=True), ) diff --git a/sytest-blacklist b/sytest-blacklist index 57e603a4a605..d5fa36cec7ae 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -21,10 +21,6 @@ Newly created users see their own presence in /initialSync (SYT-34) # Blacklisted due to https://github.com/matrix-org/synapse/issues/1396 Should reject keys claiming to belong to a different user -# Blacklisted due to https://github.com/matrix-org/matrix-doc/pull/2314 removing -# this requirement from the spec -Inbound federation of state requires event_id as a mandatory paramater - # Blacklisted until MSC2753 is implemented Local users can peek into world_readable rooms by room ID We can't peek into rooms with shared history_visibility diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 30e7e5093a17..b19365b81a4c 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -104,58 +104,21 @@ def test_wildcard_matching(self): class StateQueryTests(unittest.FederatingHomeserverTestCase): - servlets = [ admin.register_servlets, room.register_servlets, login.register_servlets, ] - def test_without_event_id(self): - """ - Querying v1/state/ without an event ID will return the current - known state. - """ - u1 = self.register_user("u1", "pass") - u1_token = self.login("u1", "pass") - - room_1 = self.helper.create_room_as(u1, tok=u1_token) - self.inject_room_member(room_1, "@user:other.example.com", "join") - - channel = self.make_signed_federation_request( - "GET", "/_matrix/federation/v1/state/%s" % (room_1,) - ) - self.assertEqual(200, channel.code, channel.result) - - self.assertEqual( - channel.json_body["room_version"], - self.hs.config.server.default_room_version.identifier, - ) - - members = set( - map( - lambda x: x["state_key"], - filter( - lambda x: x["type"] == "m.room.member", channel.json_body["pdus"] - ), - ) - ) - - self.assertEqual(members, {"@user:other.example.com", u1}) - self.assertEqual(len(channel.json_body["pdus"]), 6) - def test_needs_to_be_in_room(self): - """ - Querying v1/state/ requires the server - be in the room to provide data. - """ + """/v1/state/ requires the server to be in the room""" u1 = self.register_user("u1", "pass") u1_token = self.login("u1", "pass") room_1 = self.helper.create_room_as(u1, tok=u1_token) channel = self.make_signed_federation_request( - "GET", "/_matrix/federation/v1/state/%s" % (room_1,) + "GET", "/_matrix/federation/v1/state/%s?event_id=xyz" % (room_1,) ) self.assertEqual(403, channel.code, channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") From dbe016e2585e2761e3c5fa63c4857a5ef8ef580f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Tue, 19 Apr 2022 13:33:36 +0100 Subject: [PATCH 027/263] Remove 'Non-maintainer upload' line from Debian changelog --- debian/changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index a44f7d98ff6b..b45f6a77a9ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,5 @@ matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium - * Non-maintainer upload. * Use poetry to manage the bundled virtualenv included with this package. -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 From 7dec4ce7e4bc9c330e5ec4c31be475156c4539e1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 19 Apr 2022 13:45:27 +0100 Subject: [PATCH 028/263] fix typo in debian changelog --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index b45f6a77a9ed..a46bb2bba3e6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. - -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + -- Synapse Packaging team Wed, 30 Mar 2022 12:21:43 +0100 matrix-synapse-py3 (1.57.0) stable; urgency=medium From fbdee86004a9b2e7f52be584b7b62efa434fc8be Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 19 Apr 2022 15:00:41 +0200 Subject: [PATCH 029/263] Fix a link in `README.rst` (#12495) * Fix a link in `README.rst` * newsfile --- README.rst | 2 +- changelog.d/12495.doc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12495.doc diff --git a/README.rst b/README.rst index 595fb5ff62a6..8a14401d65af 100644 --- a/README.rst +++ b/README.rst @@ -312,7 +312,7 @@ We recommend using the demo which starts 3 federated instances running on ports (to stop, you can use `./demo/stop.sh`) -See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html) +See the `demo documentation `_ for more information. If you just want to start a single instance of the app and run it directly:: diff --git a/changelog.d/12495.doc b/changelog.d/12495.doc new file mode 100644 index 000000000000..afa011167513 --- /dev/null +++ b/changelog.d/12495.doc @@ -0,0 +1 @@ +Fix a broken link in `README.rst`. From a1f87f57ff7b5971e0e3450ec7761cf8dc4e01d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?= Date: Tue, 19 Apr 2022 17:23:53 +0200 Subject: [PATCH 030/263] Implement MSC3383: include destination in X-Matrix auth header (#11398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan Christian Grünhage Co-authored-by: Marcus Hoffmann --- changelog.d/11398.feature | 1 + scripts-dev/federation_client.py | 7 +++- synapse/federation/transport/server/_base.py | 39 ++++++++++++++++---- synapse/http/matrixfederationclient.py | 12 +++++- 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 changelog.d/11398.feature diff --git a/changelog.d/11398.feature b/changelog.d/11398.feature new file mode 100644 index 000000000000..a910f4da1496 --- /dev/null +++ b/changelog.d/11398.feature @@ -0,0 +1 @@ +Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index c72e19f61d62..079d2f5ed061 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -124,7 +124,12 @@ def request( authorization_headers = [] for key, sig in signed_json["signatures"][origin_name].items(): - header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig) + header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % ( + origin_name, + key, + sig, + destination, + ) authorization_headers.append(header.encode("ascii")) print("Authorization: %s" % header, file=sys.stderr) diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 2529dee613aa..d629a3ecb5dd 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -16,7 +16,8 @@ import logging import re import time -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast from synapse.api.errors import Codes, FederationDeniedError, SynapseError from synapse.api.urls import FEDERATION_V1_PREFIX @@ -86,15 +87,24 @@ async def authenticate_request( if not auth_headers: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) for auth in auth_headers: if auth.startswith(b"X-Matrix"): - (origin, key, sig) = _parse_auth_header(auth) + (origin, key, sig, destination) = _parse_auth_header(auth) json_request["origin"] = origin json_request["signatures"].setdefault(origin, {})[key] = sig + # if the origin_server sent a destination along it needs to match our own server_name + if destination is not None and destination != self.server_name: + raise AuthenticationError( + HTTPStatus.UNAUTHORIZED, + "Destination mismatch in auth header", + Codes.UNAUTHORIZED, + ) if ( self.federation_domain_whitelist is not None and origin not in self.federation_domain_whitelist @@ -103,7 +113,9 @@ async def authenticate_request( if origin is None or not json_request["signatures"]: raise NoAuthenticationError( - 401, "Missing Authorization headers", Codes.UNAUTHORIZED + HTTPStatus.UNAUTHORIZED, + "Missing Authorization headers", + Codes.UNAUTHORIZED, ) await self.keyring.verify_json_for_server( @@ -142,13 +154,14 @@ async def reset_retry_timings(self, origin: str) -> None: logger.exception("Error resetting retry timings on %s", origin) -def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: +def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str]]: """Parse an X-Matrix auth header Args: header_bytes: header value Returns: + origin, key id, signature, destination. origin, key id, signature. Raises: @@ -157,7 +170,9 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]: try: header_str = header_bytes.decode("utf-8") params = header_str.split(" ")[1].split(",") - param_dict = {k: v for k, v in (kv.split("=", maxsplit=1) for kv in params)} + param_dict: Dict[str, str] = { + k: v for k, v in [param.split("=", maxsplit=1) for param in params] + } def strip_quotes(value: str) -> str: if value.startswith('"'): @@ -172,7 +187,15 @@ def strip_quotes(value: str) -> str: key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) - return origin, key, sig + + # get the destination server_name from the auth header if it exists + destination = param_dict.get("destination") + if destination is not None: + destination = strip_quotes(destination) + else: + destination = None + + return origin, key, sig, destination except Exception as e: logger.warning( "Error parsing auth header '%s': %s", @@ -180,7 +203,7 @@ def strip_quotes(value: str) -> str: e, ) raise AuthenticationError( - 400, "Malformed Authorization header", Codes.UNAUTHORIZED + HTTPStatus.BAD_REQUEST, "Malformed Authorization header", Codes.UNAUTHORIZED ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5097b3ca5796..e68644595546 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -704,6 +704,9 @@ def build_auth_headers( Returns: A list of headers to be added as "Authorization:" headers """ + if destination is None and destination_is is None: + raise ValueError("destination and destination_is cannot both be None!") + request: JsonDict = { "method": method.decode("ascii"), "uri": url_bytes.decode("ascii"), @@ -726,8 +729,13 @@ def build_auth_headers( for key, sig in request["signatures"][self.server_name].items(): auth_headers.append( ( - 'X-Matrix origin=%s,key="%s",sig="%s"' - % (self.server_name, key, sig) + 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' + % ( + self.server_name, + key, + sig, + request.get("destination") or request["destination_is"], + ) ).encode("ascii") ) return auth_headers From 798deb3a10e59cfa7b46994417f0c56c4a0bc9f7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:41:52 +0100 Subject: [PATCH 031/263] Fix typo in deb changelogs from release script (#12497) The release script used to incorrectly write `New synapse release 1.57.0~rc1.` instead of `New synapse release 1.57.0rc1.` --- changelog.d/12497.misc | 1 + scripts-dev/release.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12497.misc diff --git a/changelog.d/12497.misc b/changelog.d/12497.misc new file mode 100644 index 000000000000..17a661ec61b5 --- /dev/null +++ b/changelog.d/12497.misc @@ -0,0 +1 @@ +Fix a minor typo in the Debian changelogs generated by the release script. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 518eaf417cfe..6f7cf6888d36 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -230,7 +230,7 @@ def prepare(): debian_version = new_version run_until_successful( - f'dch -M -v {debian_version} "New synapse release {debian_version}."', + f'dch -M -v {debian_version} "New Synapse release {new_version}."', shell=True, ) run_until_successful('dch -M -r -D stable ""', shell=True) From b80bb7e4526bd0425b3cdd10fa5633a5a4b4e05f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 19 Apr 2022 16:42:19 +0100 Subject: [PATCH 032/263] Fix `/room/.../event/...` to return the *original* event after any edits (#12476) This is what the MSC (now) requires. Fixes https://github.com/matrix-org/synapse/issues/10310. --- changelog.d/12476.bugfix | 1 + synapse/events/utils.py | 21 ++++--- synapse/rest/client/room.py | 4 +- tests/rest/client/test_relations.py | 92 +++++++++++++++++++---------- 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 changelog.d/12476.bugfix diff --git a/changelog.d/12476.bugfix b/changelog.d/12476.bugfix new file mode 100644 index 000000000000..9ad6a71abd52 --- /dev/null +++ b/changelog.d/12476.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 43c3241fb0d1..2174b4a0948b 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -402,6 +402,7 @@ def serialize_event( *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None, + apply_edits: bool = True, ) -> JsonDict: """Serializes a single event. @@ -409,10 +410,10 @@ def serialize_event( event: The event being serialized. time_now: The current time in milliseconds config: Event serialization config - bundle_aggregations: Whether to include the bundled aggregations for this - event. Only applies to non-state events. (State events never include - bundled aggregations.) - + bundle_aggregations: A map from event_id to the aggregations to be bundled + into the event. + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `bundle_aggregations[].replace`. Returns: The serialized event """ @@ -430,8 +431,9 @@ def serialize_event( event, time_now, config, - bundle_aggregations[event.event_id], + event_aggregations, serialized_event, + apply_edits=apply_edits, ) return serialized_event @@ -470,6 +472,7 @@ def _inject_bundled_aggregations( config: SerializeEventConfig, aggregations: "BundledAggregations", serialized_event: JsonDict, + apply_edits: bool, ) -> None: """Potentially injects bundled aggregations into the unsigned portion of the serialized event. @@ -479,7 +482,8 @@ def _inject_bundled_aggregations( aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. config: Event serialization config - + apply_edits: Whether the content of the event should be modified to reflect + any replacement in `aggregations.replace`. """ serialized_aggregations = {} @@ -490,9 +494,10 @@ def _inject_bundled_aggregations( serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references if aggregations.replace: - # If there is an edit, apply it to the event. + # If there is an edit, optionally apply it to the event. edit = aggregations.replace - self._apply_edit(event, serialized_event, edit) + if apply_edits: + self._apply_edit(event, serialized_event, edit) # Include information about it in the relations dict. serialized_aggregations[RelationTypes.REPLACE] = { diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 47e152c8cc7a..937c32317669 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -669,8 +669,10 @@ async def on_GET( ) time_now = self.clock.time_msec() + # per MSC2676, /rooms/{roomId}/event/{eventId}, should return the + # *original* event, rather than the edited version event_dict = self._event_serializer.serialize_event( - event, time_now, bundle_aggregations=aggregations + event, time_now, bundle_aggregations=aggregations, apply_edits=False ) return 200, event_dict diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 6fabada8b3ae..57d97bbb47ae 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -380,13 +380,16 @@ def assert_bundle(event_json: JsonDict) -> None: {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) + # /event should return the *original* event channel = self.make_request( "GET", f"/rooms/{self.room}/event/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual( + channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"} + ) assert_bundle(channel.json_body) # Request the room messages. @@ -399,6 +402,7 @@ def assert_bundle(event_json: JsonDict) -> None: assert_bundle(self._find_event_in_chunk(channel.json_body["chunk"])) # Request the room context. + # /context should return the edited event. channel = self.make_request( "GET", f"/rooms/{self.room}/context/{self.parent_id}", @@ -406,6 +410,7 @@ def assert_bundle(event_json: JsonDict) -> None: ) self.assertEqual(200, channel.code, channel.json_body) assert_bundle(channel.json_body["event"]) + self.assertEqual(channel.json_body["event"]["content"], new_body) # Request sync, but limit the timeline so it becomes limited (and includes # bundled aggregations). @@ -470,14 +475,14 @@ def test_multi_edit(self) -> None: channel = self.make_request( "GET", - f"/rooms/{self.room}/event/{self.parent_id}", + f"/rooms/{self.room}/context/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual(channel.json_body["event"]["content"], new_body) - relations_dict = channel.json_body["unsigned"].get("m.relations") + relations_dict = channel.json_body["event"]["unsigned"].get("m.relations") self.assertIn(RelationTypes.REPLACE, relations_dict) m_replace_dict = relations_dict[RelationTypes.REPLACE] @@ -492,10 +497,9 @@ def test_edit_reply(self) -> None: """Test that editing a reply works.""" # Create a reply to edit. + original_body = {"msgtype": "m.text", "body": "A reply!"} channel = self._send_relation( - RelationTypes.REFERENCE, - "m.room.message", - content={"msgtype": "m.text", "body": "A reply!"}, + RelationTypes.REFERENCE, "m.room.message", content=original_body ) reply = channel.json_body["event_id"] @@ -508,38 +512,54 @@ def test_edit_reply(self) -> None: ) edit_event_id = channel.json_body["event_id"] + # /event returns the original event channel = self.make_request( "GET", f"/rooms/{self.room}/event/{reply}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) + event_result = channel.json_body + self.assertDictContainsSubset(original_body, event_result["content"]) - # We expect to see the new body in the dict, as well as the reference - # metadata sill intact. - self.assertDictContainsSubset(new_body, channel.json_body["content"]) - self.assertDictContainsSubset( - { - "m.relates_to": { - "event_id": self.parent_id, - "rel_type": "m.reference", - } - }, - channel.json_body["content"], + # also check /context, which returns the *edited* event + channel = self.make_request( + "GET", + f"/rooms/{self.room}/context/{reply}", + access_token=self.user_token, ) + self.assertEqual(200, channel.code, channel.json_body) + context_result = channel.json_body["event"] - # We expect that the edit relation appears in the unsigned relations - # section. - relations_dict = channel.json_body["unsigned"].get("m.relations") - self.assertIn(RelationTypes.REPLACE, relations_dict) + # check that the relations are correct for both APIs + for result_event_dict, desc in ( + (event_result, "/event"), + (context_result, "/context"), + ): + # The reference metadata should still be intact. + self.assertDictContainsSubset( + { + "m.relates_to": { + "event_id": self.parent_id, + "rel_type": "m.reference", + } + }, + result_event_dict["content"], + desc, + ) - m_replace_dict = relations_dict[RelationTypes.REPLACE] - for key in ["event_id", "sender", "origin_server_ts"]: - self.assertIn(key, m_replace_dict) + # We expect that the edit relation appears in the unsigned relations + # section. + relations_dict = result_event_dict["unsigned"].get("m.relations") + self.assertIn(RelationTypes.REPLACE, relations_dict, desc) - self.assert_dict( - {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict - ) + m_replace_dict = relations_dict[RelationTypes.REPLACE] + for key in ["event_id", "sender", "origin_server_ts"]: + self.assertIn(key, m_replace_dict, desc) + + self.assert_dict( + {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict + ) def test_edit_thread(self) -> None: """Test that editing a thread works.""" @@ -605,19 +625,31 @@ def test_edit_edit(self) -> None: ) # Request the original event. + # /event should return the original event. channel = self.make_request( "GET", f"/rooms/{self.room}/event/{self.parent_id}", access_token=self.user_token, ) self.assertEqual(200, channel.code, channel.json_body) - # The edit to the edit should be ignored. - self.assertEqual(channel.json_body["content"], new_body) + self.assertEqual( + channel.json_body["content"], {"body": "Hi!", "msgtype": "m.text"} + ) # The relations information should not include the edit to the edit. relations_dict = channel.json_body["unsigned"].get("m.relations") self.assertIn(RelationTypes.REPLACE, relations_dict) + # /context should return the event updated for the *first* edit + # (The edit to the edit should be ignored.) + channel = self.make_request( + "GET", + f"/rooms/{self.room}/context/{self.parent_id}", + access_token=self.user_token, + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assertEqual(channel.json_body["event"]["content"], new_body) + m_replace_dict = relations_dict[RelationTypes.REPLACE] for key in ["event_id", "sender", "origin_server_ts"]: self.assertIn(key, m_replace_dict) From c1482a352acd35055db93f5d0b5de1bec8974fa0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Apr 2022 16:49:45 +0100 Subject: [PATCH 033/263] Fix returned count of delete extremities admin API (#12496) --- changelog.d/12496.bugfix | 1 + .../storage/databases/main/events_forward_extremities.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12496.bugfix diff --git a/changelog.d/12496.bugfix b/changelog.d/12496.bugfix new file mode 100644 index 000000000000..a68df7c96a5f --- /dev/null +++ b/changelog.d/12496.bugfix @@ -0,0 +1 @@ +Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. diff --git a/synapse/storage/databases/main/events_forward_extremities.py b/synapse/storage/databases/main/events_forward_extremities.py index 68901b43352b..f851bff60451 100644 --- a/synapse/storage/databases/main/events_forward_extremities.py +++ b/synapse/storage/databases/main/events_forward_extremities.py @@ -66,13 +66,15 @@ def delete_forward_extremities_for_room_txn(txn: LoggingTransaction) -> int: """ txn.execute(sql, (event_id, room_id)) + + deleted_count = txn.rowcount logger.info( "Deleted %s extra forward extremities for room %s", - txn.rowcount, + deleted_count, room_id, ) - if txn.rowcount > 0: + if deleted_count > 0: # Invalidate the cache self._invalidate_cache_and_stream( txn, @@ -80,7 +82,7 @@ def delete_forward_extremities_for_room_txn(txn: LoggingTransaction) -> int: (room_id,), ) - return txn.rowcount + return deleted_count return await self.db_pool.runInteraction( "delete_forward_extremities_for_room", From eed38c502760989026fc826ce49205c02636cebd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 12:48:44 +0100 Subject: [PATCH 034/263] Add CI job to act as a canary for testing against latest dependencies (#12472) Co-authored-by: Patrick Cloke --- ...latest_deps_build_failed_issue_template.md | 4 + .github/workflows/latest_deps.yml | 156 ++++++++++++++++++ changelog.d/12472.misc | 1 + 3 files changed, 161 insertions(+) create mode 100644 .ci/latest_deps_build_failed_issue_template.md create mode 100644 .github/workflows/latest_deps.yml create mode 100644 changelog.d/12472.misc diff --git a/.ci/latest_deps_build_failed_issue_template.md b/.ci/latest_deps_build_failed_issue_template.md new file mode 100644 index 000000000000..0525402503fd --- /dev/null +++ b/.ci/latest_deps_build_failed_issue_template.md @@ -0,0 +1,4 @@ +--- +title: CI run against latest deps is failing +--- +See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}} diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml new file mode 100644 index 000000000000..1a61d179d905 --- /dev/null +++ b/.github/workflows/latest_deps.yml @@ -0,0 +1,156 @@ +# People who are freshly `pip install`ing from PyPI will pull in the latest versions of +# dependencies which match the broad requirements. Since most CI runs are against +# the locked poetry environment, run specifically against the latest dependencies to +# know if there's an upcoming breaking change. +# +# As an overview this workflow: +# - checks out develop, +# - installs from source, pulling in the dependencies like a fresh `pip install` would, and +# - runs mypy and test suites in that checkout. +# +# Based on the twisted trunk CI job. + +name: Latest dependencies + +on: + schedule: + - cron: 0 7 * * * + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # The dev dependencies aren't exposed in the wheel metadata (at least with current + # poetry-core versions), so we install with poetry. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: "3.x" + poetry-version: "1.2.0b1" + # Dump installed versions for debugging. + - run: poetry run pip list > before.txt + # Upgrade all runtime dependencies only. This is intended to mimic a fresh + # `pip install matrix-synapse[all]` as closely as possible. + - run: poetry update --no-dev + - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) + - run: poetry run mypy + trial: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - database: "sqlite" + - database: "postgres" + postgres-version: "14" + + steps: + - uses: actions/checkout@v2 + - run: sudo apt-get -qq install xmlsec1 + - name: Set up PostgreSQL ${{ matrix.postgres-version }} + if: ${{ matrix.postgres-version }} + run: | + docker run -d -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + postgres:${{ matrix.postgres-version }} + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + - run: pip install .[all,test] + - name: Await PostgreSQL + if: ${{ matrix.postgres-version }} + timeout-minutes: 2 + run: until pg_isready -h localhost; do sleep 1; done + - run: python -m twisted.trial --jobs=2 tests + env: + SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} + SYNAPSE_POSTGRES_HOST: localhost + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + - name: Dump logs + # Logs are most useful when the command fails, always include them. + if: ${{ always() }} + # Note: Dumps to workflow logs instead of using actions/upload-artifact + # This keeps logs colocated with failing jobs + # It also ignores find's exit code; this is a best effort affair + run: >- + find _trial_temp -name '*.log' + -exec echo "::group::{}" \; + -exec cat {} \; + -exec echo "::endgroup::" \; + || true + + + sytest: + runs-on: ubuntu-latest + container: + image: matrixdotorg/sytest-synapse:testing + volumes: + - ${{ github.workspace }}:/src + strategy: + fail-fast: false + matrix: + include: + - sytest-tag: focal + + - sytest-tag: focal + postgres: postgres + workers: workers + redis: redis + env: + POSTGRES: ${{ matrix.postgres && 1}} + WORKERS: ${{ matrix.workers && 1 }} + REDIS: ${{ matrix.redis && 1 }} + BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} + + steps: + - uses: actions/checkout@v2 + - name: Ensure sytest runs `pip install` + # Delete the lockfile so sytest will `pip install` rather than `poetry install` + run: rm /src/poetry.lock + working-directory: /src + - name: Prepare test blacklist + run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Run SyTest + run: /bootstrap.sh synapse + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Upload SyTest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* + + + # TODO: run complement (as with twisted trunk, see #12473). + + # open an issue if the build fails, so we know about it. + open-issue: + if: failure() + needs: + # TODO: should mypy be included here? It feels more brittle than the other two. + - mypy + - trial + - sytest + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + update_existing: true + filename: .ci/latest_deps_build_failed_issue_template.md + diff --git a/changelog.d/12472.misc b/changelog.d/12472.misc new file mode 100644 index 000000000000..ed306209cc42 --- /dev/null +++ b/changelog.d/12472.misc @@ -0,0 +1 @@ +Add a CI job which tests Synapse against the latest version of all dependencies. From 4bc8cb4669ddeb719a3a6de39b093fc3be8db6fe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 20 Apr 2022 14:57:39 +0300 Subject: [PATCH 035/263] Implement MSC2815: allow room moderators to view redacted event content (#12427) Implements matrix-org/matrix-spec-proposals#2815 Signed-off-by: Tulir Asokan --- changelog.d/12427.feature | 1 + synapse/api/errors.py | 18 ++++++++ synapse/config/experimental.py | 3 ++ synapse/handlers/events.py | 15 +++++- synapse/rest/client/room.py | 46 ++++++++++++++++++- synapse/rest/client/versions.py | 2 + .../storage/databases/main/events_worker.py | 18 ++++++++ 7 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12427.feature diff --git a/changelog.d/12427.feature b/changelog.d/12427.feature new file mode 100644 index 000000000000..e6913c8c0973 --- /dev/null +++ b/changelog.d/12427.feature @@ -0,0 +1 @@ +Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index e92db29f6dc6..cb3b7323d568 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -79,6 +79,8 @@ class Codes: UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN" UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN" + UNREDACTED_CONTENT_DELETED = "FI.MAU.MSC2815_UNREDACTED_CONTENT_DELETED" + class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes. @@ -483,6 +485,22 @@ def __init__(self, inner_exception: BaseException, can_retry: bool): self.can_retry = can_retry +class UnredactedContentDeletedError(SynapseError): + def __init__(self, content_keep_ms: Optional[int] = None): + super().__init__( + 404, + "The content for that event has already been erased from the database", + errcode=Codes.UNREDACTED_CONTENT_DELETED, + ) + self.content_keep_ms = content_keep_ms + + def error_dict(self) -> "JsonDict": + extra = {} + if self.content_keep_ms is not None: + extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms} + return cs_error(self.msg, self.errcode, **extra) + + def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs: Any) -> "JsonDict": """Utility method for constructing an error response for client-server interactions. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 979059e72386..421ed7481baf 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -78,3 +78,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC2654: Unread counts self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) + + # MSC2815 (allow room moderators to view redacted event content) + self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index e89c4df31466..5b94b00bc3e7 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -21,6 +21,7 @@ from synapse.events import EventBase from synapse.events.utils import SerializeEventConfig from synapse.handlers.presence import format_user_presence_state +from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, UserID from synapse.visibility import filter_events_for_client @@ -141,7 +142,11 @@ def __init__(self, hs: "HomeServer"): self.storage = hs.get_storage() async def get_event( - self, user: UserID, room_id: Optional[str], event_id: str + self, + user: UserID, + room_id: Optional[str], + event_id: str, + show_redacted: bool = False, ) -> Optional[EventBase]: """Retrieve a single specified event. @@ -150,6 +155,7 @@ async def get_event( room_id: The expected room id. We'll return None if the event's room does not match. event_id: The event ID to obtain. + show_redacted: Should the full content of redacted events be returned? Returns: An event, or None if there is no event matching this ID. Raises: @@ -157,7 +163,12 @@ async def get_event( AuthError if the user does not have the rights to inspect this event. """ - event = await self.store.get_event(event_id, check_room_id=room_id) + redact_behaviour = ( + EventRedactBehaviour.AS_IS if show_redacted else EventRedactBehaviour.REDACT + ) + event = await self.store.get_event( + event_id, check_room_id=room_id, redact_behaviour=redact_behaviour + ) if not event: return None diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 937c32317669..906fe09e9713 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -21,6 +21,7 @@ from twisted.web.server import Request +from synapse import event_auth from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, @@ -29,6 +30,7 @@ MissingClientTokenError, ShadowBanError, SynapseError, + UnredactedContentDeletedError, ) from synapse.api.filtering import Filter from synapse.events.utils import format_event_for_client_v2 @@ -643,18 +645,55 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.clock = hs.get_clock() self._store = hs.get_datastores().main + self._state = hs.get_state_handler() self.event_handler = hs.get_event_handler() self._event_serializer = hs.get_event_client_serializer() self._relations_handler = hs.get_relations_handler() self.auth = hs.get_auth() + self.content_keep_ms = hs.config.server.redaction_retention_period + self.msc2815_enabled = hs.config.experimental.msc2815_enabled async def on_GET( self, request: SynapseRequest, room_id: str, event_id: str ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) + + include_unredacted_content = self.msc2815_enabled and ( + parse_string( + request, + "fi.mau.msc2815.include_unredacted_content", + allowed_values=("true", "false"), + ) + == "true" + ) + if include_unredacted_content and not await self.auth.is_server_admin( + requester.user + ): + power_level_event = await self._state.get_current_state( + room_id, EventTypes.PowerLevels, "" + ) + + auth_events = {} + if power_level_event: + auth_events[(EventTypes.PowerLevels, "")] = power_level_event + + redact_level = event_auth.get_named_level(auth_events, "redact", 50) + user_level = event_auth.get_user_power_level( + requester.user.to_string(), auth_events + ) + if user_level < redact_level: + raise SynapseError( + 403, + "You don't have permission to view redacted events in this room.", + errcode=Codes.FORBIDDEN, + ) + try: event = await self.event_handler.get_event( - requester.user, room_id, event_id + requester.user, + room_id, + event_id, + show_redacted=include_unredacted_content, ) except AuthError: # This endpoint is supposed to return a 404 when the requester does @@ -663,6 +702,11 @@ async def on_GET( raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if event: + if include_unredacted_content and await self._store.have_censored_event( + event_id + ): + raise UnredactedContentDeletedError(self.content_keep_ms) + # Ensure there are bundled aggregations available. aggregations = await self._relations_handler.get_bundled_aggregations( [event], requester.user.to_string() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7b29026381ed..bfc1d4ee0829 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -101,6 +101,8 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above + # Allows moderators to fetch redacted event content as described in MSC2815 + "fi.mau.msc2815": self.config.experimental.msc2815_enabled, }, }, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 5288cdba035d..60876204bd0f 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -303,6 +303,24 @@ async def get_received_ts(self, event_id: str) -> Optional[int]: desc="get_received_ts", ) + async def have_censored_event(self, event_id: str) -> bool: + """Check if an event has been censored, i.e. if the content of the event has been erased + from the database due to a redaction. + + Args: + event_id: The event ID that was redacted. + + Returns: + True if the event has been censored, False otherwise. + """ + censored_redactions_list = await self.db_pool.simple_select_onecol( + table="redactions", + keyvalues={"redacts": event_id}, + retcol="have_censored", + desc="get_have_censored", + ) + return any(censored_redactions_list) + # Inform mypy that if allow_none is False (the default) then get_event # always returns an EventBase. @overload From d0c1f4ca4c66d4b067ff5fd1be43285f54773d4f Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 20 Apr 2022 09:56:59 -0400 Subject: [PATCH 036/263] Remove unnecessary config overrides for MSC3666. (#12511) --- changelog.d/12511.misc | 1 + tests/rest/client/test_relations.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/12511.misc diff --git a/changelog.d/12511.misc b/changelog.d/12511.misc new file mode 100644 index 000000000000..a314bedfc4ba --- /dev/null +++ b/changelog.d/12511.misc @@ -0,0 +1 @@ +Remove unnecessary configuration overrides in tests. diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 57d97bbb47ae..65743cdf6777 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -355,7 +355,6 @@ def test_ignore_invalid_room(self) -> None: self.assertEqual(200, channel.code, channel.json_body) self.assertNotIn("m.relations", channel.json_body["unsigned"]) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_edit(self) -> None: """Test that a simple edit works.""" @@ -998,7 +997,6 @@ def assert_bundle(event_json: JsonDict) -> None: ] assert_bundle(self._find_event_in_chunk(chunk)) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_annotation(self) -> None: """ Test that annotations get correctly bundled. @@ -1023,7 +1021,6 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 7) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_reference(self) -> None: """ Test that references get correctly bundled. @@ -1042,7 +1039,6 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7) - @unittest.override_config({"experimental_features": {"msc3666_enabled": True}}) def test_thread(self) -> None: """ Test that threads get correctly bundled. From ecef741add0adfdd59f960e8b46503a5604303bd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 15:18:21 +0100 Subject: [PATCH 037/263] Recommend poetry in docs (#12475) * Recommend poetry in docs - readme - contributor guide - upgrade notes - new dev cheat sheet for poetry Co-authored-by: Shay Co-authored-by: Patrick Cloke --- README.rst | 31 ++-- changelog.d/12475.doc | 1 + docs/development/contributing_guide.md | 46 ++--- docs/development/dependencies.md | 239 +++++++++++++++++++++++++ docs/upgrade.md | 34 ++-- 5 files changed, 301 insertions(+), 50 deletions(-) create mode 100644 changelog.d/12475.doc create mode 100644 docs/development/dependencies.md diff --git a/README.rst b/README.rst index 8a14401d65af..d71d73367931 100644 --- a/README.rst +++ b/README.rst @@ -293,24 +293,27 @@ directory of your choice:: git clone https://github.com/matrix-org/synapse.git cd synapse -Synapse has a number of external dependencies, that are easiest -to install using pip and a virtualenv:: +Synapse has a number of external dependencies. We maintain a fixed development +environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend - python3 -m venv ./env - source ./env/bin/activate - pip install -e ".[all,dev]" + pip install --user pipx + pipx install poetry -This will run a process of downloading and installing all the needed -dependencies into a virtual env. If any dependencies fail to install, -try installing the failing modules individually:: +as described `here `_. +(See `poetry's installation docs ` +for other installation methods.) Then ask poetry to create a virtual environment +from the project and install Synapse's dependencies:: + + poetry install --extras "all test" - pip install -e "module-name" +This will run a process of downloading and installing all the needed +dependencies into a virtual env. We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082` - ./demo/start.sh + poetry run ./demo/start.sh -(to stop, you can use `./demo/stop.sh`) +(to stop, you can use `poetry run ./demo/stop.sh`) See the `demo documentation `_ for more information. @@ -318,14 +321,14 @@ for more information. If you just want to start a single instance of the app and run it directly:: # Create the homeserver.yaml config once - python -m synapse.app.homeserver \ + poetry run synapse_homeserver \ --server-name my.domain.name \ --config-path homeserver.yaml \ --generate-config \ --report-stats=[yes|no] # Start the app - python -m synapse.app.homeserver --config-path homeserver.yaml + poetry run synapse_homeserver --config-path homeserver.yaml Running the unit tests @@ -334,7 +337,7 @@ Running the unit tests After getting up and running, you may wish to run Synapse's unit tests to check that everything is installed correctly:: - trial tests + poetry run trial tests This should end with a 'PASSED' result (note that exact numbers will differ):: diff --git a/changelog.d/12475.doc b/changelog.d/12475.doc new file mode 100644 index 000000000000..f4481d0613c2 --- /dev/null +++ b/changelog.d/12475.doc @@ -0,0 +1 @@ +Strongly recommend `poetry` for development. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 0d9cf6019607..3b5c77401863 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -48,19 +48,28 @@ can find many good git tutorials on the web. # 4. Install the dependencies -Once you have installed Python 3 and added the source, please open a terminal and -setup a *virtualenv*, as follows: +Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies +and development environment. Once you have installed Python 3 and added the +source, you should install `poetry`. +Of their installation methods, we recommend +[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx), + +```shell +pip install --user pipx +pipx install poetry +``` + +but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) +for other installation methods. + +Next, open a terminal and install dependencies as follows: ```sh cd path/where/you/have/cloned/the/repository -python3 -m venv ./env -source ./env/bin/activate -pip install wheel -pip install -e ".[all,dev]" -pip install tox +poetry install --extras all ``` -This will install the developer dependencies for the project. +This will install the runtime and developer dependencies for the project. # 5. Get in touch. @@ -117,11 +126,10 @@ The linters look at your code and do two things: - ensure that your code follows the coding style adopted by the project; - catch a number of errors in your code. -The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies). +The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies). ```sh -source ./env/bin/activate -./scripts-dev/lint.sh +poetry run ./scripts-dev/lint.sh ``` Note that this script *will modify your files* to fix styling errors. @@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com (much faster!), you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh -d +poetry run ./scripts-dev/lint.sh -d ``` Or if you know exactly which files you wish to lint, you can instead run: ```sh -source ./env/bin/activate -./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder +poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder ``` ## Run the unit tests (Twisted trial). @@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything was broken. They are slower than the linters but will typically catch more errors. ```sh -source ./env/bin/activate -trial tests +poetry run trial tests ``` If you wish to only run *some* unit tests, you may specify another module instead of `tests` - or a test class or a method: ```sh -source ./env/bin/activate -trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite +poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite ``` If your tests fail, you may wish to look at the logs (the default log level is `ERROR`): @@ -169,7 +173,7 @@ less _trial_temp/test.log To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`: ```sh -SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests +SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests ``` By default, tests will use an in-memory SQLite database for test data. For additional @@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process' working directory. Typically, this ends up being `_trial_temp/test.db`. For example: ```sh -SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests +SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests ``` The database file can then be inspected with: diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md new file mode 100644 index 000000000000..8ef7d357d8cd --- /dev/null +++ b/docs/development/dependencies.md @@ -0,0 +1,239 @@ +# Managing dependencies with Poetry + +This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/). + +# Background + +Synapse uses a variety of third-party Python packages to function as a homeserver. +Some of these are direct dependencies, listed in `pyproject.toml` under the +`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the +things that our direct dependencies themselves depend on, and so on recursively.) + +We maintain a locked list of all our dependencies (transitive included) so that +we can track exactly which version of each dependency appears in a given release. +See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665) +for discussion of why we wanted this for Synapse. We chose to use +[`poetry`](https://python-poetry.org/) to manage this locked list; see +[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819) +for the reasoning. + +The locked dependencies get included in our "self-contained" releases: namely, +our docker images and our debian packages. We also use the locked dependencies +in development and our continuous integration. + +Separately, our "broad" dependencies—the version ranges specified in +`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded +to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from +the Synapse source tree directly will _not_ use the locked dependencies; instead, +they'll pull in the latest version of each package available at install time. + +## Example dependency + +An example may help. We have a broad dependency on +[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in +this snippet from pyproject.toml [as of Synapse 1.57]( +https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133 +): + +```toml +[tool.poetry.dependencies] +# ... +phonenumbers = ">=8.2.0" +``` + +In our lockfile this is +[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685) +to version 8.12.44, even though +[newer versions are available](https://pypi.org/project/phonenumbers/#history). + +```toml +[[package]] +name = "phonenumbers" +version = "8.12.44" +description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." +category = "main" +optional = false +python-versions = "*" +``` + +The lockfile also includes a +[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181) +of the sdists and wheels provided for this version of `phonenumbers`. + +```toml +[metadata.files] +# ... +phonenumbers = [ + {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"}, + {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"}, +] +``` + +We can see this pinned version inside the docker image for that release: + +``` +$ docker pull matrixdotorg/synapse:v1.57.0 +... +$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers +Name: phonenumbers +Version: 8.12.44 +Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers. +Home-page: https://github.com/daviddrysdale/python-phonenumbers +Author: David Drysdale +Author-email: dmd@lurklurk.org +License: Apache License 2.0 +Location: /usr/local/lib/python3.9/site-packages +Requires: +Required-by: matrix-synapse +``` + +Whereas the wheel metadata just contains the broad dependencies: + +``` +$ cd /tmp +$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl +... +$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers +Requires-Dist: phonenumbers (>=8.2.0) +``` + +# Tooling recommendation: direnv + +[`direnv`](https://direnv.net/) is a tool for activating environments in your +shell inside a given directory. Its support for poetry is unofficial (a +community wiki recipe only), but works solidly in our experience. We thoroughly +recommend it for daily use. To use it: + +1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely + packaged for your system already. +2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry) + needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`). +3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`. +4. Convince yourself that you trust this `.envrc` configuration and project. + Then formally confirm this to `direnv` by running `direnv allow`. + +Then whenever you navigate to the synapse checkout, you should be able to run +e.g. `mypy` instead of `poetry run mypy`; `python` instead of +`poetry run python`; and your shell commands will automatically run in the +context of poetry's venv, without having to run `poetry shell` beforehand. + + +# How do I... + +## ...reset my venv to the locked environment? + +```shell +poetry install --extras all --remove-untracked +``` + +## ...run a command in the `poetry` virtualenv? + +Use `poetry run cmd args` when you need the python virtualenv context. +To avoid typing `poetry run` all the time, you can run `poetry shell` +to start a new shell in the poetry virtualenv context. Within `poetry shell`, +`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv +and isolated from the rest o the system. + +Roughly speaking, the translation from a traditional virtualenv is: +- `env/bin/activate` -> `poetry shell`, and +- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.) + +See also the direnv recommendation above, which makes `poetry run` and +`poetry shell` unnecessary. + + +## ...inspect the `poetry` virtualenv? + +Some suggestions: + +```shell +# Current env only +poetry env info +# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7, +# and another for Python 3.10. +poetry env list --full-path +poetry run pip list +``` + +Note that `poetry show` describes the abstract *lock file* rather than your +on-disk environment. With that said, `poetry show --tree` can sometimes be +useful. + + +## ...add a new dependency? + +Either: +- manually update `pyproject.toml`; then `poetry lock --no-update`; or else +- `poetry add packagename`. See `poetry add --help`; note the `--dev`, + `--extras` and `--optional` flags in particular. + - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39). + +Include the updated `pyproject.toml` and `poetry.lock` files in your commit. + +## ...remove a dependency? + +This is not done often and is untested, but + +```shell +poetry remove packagename +``` + +ought to do the trick. Alternatively, manually update `pyproject.toml` and +`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock` +files in your commit. + +## ...update the version range for an existing dependency? + +Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`. +Include the updated `pyproject.toml` and `poetry.lock` in your commit. + +## ...update a dependency in the locked environment? + +Use + +```shell +poetry update packagename +``` + +to use the latest version of `packagename` in the locked environment, without +affecting the broad dependencies listed in the wheel. + +There doesn't seem to be a way to do this whilst locking a _specific_ version of +`packagename`. We can workaround this (crudely) as follows: + +```shell +poetry add packagename==1.2.3 +# This should update pyproject.lock. + +# Now undo the changes to pyproject.toml. For example +# git restore pyproject.toml + +# Get poetry to recompute the content-hash of pyproject.toml without changing +# the locked package versions. +poetry lock --no-update +``` + +Either way, include the updated `poetry.lock` file in your commit. + +## ...export a `requirements.txt` file? + +```shell +poetry export --extras all +``` + +Be wary of bugs in `poetry export` and `pip install -r requirements.txt`. + +Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may +be required. + +## ...build a test wheel? + +I usually use + +```shell +poetry run pip install build && poetry run python -m build +``` + +because [`build`](https://github.com/pypa/build) is a standardish tool which +doesn't require poetry. (It's what we use in CI too). However, you could try +`poetry build` too. diff --git a/docs/upgrade.md b/docs/upgrade.md index a0c797ea9fee..3a8aeb039533 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -19,32 +19,36 @@ this document. packages](setup/installation.md#prebuilt-packages), you will need to follow the normal process for upgrading those packages. +- If Synapse was installed using pip then upgrade to the latest + version by running: + + ```bash + pip install --upgrade matrix-synapse + ``` + - If Synapse was installed from source, then: - 1. Activate the virtualenv before upgrading. For example, if - Synapse is installed in a virtualenv in `~/synapse/env` then + 1. Obtain the latest version of the source code. Git users can run + `git pull` to do this. + + 2. If you're running Synapse in a virtualenv, make sure to activate it before + upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then run: ```bash source ~/synapse/env/bin/activate + pip install --upgrade . ``` + Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`. - 2. If Synapse was installed using pip then upgrade to the latest - version by running: - - ```bash - pip install --upgrade matrix-synapse - ``` - - If Synapse was installed using git then upgrade to the latest - version by running: - + 3. If you're using `poetry` to manage a Synapse installation, run: ```bash - git pull - pip install --upgrade . + poetry install ``` + Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`. + It's probably easiest to run `poetry install --extras all`. - 3. Restart Synapse: + 4. Restart Synapse: ```bash synctl restart From 3e2e76ca1507ff6b0ed4e5095fd602b61a26704f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 20 Apr 2022 15:22:53 +0100 Subject: [PATCH 038/263] Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. (#12512) * poetry update matrix-synapse-ldap3 * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/12512.docker | 1 + poetry.lock | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12512.docker diff --git a/changelog.d/12512.docker b/changelog.d/12512.docker new file mode 100644 index 000000000000..24c0f6a3b1da --- /dev/null +++ b/changelog.d/12512.docker @@ -0,0 +1 @@ +Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index bbe8eba96d40..c11e2c684873 100644 --- a/poetry.lock +++ b/poetry.lock @@ -558,17 +558,20 @@ test = ["tox", "twisted", "aiounittest"] [[package]] name = "matrix-synapse-ldap3" -version = "0.1.5" +version = "0.2.0" description = "An LDAP3 auth provider for Synapse" category = "main" optional = true -python-versions = "*" +python-versions = ">=3.7" [package.dependencies] ldap3 = ">=2.8" -service_identity = "*" +service-identity = "*" Twisted = ">=15.1.0" +[package.extras] +dev = ["matrix-synapse", "tox", "ldaptor", "mypy (==0.910)", "types-setuptools", "black (==21.9b0)", "flake8 (==4.0.1)", "isort (==5.9.3)"] + [[package]] name = "mccabe" version = "0.6.1" @@ -2084,7 +2087,8 @@ matrix-common = [ {file = "matrix_common-1.1.0.tar.gz", hash = "sha256:a8238748afc2b37079818367fed5156f355771b07c8ff0a175934f47e0ff3276"}, ] matrix-synapse-ldap3 = [ - {file = "matrix-synapse-ldap3-0.1.5.tar.gz", hash = "sha256:9fdf8df7c8ec756642aa0fea53b31c0b2f1924f70d7f049a2090b523125456fe"}, + {file = "matrix-synapse-ldap3-0.2.0.tar.gz", hash = "sha256:91a0715b43a41ec3033244174fca20846836da98fda711fb01687f7199eecd2e"}, + {file = "matrix_synapse_ldap3-0.2.0-py3-none-any.whl", hash = "sha256:0128ca7c3058987adc2e8a88463bb46879915bfd3d373309632813b353e30f9f"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, From 05e8a5d298011508e04ecf4b2f73aaa737ed6035 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 20 Apr 2022 15:30:03 +0100 Subject: [PATCH 039/263] 1.57.1 --- CHANGES.md | 13 +++++++++++++ changelog.d/12512.docker | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- synapse/__init__.py | 2 +- 5 files changed, 21 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/12512.docker diff --git a/CHANGES.md b/CHANGES.md index 74a5dbf424b1..63c66af5bf5a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +Synapse 1.57.1 (2022-04-20) +=========================== + +This is a patch release that only affects the Docker image. It is only of interest to administrators using the LDAP module to authenticate their users. +If you have already upgraded to Synapse 1.57.0 without problem, then you have no need to upgrade to this patch release. + + +Updates to the Docker image +--------------------------- + +- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) + + Synapse 1.57.0 (2022-04-19) =========================== diff --git a/changelog.d/12512.docker b/changelog.d/12512.docker deleted file mode 100644 index 24c0f6a3b1da..000000000000 --- a/changelog.d/12512.docker +++ /dev/null @@ -1 +0,0 @@ -Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 71dcf9de8e3e..2db6ed949100 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.57.1) stable; urgency=medium + + * New synapse release 1.57.1. + + -- Synapse Packaging team Wed, 20 Apr 2022 15:27:21 +0100 + matrix-synapse-py3 (1.57.0) stable; urgency=medium * New synapse release 1.57.0. diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3fb7..65f5a5f59fc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.57.0" +version = "1.57.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d3c..b62eed66e273 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -68,7 +68,7 @@ except ImportError: pass -__version__ = "1.57.0" +__version__ = "1.57.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From f8f06fc773e6d1dd9674be77d3e6a3665830a33a Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 20 Apr 2022 15:48:05 +0100 Subject: [PATCH 040/263] Clarify changelog entry --- CHANGES.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 63c66af5bf5a..a7d2529b557e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,16 @@ Synapse 1.57.1 (2022-04-20) =========================== -This is a patch release that only affects the Docker image. It is only of interest to administrators using the LDAP module to authenticate their users. +This is a patch release that only affects the Docker image. It is only of interest to administrators using [the LDAP module][LDAPModule] to authenticate their users. If you have already upgraded to Synapse 1.57.0 without problem, then you have no need to upgrade to this patch release. +[LDAPModule]: https://github.com/matrix-org/matrix-synapse-ldap3 + Updates to the Docker image --------------------------- -- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) +- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. This matches the version that was present in the Docker image for Synapse v1.56.0. ([\#12512](https://github.com/matrix-org/synapse/issues/12512)) Synapse 1.57.0 (2022-04-19) From 103f51d867dc3966e44d79e949e78380efef5e2d Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 20 Apr 2022 12:03:03 -0400 Subject: [PATCH 041/263] Fix Jinja templating error when generating thumbnail URLs. (#12510) scale is meant to be a constant string, not refer to a variable. --- changelog.d/12510.bugfix | 1 + synapse/res/templates/notif.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12510.bugfix diff --git a/changelog.d/12510.bugfix b/changelog.d/12510.bugfix new file mode 100644 index 000000000000..d5856e982a57 --- /dev/null +++ b/changelog.d/12510.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. diff --git a/synapse/res/templates/notif.html b/synapse/res/templates/notif.html index 0aaef97df893..7d86681fed53 100644 --- a/synapse/res/templates/notif.html +++ b/synapse/res/templates/notif.html @@ -30,7 +30,7 @@ {%- elif message.msgtype == "m.notice" %} {{ message.body_text_html }} {%- elif message.msgtype == "m.image" and message.image_url %} - + {%- elif message.msgtype == "m.file" %} {{ message.body_text_plain }} {%- else %} From e5a76ec00b443c6e15ac8de2838a5c60a47a67ec Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 17:33:20 +0100 Subject: [PATCH 042/263] Dump setuptools; correct pyproject version number (#12478) --- .dockerignore | 4 - .github/workflows/tests.yml | 2 +- .gitignore | 3 +- MANIFEST.in | 54 ---------- changelog.d/12337.feature | 1 + changelog.d/12478.misc | 1 + docker/Dockerfile | 6 +- mypy.ini | 1 - pyproject.toml | 4 +- setup.cfg | 9 -- setup.py | 183 --------------------------------- synapse/__init__.py | 4 +- synapse/python_dependencies.py | 151 --------------------------- 13 files changed, 11 insertions(+), 412 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 changelog.d/12337.feature create mode 100644 changelog.d/12478.misc delete mode 100644 setup.cfg delete mode 100755 setup.py delete mode 100644 synapse/python_dependencies.py diff --git a/.dockerignore b/.dockerignore index a236760cf1fd..7809863ef328 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,8 +8,4 @@ !pyproject.toml !poetry.lock -# TODO: remove these once we have moved over to using poetry-core in pyproject.toml -!MANIFEST.in -!setup.py - **/__pycache__ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7946b76dba8c..cad4cb6d77b3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: pip install -e . + - run: pip install . - run: scripts-dev/generate_sample_config.sh --check - run: scripts-dev/config-lint.sh diff --git a/.gitignore b/.gitignore index c011cd27a4df..e58affb24125 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,7 @@ _trial_temp*/ .DS_Store __pycache__/ -# We do want the poetry lockfile. TODO: is there a good reason for ignoring -# '*.lock' above? If not, let's nuke it. +# We do want the poetry lockfile. !poetry.lock # stuff that is likely to exist when you run a server locally diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d744c090acde..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,54 +0,0 @@ -include LICENSE -include VERSION -include *.rst -include *.md -include demo/README -include demo/demo.tls.dh -include demo/*.py -include demo/*.sh - -include synapse/py.typed -recursive-include synapse/storage *.sql -recursive-include synapse/storage *.sql.postgres -recursive-include synapse/storage *.sql.sqlite -recursive-include synapse/storage *.py -recursive-include synapse/storage *.txt -recursive-include synapse/storage *.md - -recursive-include docs * -recursive-include scripts-dev * -recursive-include synapse *.pyi -recursive-include tests *.py -recursive-include tests *.pem -recursive-include tests *.p8 -recursive-include tests *.crt -recursive-include tests *.key - -recursive-include synapse/res * -recursive-include synapse/static *.css -recursive-include synapse/static *.gif -recursive-include synapse/static *.html -recursive-include synapse/static *.js - -exclude .codecov.yml -exclude .coveragerc -exclude .dockerignore -exclude .editorconfig -exclude Dockerfile -exclude mypy.ini -exclude sytest-blacklist -exclude test_postgresql.sh - -include book.toml -include pyproject.toml -recursive-include changelog.d * - -include .flake8 -prune .circleci -prune .github -prune .ci -prune contrib -prune debian -prune demo/etc -prune docker -prune stubs diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature new file mode 100644 index 000000000000..6c4444c70722 --- /dev/null +++ b/changelog.d/12337.feature @@ -0,0 +1 @@ +Use poetry to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc new file mode 100644 index 000000000000..061a604a1e16 --- /dev/null +++ b/changelog.d/12478.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f87702cc88b..4523c60645bd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ WORKDIR /synapse # Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock README.rst /synapse/ +COPY pyproject.toml poetry.lock /synapse/ RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt @@ -98,9 +98,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ # ... and what we need to `pip install`. -# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy -# pyproject.toml here, ditching setup.py and MANIFEST.in. -COPY setup.py MANIFEST.in README.rst /synapse/ +COPY pyproject.toml README.rst /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse diff --git a/mypy.ini b/mypy.ini index 5246f987c009..b2c3b1524ced 100644 --- a/mypy.ini +++ b/mypy.ini @@ -13,7 +13,6 @@ no_implicit_optional = True files = docker/, scripts-dev/, - setup.py, synapse/, tests/ diff --git a/pyproject.toml b/pyproject.toml index 7f58c37e3fb7..c04857b79165 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -280,5 +280,5 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6213f3265b59..000000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[check-manifest] -ignore = - .git-blame-ignore-revs - contrib - contrib/* - docs/* - pylint.cfg - tox.ini - diff --git a/setup.py b/setup.py deleted file mode 100755 index ecd30247ed6b..000000000000 --- a/setup.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2017 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2017-2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Any, Dict - -from setuptools import Command, find_packages, setup - -here = os.path.abspath(os.path.dirname(__file__)) - - -# Some notes on `setup.py test`: -# -# Once upon a time we used to try to make `setup.py test` run `tox` to run the -# tests. That's a bad idea for three reasons: -# -# 1: `setup.py test` is supposed to find out whether the tests work in the -# *current* environmentt, not whatever tox sets up. -# 2: Empirically, trying to install tox during the test run wasn't working ("No -# module named virtualenv"). -# 3: The tox documentation advises against it[1]. -# -# Even further back in time, we used to use setuptools_trial [2]. That has its -# own set of issues: for instance, it requires installation of Twisted to build -# an sdist (because the recommended mode of usage is to add it to -# `setup_requires`). That in turn means that in order to successfully run tox -# you have to have the python header files installed for whichever version of -# python tox uses (which is python3 on recent ubuntus, for example). -# -# So, for now at least, we stick with what appears to be the convention among -# Twisted projects, and don't attempt to do anything when someone runs -# `setup.py test`; instead we direct people to run `trial` directly if they -# care. -# -# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command -# [2]: https://pypi.python.org/pypi/setuptools_trial -class TestCommand(Command): - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - print( - """Synapse's tests cannot be run via setup.py. To run them, try: - PYTHONPATH="." trial tests -""" - ) - - -def read_file(path_segments): - """Read a file from the package. Takes a list of strings to join to - make the path""" - file_path = os.path.join(here, *path_segments) - with open(file_path) as f: - return f.read() - - -def exec_file(path_segments): - """Execute a single python file to get the variables defined in it""" - result: Dict[str, Any] = {} - code = read_file(path_segments) - exec(code, result) - return result - - -version = exec_file(("synapse", "__init__.py"))["__version__"] -dependencies = exec_file(("synapse", "python_dependencies.py")) -long_description = read_file(("README.rst",)) - -REQUIREMENTS = dependencies["REQUIREMENTS"] -CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"] -ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"] - -# Make `pip install matrix-synapse[all]` install all the optional dependencies. -CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) - -# Developer dependencies should not get included in "all". -# -# We pin black so that our tests don't start failing on new releases. -CONDITIONAL_REQUIREMENTS["lint"] = [ - "isort==5.7.0", - "black==22.3.0", - "flake8-comprehensions", - "flake8-bugbear==21.3.2", - "flake8", -] - -CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy==0.931", - "mypy-zope==0.3.5", - "types-bleach>=4.1.0", - "types-jsonschema>=3.2.0", - "types-opentracing>=2.4.2", - "types-Pillow>=8.3.4", - "types-psycopg2>=2.9.9", - "types-pyOpenSSL>=20.0.7", - "types-PyYAML>=5.4.10", - "types-requests>=2.26.0", - "types-setuptools>=57.4.0", -] - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# parameterized_class decorator was introduced in parameterized 0.7.0 -CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"] - -CONDITIONAL_REQUIREMENTS["dev"] = ( - CONDITIONAL_REQUIREMENTS["lint"] - + CONDITIONAL_REQUIREMENTS["mypy"] - + CONDITIONAL_REQUIREMENTS["test"] - + [ - # The following are used by the release script - "click==8.1.0", - "redbaron==0.9.2", - "GitPython==3.1.14", - "commonmark==0.9.1", - "pygithub==1.55", - # The following are executed as commands by the release script. - "twine", - "towncrier", - ] -) - -setup( - name="matrix-synapse", - version=version, - packages=find_packages(exclude=["tests", "tests.*"]), - description="Reference homeserver for the Matrix decentralised comms protocol", - install_requires=REQUIREMENTS, - extras_require=CONDITIONAL_REQUIREMENTS, - include_package_data=True, - zip_safe=False, - long_description=long_description, - long_description_content_type="text/x-rst", - python_requires="~=3.7", - entry_points={ - "console_scripts": [ - # Application - "synapse_homeserver = synapse.app.homeserver:main", - "synapse_worker = synapse.app.generic_worker:main", - "synctl = synapse._scripts.synctl:main", - # Scripts - "export_signing_key = synapse._scripts.export_signing_key:main", - "generate_config = synapse._scripts.generate_config:main", - "generate_log_config = synapse._scripts.generate_log_config:main", - "generate_signing_key = synapse._scripts.generate_signing_key:main", - "hash_password = synapse._scripts.hash_password:main", - "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main", - "synapse_port_db = synapse._scripts.synapse_port_db:main", - "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main", - "update_synapse_database = synapse._scripts.update_synapse_database:main", - ] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Communications :: Chat", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - cmdclass={"test": TestCommand}, -) diff --git a/synapse/__init__.py b/synapse/__init__.py index bf88f1d93d3c..161394175939 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import os import sys +from matrix_common.versionstring import get_distribution_version_string + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -68,7 +70,7 @@ except ImportError: pass -__version__ = "1.57.0" +__version__ = get_distribution_version_string("matrix-synapse") if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py deleted file mode 100644 index ec199a161db8..000000000000 --- a/synapse/python_dependencies.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import itertools -import logging -from typing import Set - -logger = logging.getLogger(__name__) - - -# REQUIREMENTS is a simple list of requirement specifiers[1], and must be -# installed. It is passed to setup() as install_requires in setup.py. -# -# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict -# of lists. The dict key is the optional dependency name and can be passed to -# pip when installing. The list is a series of requirement specifiers[1] to be -# installed when that optional dependency requirement is specified. It is passed -# to setup() as extras_require in setup.py -# -# Note that these both represent runtime dependencies (and the versions -# installed are checked at runtime). -# -# Also note that we replicate these constraints in the Synapse Dockerfile while -# pre-installing dependencies. If these constraints are updated here, the same -# change should be made in the Dockerfile. -# -# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers. - -REQUIREMENTS = [ - # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 - "jsonschema>=3.0.0", - # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 - "frozendict>=1,!=2.1.2", - "unpaddedbase64>=1.1.0", - "canonicaljson>=1.4.0", - # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0", - "pynacl>=1.2.1", - # validating SSL certs for IP addresses requires service_identity 18.1. - "service_identity>=18.1.0", - # Twisted 18.9 introduces some logger improvements that the structured - # logger utilises - "Twisted[tls]>=18.9.0", - "treq>=15.1", - # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyopenssl>=16.0.0", - "pyyaml>=3.11", - "pyasn1>=0.1.9", - "pyasn1-modules>=0.0.7", - "bcrypt>=3.1.0", - "pillow>=5.4.0", - "sortedcontainers>=1.4.4", - "pymacaroons>=0.13.0", - "msgpack>=0.5.2", - "phonenumbers>=8.2.0", - # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. - "prometheus_client>=0.4.0", - # we use `order`, which arrived in attrs 19.2.0. - # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.2.0,!=21.1.0", - "netaddr>=0.7.18", - # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not - # end up with a broken installation, with recent MarkupSafe but old Jinja, we - # add a lower bound to the Jinja2 dependency. - "Jinja2>=3.0", - "bleach>=1.4.3", - # We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. - "typing-extensions>=3.10.0", - # We enforce that we have a `cryptography` version that bundles an `openssl` - # with the latest security patches. - "cryptography>=3.4.7", - # ijson 3.1.4 fixes a bug with "." in property names - "ijson>=3.1.4", - "matrix-common~=1.1.0", - # We need packaging.requirements.Requirement, added in 16.1. - "packaging>=16.1", - # At the time of writing, we only use functions from the version `importlib.metadata` - # which shipped in Python 3.8. This corresponds to version 1.4 of the backport. - "importlib_metadata>=1.4 ; python_version < '3.8'", -] - -CONDITIONAL_REQUIREMENTS = { - "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], - "postgres": [ - # we use execute_values with the fetch param, which arrived in psycopg 2.8. - "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", - ], - "saml2": [ - "pysaml2>=4.5.0", - ], - "oidc": ["authlib>=0.14.0"], - # systemd-python is necessary for logging to the systemd journal via - # `systemd.journal.JournalHandler`, as is documented in - # `contrib/systemd/log_config.yaml`. - "systemd": ["systemd-python>=231"], - "url_preview": ["lxml>=4.2.0"], - "sentry": ["sentry-sdk>=0.7.2"], - "opentracing": ["jaeger-client>=4.0.0", "opentracing>=2.2.0"], - "jwt": ["pyjwt>=1.6.4"], - # hiredis is not a *strict* dependency, but it makes things much faster. - # (if it is not installed, we fall back to slow code.) - "redis": ["txredisapi>=1.4.7", "hiredis"], - # Required to use experimental `caches.track_memory_usage` config option. - "cache_memory": ["pympler"], -} - -ALL_OPTIONAL_REQUIREMENTS: Set[str] = set() - -for name, optional_deps in CONDITIONAL_REQUIREMENTS.items(): - # Exclude systemd as it's a system-based requirement. - # Exclude lint as it's a dev-based requirement. - if name not in ["systemd"]: - ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS - - -# ensure there are no double-quote characters in any of the deps (otherwise the -# 'pip install' incantation in DependencyException will break) -for dep in itertools.chain( - REQUIREMENTS, - *CONDITIONAL_REQUIREMENTS.values(), -): - if '"' in dep: - raise Exception( - "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) - ) - - -def list_requirements(): - return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) - - -if __name__ == "__main__": - import sys - - sys.stdout.writelines(req + "\n" for req in list_requirements()) From 09b4f6e46d3cbc3d93c52a7b76db17f178167b3e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 20 Apr 2022 19:16:49 +0100 Subject: [PATCH 043/263] Remove leftover references to setup.py (#12514) * Remove leftover references to setup.py Missed in #12478. * Changelog --- changelog.d/12514.misc | 1 + mypy.ini | 4 ++-- scripts-dev/lint.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12514.misc diff --git a/changelog.d/12514.misc b/changelog.d/12514.misc new file mode 100644 index 000000000000..061a604a1e16 --- /dev/null +++ b/changelog.d/12514.misc @@ -0,0 +1 @@ +Use poetry-core instead of setuptools to build wheels. diff --git a/mypy.ini b/mypy.ini index b2c3b1524ced..a663bf69752a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -233,8 +233,8 @@ disallow_untyped_defs = True ;; The `typeshed` project maintains stubs here: ;; https://github.com/python/typeshed/tree/master/stubs ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI, -;; which we can pull in as a dev dependency by adding to `setup.py`'s -;; `CONDITIONAL_REQUIREMENTS["mypy"]` list. +;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s +;; `[tool.poetry.dev-dependencies]` list. [mypy-authlib.*] ignore_missing_imports = True diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 91a704d98247..377348b107ea 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -91,7 +91,7 @@ else files=( "synapse" "docker" "tests" "scripts-dev" - "contrib" "setup.py" "synmark" "stubs" ".ci" + "contrib" "synmark" "stubs" ".ci" ) fi fi From f5668f0b4a6cca659ae98d3cb3714692ba488e89 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 21 Apr 2022 07:42:03 +0100 Subject: [PATCH 044/263] Await un-partial-stating after a partial-state join (#12399) When we join a room via the faster-joins mechanism, we end up with "partial state" at some points on the event DAG. Many parts of the codebase need to wait for the full state to load. So, we implement a mechanism to keep track of which events have partial state, and wait for them to be fully-populated. --- changelog.d/12399.misc | 1 + docker/complement/conf/homeserver.yaml | 4 +- scripts-dev/complement.sh | 2 +- synapse/handlers/federation_event.py | 1 + .../storage/databases/main/events_worker.py | 10 +- synapse/storage/databases/main/state.py | 1 + synapse/storage/state.py | 28 +++- .../util/partial_state_events_tracker.py | 120 ++++++++++++++++++ tests/storage/util/__init__.py | 13 ++ .../util/test_partial_state_events_tracker.py | 117 +++++++++++++++++ 10 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12399.misc create mode 100644 synapse/storage/util/partial_state_events_tracker.py create mode 100644 tests/storage/util/__init__.py create mode 100644 tests/storage/util/test_partial_state_events_tracker.py diff --git a/changelog.d/12399.misc b/changelog.d/12399.misc new file mode 100644 index 000000000000..cd2e09626d35 --- /dev/null +++ b/changelog.d/12399.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml index c9d6a312401a..174f87f52ee0 100644 --- a/docker/complement/conf/homeserver.yaml +++ b/docker/complement/conf/homeserver.yaml @@ -103,8 +103,10 @@ experimental_features: spaces_enabled: true # Enable history backfilling support msc2716_enabled: true - # server-side support for partial state in /send_join + # server-side support for partial state in /send_join responses msc3706_enabled: true + # client-side support for partial state in /send_join responses + faster_joins: true # Enable jump to date endpoint msc3030_enabled: true diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index d34e9f355483..e0feba05fa4f 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -64,4 +64,4 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF # Run the tests! echo "Images built; running complement" cd "$COMPLEMENT_DIR" -go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/... +go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/... diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 32bf02818c54..693b544286ec 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -515,6 +515,7 @@ async def update_state_for_partial_state_event( ) return await self._store.update_state_for_partial_state_event(event, context) + self._state_store.notify_event_un_partial_stated(event.event_id) async def backfill( self, dest: str, room_id: str, limit: int, extremities: Collection[str] diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 60876204bd0f..6d6e146ff160 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1974,7 +1974,15 @@ def get_event_id_for_timestamp_txn(txn: LoggingTransaction) -> Optional[str]: async def get_partial_state_events( self, event_ids: Collection[str] ) -> Dict[str, bool]: - """Checks which of the given events have partial state""" + """Checks which of the given events have partial state + + Args: + event_ids: the events we want to check for partial state. + + Returns: + a dict mapping from event id to partial-stateness. We return True for + any of the events which are unknown (or are outliers). + """ result = await self.db_pool.simple_select_many_batch( table="partial_state_events", column="event_id", diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 7a1b013fa3ac..e653841fe503 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -396,6 +396,7 @@ def _update_state_for_partial_state_event_txn( ) # TODO(faster_joins): need to do something about workers here + txn.call_after(self.is_partial_state_event.invalidate, (event.event_id,)) txn.call_after( self._get_state_group_for_event.prefill, (event.event_id,), diff --git a/synapse/storage/state.py b/synapse/storage/state.py index cda194e8c8b7..d1d58592145e 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -31,6 +31,7 @@ from synapse.api.constants import EventTypes from synapse.events import EventBase +from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker from synapse.types import MutableStateMap, StateKey, StateMap if TYPE_CHECKING: @@ -542,6 +543,10 @@ class StateGroupStorage: def __init__(self, hs: "HomeServer", stores: "Databases"): self.stores = stores + self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) + + def notify_event_un_partial_stated(self, event_id: str) -> None: + self._partial_state_events_tracker.notify_un_partial_stated(event_id) async def get_state_group_delta( self, state_group: int @@ -579,7 +584,7 @@ async def get_state_groups_ids( if not event_ids: return {} - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups(groups) @@ -668,7 +673,7 @@ async def get_state_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -709,7 +714,7 @@ async def get_state_ids_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.stores.main._get_state_group_for_events(event_ids) + event_to_groups = await self._get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -785,6 +790,23 @@ def _get_state_for_groups( groups, state_filter or StateFilter.all() ) + async def _get_state_group_for_events( + self, + event_ids: Collection[str], + await_full_state: bool = True, + ) -> Mapping[str, int]: + """Returns mapping event_id -> state_group + + Args: + event_ids: events to get state groups for + await_full_state: if true, will block if we do not yet have complete + state at this event. + """ + if await_full_state: + await self._partial_state_events_tracker.await_full_state(event_ids) + + return await self.stores.main._get_state_group_for_events(event_ids) + async def store_state_group( self, event_id: str, diff --git a/synapse/storage/util/partial_state_events_tracker.py b/synapse/storage/util/partial_state_events_tracker.py new file mode 100644 index 000000000000..a61a951ef0c5 --- /dev/null +++ b/synapse/storage/util/partial_state_events_tracker.py @@ -0,0 +1,120 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from collections import defaultdict +from typing import Collection, Dict, Set + +from twisted.internet import defer +from twisted.internet.defer import Deferred + +from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable +from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.util import unwrapFirstError + +logger = logging.getLogger(__name__) + + +class PartialStateEventsTracker: + """Keeps track of which events have partial state, after a partial-state join""" + + def __init__(self, store: EventsWorkerStore): + self._store = store + # a map from event id to a set of Deferreds which are waiting for that event to be + # un-partial-stated. + self._observers: Dict[str, Set[Deferred[None]]] = defaultdict(set) + + def notify_un_partial_stated(self, event_id: str) -> None: + """Notify that we now have full state for a given event + + Called by the state-resynchronization loop whenever we resynchronize the state + for a particular event. Unblocks any callers to await_full_state() for that + event. + + Args: + event_id: the event that now has full state. + """ + observers = self._observers.pop(event_id, None) + if not observers: + return + logger.info( + "Notifying %i things waiting for un-partial-stating of event %s", + len(observers), + event_id, + ) + with PreserveLoggingContext(): + for o in observers: + o.callback(None) + + async def await_full_state(self, event_ids: Collection[str]) -> None: + """Wait for all the given events to have full state. + + Args: + event_ids: the list of event ids that we want full state for + """ + # first try the happy path: if there are no partial-state events, we can return + # quickly + partial_state_event_ids = [ + ev + for ev, p in (await self._store.get_partial_state_events(event_ids)).items() + if p + ] + + if not partial_state_event_ids: + return + + logger.info( + "Awaiting un-partial-stating of events %s", + partial_state_event_ids, + stack_info=True, + ) + + # create an observer for each lazy-joined event + observers: Dict[str, Deferred[None]] = { + event_id: Deferred() for event_id in partial_state_event_ids + } + for event_id, observer in observers.items(): + self._observers[event_id].add(observer) + + try: + # some of them may have been un-lazy-joined between us checking the db and + # registering the observer, in which case we'd wait forever for the + # notification. Call back the observers now. + for event_id, partial in ( + await self._store.get_partial_state_events(observers.keys()) + ).items(): + # there may have been a call to notify_un_partial_stated during the + # db query, so the observers may already have been called. + if not partial and not observers[event_id].called: + observers[event_id].callback(None) + + await make_deferred_yieldable( + defer.gatherResults( + observers.values(), + consumeErrors=True, + ) + ).addErrback(unwrapFirstError) + logger.info("Events %s all un-partial-stated", observers.keys()) + finally: + # remove any observers we created. This should happen when the notification + # is received, but that might not happen for two reasons: + # (a) we're bailing out early on an exception (including us being + # cancelled during the await) + # (b) the event got de-lazy-joined before we set up the observer. + for event_id, observer in observers.items(): + observer_set = self._observers.get(event_id) + if observer_set: + observer_set.discard(observer) + if not observer_set: + del self._observers[event_id] diff --git a/tests/storage/util/__init__.py b/tests/storage/util/__init__.py new file mode 100644 index 000000000000..3a5f22c02235 --- /dev/null +++ b/tests/storage/util/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/storage/util/test_partial_state_events_tracker.py b/tests/storage/util/test_partial_state_events_tracker.py new file mode 100644 index 000000000000..303e190b6cc2 --- /dev/null +++ b/tests/storage/util/test_partial_state_events_tracker.py @@ -0,0 +1,117 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict +from unittest import mock + +from twisted.internet.defer import CancelledError, ensureDeferred + +from synapse.storage.util.partial_state_events_tracker import PartialStateEventsTracker + +from tests.unittest import TestCase + + +class PartialStateEventsTrackerTestCase(TestCase): + def setUp(self) -> None: + # the results to be returned by the mocked get_partial_state_events + self._events_dict: Dict[str, bool] = {} + + async def get_partial_state_events(events): + return {e: self._events_dict[e] for e in events} + + self.mock_store = mock.Mock(spec_set=["get_partial_state_events"]) + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events + + self.tracker = PartialStateEventsTracker(self.mock_store) + + def test_does_not_block_for_full_state_events(self): + self._events_dict = {"event1": False, "event2": False} + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + self.mock_store.get_partial_state_events.assert_called_once_with( + ["event1", "event2"] + ) + + def test_blocks_for_partial_state_events(self): + self._events_dict = {"event1": True, "event2": False} + + d = ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + + # there should be no result yet + self.assertNoResult(d) + + # notifying that the event has been de-partial-stated should unblock + self.tracker.notify_un_partial_stated("event1") + self.successResultOf(d) + + def test_un_partial_state_race(self): + # if the event is un-partial-stated between the initial check and the + # registration of the listener, it should not block. + self._events_dict = {"event1": True, "event2": False} + + async def get_partial_state_events(events): + res = {e: self._events_dict[e] for e in events} + # change the result for next time + self._events_dict = {"event1": False, "event2": False} + return res + + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + def test_un_partial_state_during_get_partial_state_events(self): + # we should correctly handle a call to notify_un_partial_stated during the + # second call to get_partial_state_events. + + self._events_dict = {"event1": True, "event2": False} + + async def get_partial_state_events1(events): + self.mock_store.get_partial_state_events.side_effect = ( + get_partial_state_events2 + ) + return {e: self._events_dict[e] for e in events} + + async def get_partial_state_events2(events): + self.tracker.notify_un_partial_stated("event1") + self._events_dict["event1"] = False + return {e: self._events_dict[e] for e in events} + + self.mock_store.get_partial_state_events.side_effect = get_partial_state_events1 + + self.successResultOf( + ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + ) + + def test_cancellation(self): + self._events_dict = {"event1": True, "event2": False} + + d1 = ensureDeferred(self.tracker.await_full_state(["event1", "event2"])) + self.assertNoResult(d1) + + d2 = ensureDeferred(self.tracker.await_full_state(["event1"])) + self.assertNoResult(d2) + + d1.cancel() + self.assertFailure(d1, CancelledError) + + # d2 should still be waiting! + self.assertNoResult(d2) + + self.tracker.notify_un_partial_stated("event1") + self.successResultOf(d2) From f46b223354895efee181507a96a3a9a72c933a4b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Apr 2022 14:23:40 +0100 Subject: [PATCH 045/263] turn-howto: fix some links --- docs/turn-howto.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/turn-howto.md b/docs/turn-howto.md index 3a2cd04e36a9..37a311ad9cc7 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -302,14 +302,14 @@ Here are a few things to try: (Understanding the output is beyond the scope of this document!) - * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/. + * You can test your Matrix homeserver TURN setup with . Note that this test is not fully reliable yet, so don't be discouraged if the test fails. [Here](https://github.com/matrix-org/voip-tester) is the github repo of the source of the tester, where you can file bug reports. * There is a WebRTC test tool at - https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To + . To use it, you will need a username/password for your TURN server. You can either: From b82fff66df6ea050d4d8da880c2ea2840e3fce0d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 22 Apr 2022 16:03:46 +0100 Subject: [PATCH 046/263] MSC3202: Fix device_unused_fallback_keys -> device_unused_fallback_key_types (#12520) * Fix device_unused_fallback_keys -> device_unused_fallback_key_types * changelog --- changelog.d/12520.bugfix | 1 + synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12520.bugfix diff --git a/changelog.d/12520.bugfix b/changelog.d/12520.bugfix new file mode 100644 index 000000000000..c73005fde837 --- /dev/null +++ b/changelog.d/12520.bugfix @@ -0,0 +1 @@ +Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. \ No newline at end of file diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d23d9221bc5b..a610fb785d38 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -42,7 +42,7 @@ # user ID -> {device ID -> {algorithm -> count}} TransactionOneTimeKeyCounts = Dict[str, Dict[str, Dict[str, int]]] -# Type for the `device_unused_fallback_keys` field in an appservice transaction +# Type for the `device_unused_fallback_key_types` field in an appservice transaction # user ID -> {device ID -> [algorithm]} TransactionUnusedFallbackKeys = Dict[str, Dict[str, List[str]]] diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 0cdbb04bfbe3..adc6b074dae6 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -278,7 +278,7 @@ async def push_bulk( ] = one_time_key_counts if unused_fallback_keys: body[ - "org.matrix.msc3202.device_unused_fallback_keys" + "org.matrix.msc3202.device_unused_fallback_key_types" ] = unused_fallback_keys if device_list_summary: body["org.matrix.msc3202.device_lists"] = { From a50fb411b366576c3d4796b04c45385bdb7a6897 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Fri, 22 Apr 2022 18:20:06 +0100 Subject: [PATCH 047/263] Update `delay_cancellation` to accept any awaitable (#12468) This will mainly be useful when dealing with module callbacks, which are all typed as returning `Awaitable`s instead of coroutines or `Deferred`s. Signed-off-by: Sean Quah --- changelog.d/12468.misc | 1 + synapse/storage/database.py | 3 +- synapse/util/async_helpers.py | 52 ++++++++++++++++++++++++++------ tests/util/test_async_helpers.py | 33 ++++++++++++++++++-- 4 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 changelog.d/12468.misc diff --git a/changelog.d/12468.misc b/changelog.d/12468.misc new file mode 100644 index 000000000000..3d5d25247f64 --- /dev/null +++ b/changelog.d/12468.misc @@ -0,0 +1 @@ +Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 5eb545c86e2a..df1e9c1b831d 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -41,7 +41,6 @@ from typing_extensions import Literal from twisted.enterprise import adbapi -from twisted.internet import defer from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig @@ -794,7 +793,7 @@ async def _runInteraction() -> R: # We also wait until everything above is done before releasing the # `CancelledError`, so that logging contexts won't get used after they have been # finished. - return await delay_cancellation(defer.ensureDeferred(_runInteraction())) + return await delay_cancellation(_runInteraction()) async def runWithConnection( self, diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 650e44de2287..e27c5d298f6a 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -14,6 +14,7 @@ # limitations under the License. import abc +import asyncio import collections import inspect import itertools @@ -25,6 +26,7 @@ Awaitable, Callable, Collection, + Coroutine, Dict, Generic, Hashable, @@ -701,27 +703,57 @@ def stop_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": return new_deferred -def delay_cancellation(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]": - """Delay cancellation of a `Deferred` until it resolves. +@overload +def delay_cancellation(awaitable: "defer.Deferred[T]") -> "defer.Deferred[T]": + ... + + +@overload +def delay_cancellation(awaitable: Coroutine[Any, Any, T]) -> "defer.Deferred[T]": + ... + + +@overload +def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]: + ... + + +def delay_cancellation(awaitable: Awaitable[T]) -> Awaitable[T]: + """Delay cancellation of a coroutine or `Deferred` awaitable until it resolves. Has the same effect as `stop_cancellation`, but the returned `Deferred` will not - resolve with a `CancelledError` until the original `Deferred` resolves. + resolve with a `CancelledError` until the original awaitable resolves. Args: - deferred: The `Deferred` to protect against cancellation. May optionally follow - the Synapse logcontext rules. + deferred: The coroutine or `Deferred` to protect against cancellation. May + optionally follow the Synapse logcontext rules. Returns: - A new `Deferred`, which will contain the result of the original `Deferred`. - The new `Deferred` will not propagate cancellation through to the original. - When cancelled, the new `Deferred` will wait until the original `Deferred` - resolves before failing with a `CancelledError`. + A new `Deferred`, which will contain the result of the original coroutine or + `Deferred`. The new `Deferred` will not propagate cancellation through to the + original coroutine or `Deferred`. - The new `Deferred` will follow the Synapse logcontext rules if `deferred` + When cancelled, the new `Deferred` will wait until the original coroutine or + `Deferred` resolves before failing with a `CancelledError`. + + The new `Deferred` will follow the Synapse logcontext rules if `awaitable` follows the Synapse logcontext rules. Otherwise the new `Deferred` should be wrapped with `make_deferred_yieldable`. """ + # First, convert the awaitable into a `Deferred`. + if isinstance(awaitable, defer.Deferred): + deferred = awaitable + elif asyncio.iscoroutine(awaitable): + # Ideally we'd use `Deferred.fromCoroutine()` here, to save on redundant + # type-checking, but we'd need Twisted >= 21.2. + deferred = defer.ensureDeferred(awaitable) + else: + # We have no idea what to do with this awaitable. + # We assume it's already resolved, such as `DoneAwaitable`s or `Future`s from + # `make_awaitable`, and let the caller `await` it normally. + return awaitable + def handle_cancel(new_deferred: "defer.Deferred[T]") -> None: # before the new deferred is cancelled, we `pause` it to stop the cancellation # propagating. we then `unpause` it once the wrapped deferred completes, to diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index e5bc416de12f..daacc54c72bd 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -382,7 +382,7 @@ def test_cancellation(self): class DelayCancellationTests(TestCase): """Tests for the `delay_cancellation` function.""" - def test_cancellation(self): + def test_deferred_cancellation(self): """Test that cancellation of the new `Deferred` waits for the original.""" deferred: "Deferred[str]" = Deferred() wrapper_deferred = delay_cancellation(deferred) @@ -403,6 +403,35 @@ def test_cancellation(self): # Now that the original `Deferred` has failed, we should get a `CancelledError`. self.failureResultOf(wrapper_deferred, CancelledError) + def test_coroutine_cancellation(self): + """Test that cancellation of the new `Deferred` waits for the original.""" + blocking_deferred: "Deferred[None]" = Deferred() + completion_deferred: "Deferred[None]" = Deferred() + + async def task(): + await blocking_deferred + completion_deferred.callback(None) + # Raise an exception. Twisted should consume it, otherwise unwanted + # tracebacks will be printed in logs. + raise ValueError("abc") + + wrapper_deferred = delay_cancellation(task()) + + # Cancel the new `Deferred`. + wrapper_deferred.cancel() + self.assertNoResult(wrapper_deferred) + self.assertFalse( + blocking_deferred.called, "Cancellation was propagated too deep" + ) + self.assertFalse(completion_deferred.called) + + # Unblock the task. + blocking_deferred.callback(None) + self.assertTrue(completion_deferred.called) + + # Now that the original coroutine has failed, we should get a `CancelledError`. + self.failureResultOf(wrapper_deferred, CancelledError) + def test_suppresses_second_cancellation(self): """Test that a second cancellation is suppressed. @@ -451,7 +480,7 @@ async def inner(): async def outer(): with LoggingContext("c") as c: try: - await delay_cancellation(defer.ensureDeferred(inner())) + await delay_cancellation(inner()) self.fail("`CancelledError` was not raised") except CancelledError: self.assertEqual(c, current_context()) From a36a38b1ca987c39eb70d9bf5ea4bd6999876a78 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:17:03 +0200 Subject: [PATCH 048/263] Add some example configurations for worker (#12492) Signed-off-by: Dirk Klimpel --- changelog.d/12492.doc | 1 + .../workers/background_worker.yaml | 8 ++++ .../workers/event_persister.yaml | 23 ++++++++++ .../workers/generic_worker.yaml | 3 +- docs/workers.md | 45 +++++++++---------- 5 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12492.doc create mode 100644 docs/systemd-with-workers/workers/background_worker.yaml create mode 100644 docs/systemd-with-workers/workers/event_persister.yaml diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc new file mode 100644 index 000000000000..4a3e2f4f0698 --- /dev/null +++ b/changelog.d/12492.doc @@ -0,0 +1 @@ +Add some example configurations for workers and update architectural diagram. diff --git a/docs/systemd-with-workers/workers/background_worker.yaml b/docs/systemd-with-workers/workers/background_worker.yaml new file mode 100644 index 000000000000..9fbfbda7db94 --- /dev/null +++ b/docs/systemd-with-workers/workers/background_worker.yaml @@ -0,0 +1,8 @@ +worker_app: synapse.app.generic_worker +worker_name: background_worker + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/background-worker-log.yaml diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml new file mode 100644 index 000000000000..9bc6997bad99 --- /dev/null +++ b/docs/systemd-with-workers/workers/event_persister.yaml @@ -0,0 +1,23 @@ +worker_app: synapse.app.generic_worker +worker_name: event_persister1 + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + + # Enable listener if this stream writer handles endpoints for the `typing` or + # `to_device` streams. Uses a different port to the `replication` listener to + # avoid exposing the `replication` listener publicly. + # + #- type: http + # port: 8035 + # resources: + # - names: [client] + +worker_log_config: /etc/matrix-synapse/event-persister-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml index 8561e2cda59c..a82f9c161f81 100644 --- a/docs/systemd-with-workers/workers/generic_worker.yaml +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -1,12 +1,13 @@ worker_app: synapse.app.generic_worker worker_name: generic_worker1 +# The replication listener on the main synapse process. worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - type: http - port: 8011 + port: 8083 resources: - names: [client, federation] diff --git a/docs/workers.md b/docs/workers.md index 858411b15ec3..afdcd785e408 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -138,20 +138,7 @@ as the `listeners` option in the shared config. For example: ```yaml -worker_app: synapse.app.generic_worker -worker_name: worker1 - -# The replication listener on the main synapse process. -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8083 - resources: - - names: [client, federation] - -worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml +{{#include systemd-with-workers/workers/generic_worker.yaml}} ``` ...is a full configuration for a generic worker instance, which will expose a @@ -363,6 +350,12 @@ stream_writers: events: event_persister1 ``` +An example for a stream writer instance: + +```yaml +{{#include systemd-with-workers/workers/event_persister.yaml}} +``` + Some of the streams have associated endpoints which, for maximum efficiency, should be routed to the workers handling that stream. See below for the currently supported streams and the endpoints associated with them: @@ -436,6 +429,12 @@ run_background_tasks_on: background_worker You might also wish to investigate the `update_user_directory` and `media_instance_running_background_jobs` settings. +An example for a dedicated background worker instance: + +```yaml +{{#include systemd-with-workers/workers/background_worker.yaml}} +``` + ### `synapse.app.pusher` Handles sending push notifications to sygnal and email. Doesn't handle any @@ -615,14 +614,14 @@ The following shows an example setup using Redis and a reverse proxy: | Main | | Generic | | Generic | | Event | | Process | | Worker 1 | | Worker 2 | | Persister | +--------------+ +--------------+ +--------------+ +--------------+ - ^ ^ | ^ | | ^ | ^ ^ - | | | | | | | | | | - | | | | | HTTP | | | | | - | +----------+<--|---|---------+ | | | | - | | +-------------|-->+----------+ | - | | | | - | | | | - v v v v -==================================================================== + ^ ^ | ^ | | ^ | | ^ ^ + | | | | | | | | | | | + | | | | | HTTP | | | | | | + | +----------+<--|---|---------+<--|---|---------+ | | + | | +-------------|-->+-------------+ | + | | | | + | | | | + v v v v +====================================================================== Redis pub/sub channel ``` From d9b71410c28320f67e4816257e56ac44492ffa28 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Mon, 25 Apr 2022 15:18:18 +0300 Subject: [PATCH 049/263] Add HAProxy delegation example to docs (#12501) Signed-off-by: Ville Petteri Huh --- changelog.d/12501.doc | 1 + docs/reverse_proxy.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 changelog.d/12501.doc diff --git a/changelog.d/12501.doc b/changelog.d/12501.doc new file mode 100644 index 000000000000..278193a69a31 --- /dev/null +++ b/changelog.d/12501.doc @@ -0,0 +1 @@ +Add HAProxy delegation example with CORS headers to docs. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 5a0c847951a6..69caa8a73ee8 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -206,6 +206,28 @@ backend matrix server matrix 127.0.0.1:8008 ``` + +[Delegation](delegate.md) example: +``` +frontend https + acl matrix-well-known-client-path path /.well-known/matrix/client + acl matrix-well-known-server-path path /.well-known/matrix/server + use_backend matrix-well-known-client if matrix-well-known-client-path + use_backend matrix-well-known-server if matrix-well-known-server-path + +backend matrix-well-known-client + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}' + +backend matrix-well-known-server + http-after-response set-header Access-Control-Allow-Origin "*" + http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + http-request return status 200 content-type application/json string '{"m.server":"matrix.example.com:443"}' +``` + ### Relayd ``` From 185da8f0f2db8e4d502a904942cbd8a6840e27c8 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 25 Apr 2022 08:25:56 -0400 Subject: [PATCH 050/263] Misc. clean-ups to the relations code (#12519) * Corrects some typos / copy & paste errors in tests. * Clarifies docstrings. * Removes an unnecessary method. --- changelog.d/12519.misc | 1 + synapse/events/utils.py | 2 +- synapse/handlers/relations.py | 94 ++++++++++------------------- tests/rest/client/test_relations.py | 74 +++++++++++------------ 4 files changed, 68 insertions(+), 103 deletions(-) create mode 100644 changelog.d/12519.misc diff --git a/changelog.d/12519.misc b/changelog.d/12519.misc new file mode 100644 index 000000000000..9c023d8e3ec6 --- /dev/null +++ b/changelog.d/12519.misc @@ -0,0 +1 @@ +Refactor the relations code for clarity. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 2174b4a0948b..f8d3ba5456a5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -479,9 +479,9 @@ def _inject_bundled_aggregations( Args: event: The event being serialized. time_now: The current time in milliseconds + config: Event serialization config aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. - config: Event serialization config apply_edits: Whether the content of the event should be modified to reflect any replacement in `aggregations.replace`. """ diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index 0be231957750..5efb5612736f 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -256,64 +256,6 @@ async def get_annotations_for_event( return filtered_results - async def _get_bundled_aggregation_for_event( - self, event: EventBase, ignored_users: FrozenSet[str] - ) -> Optional[BundledAggregations]: - """Generate bundled aggregations for an event. - - Note that this does not use a cache, but depends on cached methods. - - Args: - event: The event to calculate bundled aggregations for. - ignored_users: The users ignored by the requesting user. - - Returns: - The bundled aggregations for an event, if bundled aggregations are - enabled and the event can have bundled aggregations. - """ - - # Do not bundle aggregations for an event which represents an edit or an - # annotation. It does not make sense for them to have related events. - relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, (dict, frozendict)): - relation_type = relates_to.get("rel_type") - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): - return None - - event_id = event.event_id - room_id = event.room_id - - # The bundled aggregations to include, a mapping of relation type to a - # type-specific value. Some types include the direct return type here - # while others need more processing during serialization. - aggregations = BundledAggregations() - - annotations = await self.get_annotations_for_event( - event_id, room_id, ignored_users=ignored_users - ) - if annotations: - aggregations.annotations = {"chunk": annotations} - - references, next_token = await self.get_relations_for_event( - event_id, - event, - room_id, - RelationTypes.REFERENCE, - ignored_users=ignored_users, - ) - if references: - aggregations.references = { - "chunk": [{"event_id": event.event_id} for event in references] - } - - if next_token: - aggregations.references["next_batch"] = await next_token.to_string( - self._main_store - ) - - # Store the bundled aggregations in the event metadata for later use. - return aggregations - async def get_threads_for_events( self, event_ids: Collection[str], user_id: str, ignored_users: FrozenSet[str] ) -> Dict[str, _ThreadAggregation]: @@ -435,11 +377,39 @@ async def get_bundled_aggregations( # Fetch other relations per event. for event in events_by_id.values(): - event_result = await self._get_bundled_aggregation_for_event( - event, ignored_users + # Do not bundle aggregations for an event which represents an edit or an + # annotation. It does not make sense for them to have related events. + relates_to = event.content.get("m.relates_to") + if isinstance(relates_to, (dict, frozendict)): + relation_type = relates_to.get("rel_type") + if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): + continue + + annotations = await self.get_annotations_for_event( + event.event_id, event.room_id, ignored_users=ignored_users ) - if event_result: - results[event.event_id] = event_result + if annotations: + results.setdefault( + event.event_id, BundledAggregations() + ).annotations = {"chunk": annotations} + + references, next_token = await self.get_relations_for_event( + event.event_id, + event, + event.room_id, + RelationTypes.REFERENCE, + ignored_users=ignored_users, + ) + if references: + aggregations = results.setdefault(event.event_id, BundledAggregations()) + aggregations.references = { + "chunk": [{"event_id": ev.event_id} for ev in references] + } + + if next_token: + aggregations.references["next_batch"] = await next_token.to_string( + self._main_store + ) # Fetch any edits (but not for redacted events). # diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 65743cdf6777..39667e3225e2 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -560,43 +560,6 @@ def test_edit_reply(self) -> None: {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) - def test_edit_thread(self) -> None: - """Test that editing a thread works.""" - - # Create a thread and edit the last event. - channel = self._send_relation( - RelationTypes.THREAD, - "m.room.message", - content={"msgtype": "m.text", "body": "A threaded reply!"}, - ) - threaded_event_id = channel.json_body["event_id"] - - new_body = {"msgtype": "m.text", "body": "I've been edited!"} - self._send_relation( - RelationTypes.REPLACE, - "m.room.message", - content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, - parent_id=threaded_event_id, - ) - - # Fetch the thread root, to get the bundled aggregation for the thread. - channel = self.make_request( - "GET", - f"/rooms/{self.room}/event/{self.parent_id}", - access_token=self.user_token, - ) - self.assertEqual(200, channel.code, channel.json_body) - - # We expect that the edit message appears in the thread summary in the - # unsigned relations section. - relations_dict = channel.json_body["unsigned"].get("m.relations") - self.assertIn(RelationTypes.THREAD, relations_dict) - - thread_summary = relations_dict[RelationTypes.THREAD] - self.assertIn("latest_event", thread_summary) - latest_event_in_thread = thread_summary["latest_event"] - self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") - def test_edit_edit(self) -> None: """Test that an edit cannot be edited.""" new_body = {"msgtype": "m.text", "body": "Initial edit"} @@ -1047,7 +1010,7 @@ def test_thread(self) -> None: channel = self._send_relation(RelationTypes.THREAD, "m.room.test") thread_2 = channel.json_body["event_id"] - def assert_annotations(bundled_aggregations: JsonDict) -> None: + def assert_thread(bundled_aggregations: JsonDict) -> None: self.assertEqual(2, bundled_aggregations.get("count")) self.assertTrue(bundled_aggregations.get("current_user_participated")) # The latest thread event has some fields that don't matter. @@ -1066,7 +1029,38 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: bundled_aggregations.get("latest_event"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_annotations, 9) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) + + def test_thread_edit_latest_event(self) -> None: + """Test that editing the latest event in a thread works.""" + + # Create a thread and edit the last event. + channel = self._send_relation( + RelationTypes.THREAD, + "m.room.message", + content={"msgtype": "m.text", "body": "A threaded reply!"}, + ) + threaded_event_id = channel.json_body["event_id"] + + new_body = {"msgtype": "m.text", "body": "I've been edited!"} + channel = self._send_relation( + RelationTypes.REPLACE, + "m.room.message", + content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, + parent_id=threaded_event_id, + ) + + # Fetch the thread root, to get the bundled aggregation for the thread. + relations_dict = self._get_bundled_aggregations() + + # We expect that the edit message appears in the thread summary in the + # unsigned relations section. + self.assertIn(RelationTypes.THREAD, relations_dict) + + thread_summary = relations_dict[RelationTypes.THREAD] + self.assertIn("latest_event", thread_summary) + latest_event_in_thread = thread_summary["latest_event"] + self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") def test_aggregation_get_event_for_annotation(self) -> None: """Test that annotations do not get bundled aggregations included @@ -1093,7 +1087,7 @@ def test_aggregation_get_event_for_thread(self) -> None: channel = self._send_relation(RelationTypes.THREAD, "m.room.test") thread_id = channel.json_body["event_id"] - # Annotate the annotation. + # Annotate the thread. self._send_relation( RelationTypes.ANNOTATION, "m.reaction", "a", parent_id=thread_id ) From 8bac3e043541003187e9f16598ff54b0327b9a04 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 25 Apr 2022 13:32:35 +0100 Subject: [PATCH 051/263] disallow-untyped-defs in `docker` and `stubs` directories (#12528) --- changelog.d/12528.misc | 1 + docker/configure_workers_and_start.py | 24 ++++++++-------- docker/start.py | 40 +++++++++++++++------------ stubs/sortedcontainers/sorteddict.pyi | 2 +- stubs/sortedcontainers/sortedlist.pyi | 6 ++-- stubs/sortedcontainers/sortedset.pyi | 2 +- stubs/txredisapi.pyi | 15 ++++++---- 7 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 changelog.d/12528.misc diff --git a/changelog.d/12528.misc b/changelog.d/12528.misc new file mode 100644 index 000000000000..f64b5d24b005 --- /dev/null +++ b/changelog.d/12528.misc @@ -0,0 +1 @@ +Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 23cac18e8dbd..3bda6c300ba8 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Mapping, Set +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set import jinja2 import yaml @@ -201,7 +201,7 @@ # Utility functions -def log(txt: str): +def log(txt: str) -> None: """Log something to the stdout. Args: @@ -210,7 +210,7 @@ def log(txt: str): print(txt) -def error(txt: str): +def error(txt: str) -> NoReturn: """Log something and exit with an error code. Args: @@ -220,7 +220,7 @@ def error(txt: str): sys.exit(2) -def convert(src: str, dst: str, **template_vars): +def convert(src: str, dst: str, **template_vars: object) -> None: """Generate a file from a template Args: @@ -290,7 +290,7 @@ def add_sharding_to_shared_config( shared_config.setdefault("media_instance_running_background_jobs", worker_name) -def generate_base_homeserver_config(): +def generate_base_homeserver_config() -> None: """Starts Synapse and generates a basic homeserver config, which will later be modified for worker support. @@ -302,12 +302,14 @@ def generate_base_homeserver_config(): subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"]) -def generate_worker_files(environ, config_path: str, data_dir: str): +def generate_worker_files( + environ: Mapping[str, str], config_path: str, data_dir: str +) -> None: """Read the desired list of workers from environment variables and generate shared homeserver, nginx and supervisord configs. Args: - environ: _Environ[str] + environ: os.environ instance. config_path: The location of the generated Synapse main worker config file. data_dir: The location of the synapse data directory. Where log and user-facing config files live. @@ -369,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str): nginx_locations = {} # Read the desired worker configuration from the environment - worker_types = environ.get("SYNAPSE_WORKER_TYPES") - if worker_types is None: + worker_types_env = environ.get("SYNAPSE_WORKER_TYPES") + if worker_types_env is None: # No workers, just the main process worker_types = [] else: # Split type names by comma - worker_types = worker_types.split(",") + worker_types = worker_types_env.split(",") # Create the worker configuration directory if it doesn't already exist os.makedirs("/conf/workers", exist_ok=True) @@ -547,7 +549,7 @@ def generate_worker_log_config( return log_config_filepath -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") diff --git a/docker/start.py b/docker/start.py index ac62bbc8baf9..4ac8f034777f 100755 --- a/docker/start.py +++ b/docker/start.py @@ -6,27 +6,28 @@ import platform import subprocess import sys +from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional import jinja2 # Utility functions -def log(txt): +def log(txt: str) -> None: print(txt, file=sys.stderr) -def error(txt): +def error(txt: str) -> NoReturn: log(txt) sys.exit(2) -def convert(src, dst, environ): +def convert(src: str, dst: str, environ: Mapping[str, object]) -> None: """Generate a file from a template Args: - src (str): path to input file - dst (str): path to file to write - environ (dict): environment dictionary, for replacement mappings. + src: path to input file + dst: path to file to write + environ: environment dictionary, for replacement mappings. """ with open(src) as infile: template = infile.read() @@ -35,25 +36,30 @@ def convert(src, dst, environ): outfile.write(rendered) -def generate_config_from_template(config_dir, config_path, environ, ownership): +def generate_config_from_template( + config_dir: str, + config_path: str, + os_environ: Mapping[str, str], + ownership: Optional[str], +) -> None: """Generate a homeserver.yaml from environment variables Args: - config_dir (str): where to put generated config files - config_path (str): where to put the main config file - environ (dict): environment dictionary - ownership (str|None): ":" string which will be used to set + config_dir: where to put generated config files + config_path: where to put the main config file + os_environ: environment mapping + ownership: ":" string which will be used to set ownership of the generated configs. If None, ownership will not change. """ for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): - if v not in environ: + if v not in os_environ: error( "Environment variable '%s' is mandatory when generating a config file." % (v,) ) # populate some params from data files (if they exist, else create new ones) - environ = environ.copy() + environ: Dict[str, Any] = dict(os_environ) secrets = { "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", @@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): subprocess.check_output(args) -def run_generate_config(environ, ownership): +def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None: """Run synapse with a --generate-config param to generate a template config file Args: - environ (dict): env var dict - ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change. + environ: env vars from `os.enrivon`. + ownership: "userid:groupid" arg for chmod. If None, ownership will not change. Never returns. """ @@ -178,7 +184,7 @@ def run_generate_config(environ, ownership): os.execv(sys.executable, args) -def main(args, environ): +def main(args: List[str], environ: MutableMapping[str, str]) -> None: mode = args[1] if len(args) > 1 else "run" # if we were given an explicit user to switch to, do so diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 344d55cce118..e18d617281ab 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -103,7 +103,7 @@ class SortedDict(Dict[_KT, _VT]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_KT]: ... def bisect_left(self, value: _KT) -> int: ... def bisect_right(self, value: _KT) -> int: ... diff --git a/stubs/sortedcontainers/sortedlist.pyi b/stubs/sortedcontainers/sortedlist.pyi index f80a3a72ce04..403897e3919e 100644 --- a/stubs/sortedcontainers/sortedlist.pyi +++ b/stubs/sortedcontainers/sortedlist.pyi @@ -81,7 +81,7 @@ class SortedList(MutableSequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def _islice( self, @@ -153,14 +153,14 @@ class SortedKeyList(SortedList[_T]): maximum: Optional[int] = ..., inclusive: Tuple[bool, bool] = ..., reverse: bool = ..., - ): ... + ) -> Iterator[_T]: ... def irange_key( self, min_key: Optional[Any] = ..., max_key: Optional[Any] = ..., inclusive: Tuple[bool, bool] = ..., reserve: bool = ..., - ): ... + ) -> Iterator[_T]: ... def bisect_left(self, value: _T) -> int: ... def bisect_right(self, value: _T) -> int: ... def bisect(self, value: _T) -> int: ... diff --git a/stubs/sortedcontainers/sortedset.pyi b/stubs/sortedcontainers/sortedset.pyi index f9c290838678..43c860f4221e 100644 --- a/stubs/sortedcontainers/sortedset.pyi +++ b/stubs/sortedcontainers/sortedset.pyi @@ -103,7 +103,7 @@ class SortedSet(MutableSet[_T], Sequence[_T]): self, start: Optional[int] = ..., stop: Optional[int] = ..., - reverse=bool, + reverse: bool = ..., ) -> Iterator[_T]: ... def irange( self, diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi index 2d8ca018fbfc..695a2307c2c5 100644 --- a/stubs/txredisapi.pyi +++ b/stubs/txredisapi.pyi @@ -18,6 +18,8 @@ from typing import Any, List, Optional, Type, Union from twisted.internet import protocol from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IAddress +from twisted.python.failure import Failure class RedisProtocol(protocol.Protocol): def publish(self, channel: str, message: bytes) -> "Deferred[None]": ... @@ -34,11 +36,14 @@ class RedisProtocol(protocol.Protocol): def get(self, key: str) -> "Deferred[Any]": ... class SubscriberProtocol(RedisProtocol): - def __init__(self, *args, **kwargs): ... + def __init__(self, *args: object, **kwargs: object): ... password: Optional[str] - def subscribe(self, channels: Union[str, List[str]]): ... - def connectionMade(self): ... - def connectionLost(self, reason): ... + def subscribe(self, channels: Union[str, List[str]]) -> "Deferred[None]": ... + def connectionMade(self) -> None: ... + # type-ignore: twisted.internet.protocol.Protocol provides a default argument for + # `reason`. txredisapi's LineReceiver Protocol doesn't. But that's fine: it's what's + # actually specified in twisted.internet.interfaces.IProtocol. + def connectionLost(self, reason: Failure) -> None: ... # type: ignore[override] def lazyConnection( host: str = ..., @@ -74,7 +79,7 @@ class RedisFactory(protocol.ReconnectingClientFactory): replyTimeout: Optional[int] = None, convertNumbers: Optional[int] = True, ): ... - def buildProtocol(self, addr) -> RedisProtocol: ... + def buildProtocol(self, addr: IAddress) -> RedisProtocol: ... class SubscriberFactory(RedisFactory): def __init__(self) -> None: ... From 813d728d09cb9e3b66ae49ee894814618db9774b Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Mon, 25 Apr 2022 15:39:15 +0300 Subject: [PATCH 052/263] Correct typo in user_admin_api.md device deletion JSON (#12533) Signed-off-by: Sami Olmari --- changelog.d/12533.doc | 1 + docs/admin_api/user_admin_api.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12533.doc diff --git a/changelog.d/12533.doc b/changelog.d/12533.doc new file mode 100644 index 000000000000..2c15488111cb --- /dev/null +++ b/changelog.d/12533.doc @@ -0,0 +1 @@ +Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 4076fcab65f1..c8794299e790 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users//delete_devices "devices": [ "QBUAZIFURK", "AUIECTSRND" - ], + ] } ``` From 8a87b4435a736cd42454cad7e57b65ec911f01fa Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 25 Apr 2022 19:39:17 +0100 Subject: [PATCH 053/263] Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db` (#12529) Multiple calls to `EventsWorkerStore._get_events_from_cache_or_db` can reuse the same database fetch, which is initiated by the first call. Ensure that cancelling the first call doesn't cancel the other calls sharing the same database fetch. Signed-off-by: Sean Quah --- changelog.d/12529.misc | 1 + .../storage/databases/main/events_worker.py | 83 +++++++----- .../databases/main/test_events_worker.py | 121 +++++++++++++++++- 3 files changed, 169 insertions(+), 36 deletions(-) create mode 100644 changelog.d/12529.misc diff --git a/changelog.d/12529.misc b/changelog.d/12529.misc new file mode 100644 index 000000000000..542710874204 --- /dev/null +++ b/changelog.d/12529.misc @@ -0,0 +1 @@ +Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 6d6e146ff160..c31fc00eaace 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -75,7 +75,7 @@ from synapse.storage.util.sequence import build_sequence_generator from synapse.types import JsonDict, get_domain_from_id from synapse.util import unwrapFirstError -from synapse.util.async_helpers import ObservableDeferred +from synapse.util.async_helpers import ObservableDeferred, delay_cancellation from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches.lrucache import LruCache from synapse.util.iterutils import batch_iter @@ -640,42 +640,57 @@ async def _get_events_from_cache_or_db( missing_events_ids.difference_update(already_fetching_ids) if missing_events_ids: - log_ctx = current_context() - log_ctx.record_event_fetch(len(missing_events_ids)) - - # Add entries to `self._current_event_fetches` for each event we're - # going to pull from the DB. We use a single deferred that resolves - # to all the events we pulled from the DB (this will result in this - # function returning more events than requested, but that can happen - # already due to `_get_events_from_db`). - fetching_deferred: ObservableDeferred[ - Dict[str, EventCacheEntry] - ] = ObservableDeferred(defer.Deferred(), consumeErrors=True) - for event_id in missing_events_ids: - self._current_event_fetches[event_id] = fetching_deferred - - # Note that _get_events_from_db is also responsible for turning db rows - # into FrozenEvents (via _get_event_from_row), which involves seeing if - # the events have been redacted, and if so pulling the redaction event out - # of the database to check it. - # - try: - missing_events = await self._get_events_from_db( - missing_events_ids, - ) - event_entry_map.update(missing_events) - except Exception as e: - with PreserveLoggingContext(): - fetching_deferred.errback(e) - raise e - finally: - # Ensure that we mark these events as no longer being fetched. + async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]: + """Fetches the events in `missing_event_ids` from the database. + + Also creates entries in `self._current_event_fetches` to allow + concurrent `_get_events_from_cache_or_db` calls to reuse the same fetch. + """ + log_ctx = current_context() + log_ctx.record_event_fetch(len(missing_events_ids)) + + # Add entries to `self._current_event_fetches` for each event we're + # going to pull from the DB. We use a single deferred that resolves + # to all the events we pulled from the DB (this will result in this + # function returning more events than requested, but that can happen + # already due to `_get_events_from_db`). + fetching_deferred: ObservableDeferred[ + Dict[str, EventCacheEntry] + ] = ObservableDeferred(defer.Deferred(), consumeErrors=True) for event_id in missing_events_ids: - self._current_event_fetches.pop(event_id, None) + self._current_event_fetches[event_id] = fetching_deferred - with PreserveLoggingContext(): - fetching_deferred.callback(missing_events) + # Note that _get_events_from_db is also responsible for turning db rows + # into FrozenEvents (via _get_event_from_row), which involves seeing if + # the events have been redacted, and if so pulling the redaction event + # out of the database to check it. + # + try: + missing_events = await self._get_events_from_db( + missing_events_ids, + ) + except Exception as e: + with PreserveLoggingContext(): + fetching_deferred.errback(e) + raise e + finally: + # Ensure that we mark these events as no longer being fetched. + for event_id in missing_events_ids: + self._current_event_fetches.pop(event_id, None) + + with PreserveLoggingContext(): + fetching_deferred.callback(missing_events) + + return missing_events + + # We must allow the database fetch to complete in the presence of + # cancellations, since multiple `_get_events_from_cache_or_db` calls can + # reuse the same fetch. + missing_events: Dict[str, EventCacheEntry] = await delay_cancellation( + get_missing_events_from_db() + ) + event_entry_map.update(missing_events) if already_fetching_deferreds: # Wait for the other event requests to finish and add their results diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 1f6a9eb07bfc..bf6374f93d52 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -13,10 +13,11 @@ # limitations under the License. import json from contextlib import contextmanager -from typing import Generator +from typing import Generator, Tuple +from unittest import mock from twisted.enterprise.adbapi import ConnectionPool -from twisted.internet.defer import ensureDeferred +from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.test.proto_helpers import MemoryReactor from synapse.api.room_versions import EventFormatVersions, RoomVersions @@ -281,3 +282,119 @@ def test_recovery(self) -> None: # This next event fetch should succeed self.get_success(self.store.get_event(self.event_ids[0])) + + +class GetEventCancellationTestCase(unittest.HomeserverTestCase): + """Test cancellation of `get_event` calls.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer): + self.store: EventsWorkerStore = hs.get_datastores().main + + self.user = self.register_user("user", "pass") + self.token = self.login(self.user, "pass") + + self.room = self.helper.create_room_as(self.user, tok=self.token) + + res = self.helper.send(self.room, tok=self.token) + self.event_id = res["event_id"] + + # Reset the event cache so the tests start with it empty + self.store._get_event_cache.clear() + + @contextmanager + def blocking_get_event_calls( + self, + ) -> Generator[ + Tuple["Deferred[None]", "Deferred[None]", "Deferred[None]"], None, None + ]: + """Starts two concurrent `get_event` calls for the same event. + + Both `get_event` calls will use the same database fetch, which will be blocked + at the time this function returns. + + Returns: + A tuple containing: + * A `Deferred` that unblocks the database fetch. + * A cancellable `Deferred` for the first `get_event` call. + * A cancellable `Deferred` for the second `get_event` call. + """ + # Patch `DatabasePool.runWithConnection` to block. + unblock: "Deferred[None]" = Deferred() + original_runWithConnection = self.store.db_pool.runWithConnection + + async def runWithConnection(*args, **kwargs): + await unblock + return await original_runWithConnection(*args, **kwargs) + + with mock.patch.object( + self.store.db_pool, + "runWithConnection", + new=runWithConnection, + ): + ctx1 = LoggingContext("get_event1") + ctx2 = LoggingContext("get_event2") + + async def get_event(ctx: LoggingContext) -> None: + with ctx: + await self.store.get_event(self.event_id) + + get_event1 = ensureDeferred(get_event(ctx1)) + get_event2 = ensureDeferred(get_event(ctx2)) + + # Both `get_event` calls ought to be blocked. + self.assertNoResult(get_event1) + self.assertNoResult(get_event2) + + yield unblock, get_event1, get_event2 + + # Confirm that the two `get_event` calls shared the same database fetch. + self.assertEqual(ctx1.get_resource_usage().evt_db_fetch_count, 1) + self.assertEqual(ctx2.get_resource_usage().evt_db_fetch_count, 0) + + def test_first_get_event_cancelled(self): + """Test cancellation of the first `get_event` call sharing a database fetch. + + The first `get_event` call is the one which initiates the fetch. We expect the + fetch to complete despite the cancellation. Furthermore, the first `get_event` + call must not abort before the fetch is complete, otherwise the fetch will be + using a finished logging context. + """ + with self.blocking_get_event_calls() as (unblock, get_event1, get_event2): + # Cancel the first `get_event` call. + get_event1.cancel() + # The first `get_event` call must not abort immediately, otherwise its + # logging context will be finished while it is still in use by the database + # fetch. + self.assertNoResult(get_event1) + # The second `get_event` call must not be cancelled. + self.assertNoResult(get_event2) + + # Unblock the database fetch. + unblock.callback(None) + # A `CancelledError` should be raised out of the first `get_event` call. + exc = self.get_failure(get_event1, CancelledError).value + self.assertIsInstance(exc, CancelledError) + # The second `get_event` call should complete successfully. + self.get_success(get_event2) + + def test_second_get_event_cancelled(self): + """Test cancellation of the second `get_event` call sharing a database fetch.""" + with self.blocking_get_event_calls() as (unblock, get_event1, get_event2): + # Cancel the second `get_event` call. + get_event2.cancel() + # The first `get_event` call must not be cancelled. + self.assertNoResult(get_event1) + # The second `get_event` call gets cancelled immediately. + exc = self.get_failure(get_event2, CancelledError).value + self.assertIsInstance(exc, CancelledError) + + # Unblock the database fetch. + unblock.callback(None) + # The first `get_event` call should complete successfully. + self.get_success(get_event1) From e75c7e3b6ddc25b76b4eabf10eca1e923048ae43 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 25 Apr 2022 11:43:59 -0700 Subject: [PATCH 054/263] Add a table of contents to config manual (#12527) * Update config_documentation.md --- changelog.d/12527.doc | 2 + .../configuration/config_documentation.md | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12527.doc diff --git a/changelog.d/12527.doc b/changelog.d/12527.doc new file mode 100644 index 000000000000..e6907321e769 --- /dev/null +++ b/changelog.d/12527.doc @@ -0,0 +1,2 @@ +Add an index to the configuration manual. + diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 9c864af6ec32..968b0fbfaff4 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -64,7 +64,49 @@ apply if you want your config file to be read properly. A few helpful things to In addition, each setting has an example of its usage, with the proper indentation shown. - +## Contents +[Modules](#modules) + +[Server](#server) + +[Homeserver Blocking](#homeserver-blocking) + +[TLS](#tls) + +[Federation](#federation) + +[Caching](#caching) + +[Database](#database) + +[Logging](#logging) + +[Ratelimiting](#ratelimiting) + +[Media Store](#media-store) + +[Captcha](#captcha) + +[TURN](#turn) + +[Registration](#registration) + +[API Configuration](#api-configuration) + +[Signing Keys](#signing-keys) + +[Single Sign On Integration](#single-sign-on-integration) + +[Push](#push) + +[Rooms](#rooms) + +[Opentracing](#opentracing) + +[Workers](#workers) + +[Background Updates](#background-updates) + ## Modules Server admins can expand Synapse's functionality with external modules. @@ -3409,4 +3451,4 @@ background_updates: sleep_duration_ms: 300 min_batch_size: 10 default_batch_size: 50 -``` \ No newline at end of file +``` From 17d99f758a768b886842cf496ff236fbe3829236 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 26 Apr 2022 10:27:11 +0100 Subject: [PATCH 055/263] Optimise backfill calculation (#12522) Try to avoid an OOM by checking fewer extremities. Generally this is a big rewrite of _maybe_backfill, to try and fix some of the TODOs and other problems in it. It's best reviewed commit-by-commit. --- changelog.d/12522.bugfix | 1 + synapse/handlers/federation.py | 234 +++++++++++------- synapse/handlers/room_batch.py | 2 +- .../databases/main/event_federation.py | 30 ++- synapse/visibility.py | 7 + 5 files changed, 168 insertions(+), 106 deletions(-) create mode 100644 changelog.d/12522.bugfix diff --git a/changelog.d/12522.bugfix b/changelog.d/12522.bugfix new file mode 100644 index 000000000000..2220f05ceb75 --- /dev/null +++ b/changelog.d/12522.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1434e99056cb..d2ba70a814d0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,4 +1,4 @@ -# Copyright 2014-2021 The Matrix.org Foundation C.I.C. +# Copyright 2014-2022 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,10 +15,14 @@ """Contains handlers for federation events.""" +import enum +import itertools import logging +from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union +import attr from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from unpaddedbase64 import decode_base64 @@ -92,6 +96,24 @@ def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]: return sorted(joined_domains.items(), key=lambda d: d[1]) +class _BackfillPointType(Enum): + # a regular backwards extremity (ie, an event which we don't yet have, but which + # is referred to by other events in the DAG) + BACKWARDS_EXTREMITY = enum.auto() + + # an MSC2716 "insertion event" + INSERTION_PONT = enum.auto() + + +@attr.s(slots=True, auto_attribs=True, frozen=True) +class _BackfillPoint: + """A potential point we might backfill from""" + + event_id: str + depth: int + type: _BackfillPointType + + class FederationHandler: """Handles general incoming federation requests @@ -157,89 +179,51 @@ async def maybe_backfill( async def _maybe_backfill_inner( self, room_id: str, current_depth: int, limit: int ) -> bool: - oldest_events_with_depth = ( - await self.store.get_oldest_event_ids_with_depth_in_room(room_id) - ) + backwards_extremities = [ + _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY) + for event_id, depth in await self.store.get_oldest_event_ids_with_depth_in_room( + room_id + ) + ] - insertion_events_to_be_backfilled: Dict[str, int] = {} + insertion_events_to_be_backfilled: List[_BackfillPoint] = [] if self.hs.config.experimental.msc2716_enabled: - insertion_events_to_be_backfilled = ( - await self.store.get_insertion_event_backward_extremities_in_room( + insertion_events_to_be_backfilled = [ + _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT) + for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room( room_id ) - ) + ] logger.debug( - "_maybe_backfill_inner: extremities oldest_events_with_depth=%s insertion_events_to_be_backfilled=%s", - oldest_events_with_depth, + "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s", + backwards_extremities, insertion_events_to_be_backfilled, ) - if not oldest_events_with_depth and not insertion_events_to_be_backfilled: + if not backwards_extremities and not insertion_events_to_be_backfilled: logger.debug("Not backfilling as no extremeties found.") return False - # We only want to paginate if we can actually see the events we'll get, - # as otherwise we'll just spend a lot of resources to get redacted - # events. - # - # We do this by filtering all the backwards extremities and seeing if - # any remain. Given we don't have the extremity events themselves, we - # need to actually check the events that reference them. - # - # *Note*: the spec wants us to keep backfilling until we reach the start - # of the room in case we are allowed to see some of the history. However - # in practice that causes more issues than its worth, as a) its - # relatively rare for there to be any visible history and b) even when - # there is its often sufficiently long ago that clients would stop - # attempting to paginate before backfill reached the visible history. - # - # TODO: If we do do a backfill then we should filter the backwards - # extremities to only include those that point to visible portions of - # history. - # - # TODO: Correctly handle the case where we are allowed to see the - # forward event but not the backward extremity, e.g. in the case of - # initial join of the server where we are allowed to see the join - # event but not anything before it. This would require looking at the - # state *before* the event, ignoring the special casing certain event - # types have. - - forward_event_ids = await self.store.get_successor_events( - list(oldest_events_with_depth) - ) - - extremities_events = await self.store.get_events( - forward_event_ids, - redact_behaviour=EventRedactBehaviour.AS_IS, - get_prev_content=False, + # we now have a list of potential places to backpaginate from. We prefer to + # start with the most recent (ie, max depth), so let's sort the list. + sorted_backfill_points: List[_BackfillPoint] = sorted( + itertools.chain( + backwards_extremities, + insertion_events_to_be_backfilled, + ), + key=lambda e: -int(e.depth), ) - # We set `check_history_visibility_only` as we might otherwise get false - # positives from users having been erased. - filtered_extremities = await filter_events_for_server( - self.storage, - self.server_name, - list(extremities_events.values()), - redact=False, - check_history_visibility_only=True, - ) logger.debug( - "_maybe_backfill_inner: filtered_extremities %s", filtered_extremities + "_maybe_backfill_inner: room_id: %s: current_depth: %s, limit: %s, " + "backfill points (%d): %s", + room_id, + current_depth, + limit, + len(sorted_backfill_points), + sorted_backfill_points, ) - if not filtered_extremities and not insertion_events_to_be_backfilled: - return False - - extremities = { - **oldest_events_with_depth, - # TODO: insertion_events_to_be_backfilled is currently skipping the filtered_extremities checks - **insertion_events_to_be_backfilled, - } - - # Check if we reached a point where we should start backfilling. - sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1])) - max_depth = sorted_extremeties_tuple[0][1] - # If we're approaching an extremity we trigger a backfill, otherwise we # no-op. # @@ -249,6 +233,11 @@ async def _maybe_backfill_inner( # chose more than one times the limit in case of failure, but choosing a # much larger factor will result in triggering a backfill request much # earlier than necessary. + # + # XXX: shouldn't we do this *after* the filter by depth below? Again, we don't + # care about events that have happened after our current position. + # + max_depth = sorted_backfill_points[0].depth if current_depth - 2 * limit > max_depth: logger.debug( "Not backfilling as we don't need to. %d < %d - 2 * %d", @@ -265,31 +254,98 @@ async def _maybe_backfill_inner( # 2. we have likely previously tried and failed to backfill from that # extremity, so to avoid getting "stuck" requesting the same # backfill repeatedly we drop those extremities. - filtered_sorted_extremeties_tuple = [ - t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth - ] - - logger.debug( - "room_id: %s, backfill: current_depth: %s, limit: %s, max_depth: %s, extrems (%d): %s filtered_sorted_extremeties_tuple: %s", - room_id, - current_depth, - limit, - max_depth, - len(sorted_extremeties_tuple), - sorted_extremeties_tuple, - filtered_sorted_extremeties_tuple, - ) - + # # However, we need to check that the filtered extremities are non-empty. # If they are empty then either we can a) bail or b) still attempt to # backfill. We opt to try backfilling anyway just in case we do get # relevant events. - if filtered_sorted_extremeties_tuple: - sorted_extremeties_tuple = filtered_sorted_extremeties_tuple + # + filtered_sorted_backfill_points = [ + t for t in sorted_backfill_points if t.depth <= current_depth + ] + if filtered_sorted_backfill_points: + logger.debug( + "_maybe_backfill_inner: backfill points before current depth: %s", + filtered_sorted_backfill_points, + ) + sorted_backfill_points = filtered_sorted_backfill_points + else: + logger.debug( + "_maybe_backfill_inner: all backfill points are *after* current depth. Backfilling anyway." + ) + + # For performance's sake, we only want to paginate from a particular extremity + # if we can actually see the events we'll get. Otherwise, we'd just spend a lot + # of resources to get redacted events. We check each extremity in turn and + # ignore those which users on our server wouldn't be able to see. + # + # Additionally, we limit ourselves to backfilling from at most 5 extremities, + # for two reasons: + # + # - The check which determines if we can see an extremity's events can be + # expensive (we load the full state for the room at each of the backfill + # points, or (worse) their successors) + # - We want to avoid the server-server API request URI becoming too long. + # + # *Note*: the spec wants us to keep backfilling until we reach the start + # of the room in case we are allowed to see some of the history. However, + # in practice that causes more issues than its worth, as (a) it's + # relatively rare for there to be any visible history and (b) even when + # there is it's often sufficiently long ago that clients would stop + # attempting to paginate before backfill reached the visible history. - # We don't want to specify too many extremities as it causes the backfill - # request URI to be too long. - extremities = dict(sorted_extremeties_tuple[:5]) + extremities_to_request: List[str] = [] + for bp in sorted_backfill_points: + if len(extremities_to_request) >= 5: + break + + # For regular backwards extremities, we don't have the extremity events + # themselves, so we need to actually check the events that reference them - + # their "successor" events. + # + # TODO: Correctly handle the case where we are allowed to see the + # successor event but not the backward extremity, e.g. in the case of + # initial join of the server where we are allowed to see the join + # event but not anything before it. This would require looking at the + # state *before* the event, ignoring the special casing certain event + # types have. + if bp.type == _BackfillPointType.INSERTION_PONT: + event_ids_to_check = [bp.event_id] + else: + event_ids_to_check = await self.store.get_successor_events(bp.event_id) + + events_to_check = await self.store.get_events_as_list( + event_ids_to_check, + redact_behaviour=EventRedactBehaviour.AS_IS, + get_prev_content=False, + ) + + # We set `check_history_visibility_only` as we might otherwise get false + # positives from users having been erased. + filtered_extremities = await filter_events_for_server( + self.storage, + self.server_name, + events_to_check, + redact=False, + check_history_visibility_only=True, + ) + if filtered_extremities: + extremities_to_request.append(bp.event_id) + else: + logger.debug( + "_maybe_backfill_inner: skipping extremity %s as it would not be visible", + bp, + ) + + if not extremities_to_request: + logger.debug( + "_maybe_backfill_inner: found no extremities which would be visible" + ) + return False + + logger.debug( + "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request + ) # Now we need to decide which hosts to hit first. @@ -309,7 +365,7 @@ async def try_backfill(domains: List[str]) -> bool: for dom in domains: try: await self._federation_event_handler.backfill( - dom, room_id, limit=100, extremities=extremities + dom, room_id, limit=100, extremities=extremities_to_request ) # If this succeeded then we probably already have the # appropriate stuff. diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py index 78e299d3a5c5..29de7e5bed10 100644 --- a/synapse/handlers/room_batch.py +++ b/synapse/handlers/room_batch.py @@ -54,7 +54,7 @@ async def inherit_depth_from_prev_ids(self, prev_event_ids: List[str]) -> int: # it has a larger `depth` but before the successor event because the `stream_ordering` # is negative before the successor event. successor_event_ids = await self.store.get_successor_events( - [most_recent_prev_event_id] + most_recent_prev_event_id ) # If we can't find any successor events, then it's a forward extremity of diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 634e19e035d2..471022470843 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -695,7 +695,9 @@ def _get_auth_chain_difference_txn( # Return all events where not all sets can reach them. return {eid for eid, n in event_to_missing_sets.items() if n} - async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, int]: + async def get_oldest_event_ids_with_depth_in_room( + self, room_id + ) -> List[Tuple[str, int]]: """Gets the oldest events(backwards extremities) in the room along with the aproximate depth. @@ -708,7 +710,7 @@ async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, in room_id: Room where we want to find the oldest events Returns: - Map from event_id to depth + List of (event_id, depth) tuples """ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): @@ -741,7 +743,7 @@ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): txn.execute(sql, (room_id, False)) - return dict(txn) + return txn.fetchall() return await self.db_pool.runInteraction( "get_oldest_event_ids_with_depth_in_room", @@ -751,7 +753,7 @@ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): async def get_insertion_event_backward_extremities_in_room( self, room_id - ) -> Dict[str, int]: + ) -> List[Tuple[str, int]]: """Get the insertion events we know about that we haven't backfilled yet. We use this function so that we can compare and see if someones current @@ -763,7 +765,7 @@ async def get_insertion_event_backward_extremities_in_room( room_id: Room where we want to find the oldest events Returns: - Map from event_id to depth + List of (event_id, depth) tuples """ def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): @@ -778,8 +780,7 @@ def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): """ txn.execute(sql, (room_id,)) - - return dict(txn) + return txn.fetchall() return await self.db_pool.runInteraction( "get_insertion_event_backward_extremities_in_room", @@ -1295,22 +1296,19 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events, limi event_results.reverse() return event_results - async def get_successor_events(self, event_ids: Iterable[str]) -> List[str]: - """Fetch all events that have the given events as a prev event + async def get_successor_events(self, event_id: str) -> List[str]: + """Fetch all events that have the given event as a prev event Args: - event_ids: The events to use as the previous events. + event_id: The event to search for as a prev_event. """ - rows = await self.db_pool.simple_select_many_batch( + return await self.db_pool.simple_select_onecol( table="event_edges", - column="prev_event_id", - iterable=event_ids, - retcols=("event_id",), + keyvalues={"prev_event_id": event_id}, + retcol="event_id", desc="get_successor_events", ) - return [row["event_id"] for row in rows] - @wrap_as_background_process("delete_old_forward_extrem_cache") async def _delete_old_forward_extrem_cache(self) -> None: def _delete_old_forward_extrem_cache_txn(txn): diff --git a/synapse/visibility.py b/synapse/visibility.py index 250f0735975b..de6d2ffc526a 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -419,6 +419,13 @@ async def _event_to_memberships( return {} # for each event, get the event_ids of the membership state at those events. + # + # TODO: this means that we request the entire membership list. If there are only + # one or two users on this server, and the room is huge, this is very wasteful + # (it means more db work, and churns the *stateGroupMembersCache*). + # It might be that we could extend StateFilter to specify "give me keys matching + # *:", to avoid this. + event_to_state_ids = await storage.state.get_state_ids_for_events( frozenset(e.event_id for e in events), state_filter=StateFilter.from_types(types=((EventTypes.Member, None),)), From 99ab45423a0677ea816eb4e81c28f46dd22fc76c Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 26 Apr 2022 02:34:59 -0700 Subject: [PATCH 056/263] build debian package for jammy jellyfish (#12543) --- changelog.d/12543.misc | 1 + scripts-dev/build_debian_packages.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/12543.misc diff --git a/changelog.d/12543.misc b/changelog.d/12543.misc new file mode 100644 index 000000000000..eed7a6973ca9 --- /dev/null +++ b/changelog.d/12543.misc @@ -0,0 +1 @@ +Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index 7ff96a1ee6fe..e3e68786867e 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -26,6 +26,7 @@ "debian:sid", "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) "ubuntu:impish", # 21.10 (EOL 2022-07) + "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) ) DESC = """\ From 730fcda5463144f09ff1b34198c678021469ee28 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 10:56:12 +0100 Subject: [PATCH 057/263] Update release script to be poetry-aware Poetry now manages the project version in pyproject.toml. --- scripts-dev/release.py | 68 +++++++++++++----------------------------- 1 file changed, 21 insertions(+), 47 deletions(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 6f7cf6888d36..725e8f0479c5 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -25,13 +25,13 @@ import urllib.request from os import path from tempfile import TemporaryDirectory -from typing import List, Optional, Tuple +from typing import List, Optional import attr import click import commonmark import git -import redbaron + from click.exceptions import ClickException from github import Github from packaging import version @@ -100,7 +100,7 @@ def prepare(): repo.remote().fetch() # Get the current version and AST from root Synapse module. - current_version, parsed_synapse_ast, version_node = parse_version_from_module() + current_version = get_package_version() # Figure out what sort of release we're doing and calcuate the new version. rc = click.confirm("RC", default=True) @@ -162,7 +162,7 @@ def prepare(): click.get_current_context().abort() # Switch to the release branch. - parsed_new_version = version.parse(new_version) + parsed_new_version: version.Version = version.parse(new_version) # We assume for debian changelogs that we only do RCs or full releases. assert not parsed_new_version.is_devrelease @@ -207,17 +207,15 @@ def prepare(): # Create the new release branch release_branch = repo.create_head(release_branch_name, commit=base_branch) - # Switch to the release branch and ensure its up to date. + # Switch to the release branch and ensure it's up to date. repo.git.checkout(release_branch_name) update_branch(repo) - # Update the `__version__` variable and write it back to the file. - version_node.value = '"' + new_version + '"' - with open("synapse/__init__.py", "w") as f: - f.write(parsed_synapse_ast.dumps()) + # Update the version specified in pyproject.toml. + subprocess.check_output(["poetry", "version", new_version]) # Generate changelogs. - generate_and_write_changelog(current_version) + generate_and_write_changelog(current_version, new_version) # Generate debian changelogs if parsed_new_version.pre is not None: @@ -284,7 +282,7 @@ def tag(gh_token: Optional[str]): repo.remote().fetch() # Find out the version and tag name. - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" # Check we haven't released this version. @@ -362,7 +360,7 @@ def publish(gh_token: str): if repo.is_dirty(): raise click.ClickException("Uncommitted changes exist.") - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" if not click.confirm(f"Publish {tag_name}?", default=True): @@ -396,7 +394,7 @@ def publish(gh_token: str): def upload(): """Upload release to pypi.""" - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" pypi_asset_names = [ @@ -424,7 +422,7 @@ def upload(): def announce(): """Generate markdown to announce the release.""" - current_version, _, _ = parse_version_from_module() + current_version = get_package_version() tag_name = f"v{current_version}" click.echo( @@ -455,37 +453,11 @@ def announce(): ) -def parse_version_from_module() -> Tuple[ - version.Version, redbaron.RedBaron, redbaron.Node -]: - # Parse the AST and load the `__version__` node so that we can edit it - # later. - with open("synapse/__init__.py") as f: - red = redbaron.RedBaron(f.read()) - - version_node = None - for node in red: - if node.type != "assignment": - continue - - if node.target.type != "name": - continue - - if node.target.value != "__version__": - continue - - version_node = node - break - - if not version_node: - print("Failed to find '__version__' definition in synapse/__init__.py") - sys.exit(1) - - # Parse the current version. - current_version = version.parse(version_node.value.value.strip('"')) - assert isinstance(current_version, version.Version) - - return current_version, red, version_node +def get_package_version() -> version.Version: + version_string = subprocess.check_output(["poetry", "version", "--short"]).decode( + "utf-8" + ) + return version.Version(version_string) def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]: @@ -565,11 +537,13 @@ class VersionSection: return "\n".join(version_changelog) -def generate_and_write_changelog(current_version: version.Version): +def generate_and_write_changelog(current_version: version.Version, new_version: str): # We do this by getting a draft so that we can edit it before writing to the # changelog. result = run_until_successful( - "python3 -m towncrier --draft", shell=True, capture_output=True + f"python3 -m towncrier build --draft --version {new_version}", + shell=True, + capture_output=True, ) new_changes = result.stdout.decode("utf-8") new_changes = new_changes.replace( From 7c063da25c80906f0637f907d0c744cd778c682d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:14:41 +0100 Subject: [PATCH 058/263] Temporarily lower debian changelog version number This seems to make dch happy when we prepare the release. --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 05e6bd75a7ff..7727536a74ce 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.57.1+nmu1) UNRELEASED; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. From 30db7fdb911ead29052476db2797def29baefa1b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:15:33 +0100 Subject: [PATCH 059/263] 1.58.0rc1 --- CHANGES.md | 77 +++++++++++++++++++++++++++++++++++++++ changelog.d/11398.feature | 1 - changelog.d/12213.bugfix | 1 - changelog.d/12319.bugfix | 1 - changelog.d/12337.feature | 1 - changelog.d/12340.doc | 1 - changelog.d/12344.removal | 1 - changelog.d/12365.feature | 1 - changelog.d/12368.doc | 1 - changelog.d/12382.removal | 1 - changelog.d/12394.misc | 1 - changelog.d/12395.misc | 1 - changelog.d/12399.misc | 1 - changelog.d/12425.misc | 1 - changelog.d/12427.feature | 1 - changelog.d/12434.misc | 1 - changelog.d/12438.misc | 1 - changelog.d/12441.misc | 1 - changelog.d/12445.misc | 1 - changelog.d/12449.misc | 1 - changelog.d/12450.misc | 1 - changelog.d/12451.doc | 1 - changelog.d/12454.misc | 1 - changelog.d/12455.misc | 1 - changelog.d/12457.doc | 1 - changelog.d/12464.misc | 1 - changelog.d/12465.feature | 1 - changelog.d/12466.misc | 1 - changelog.d/12467.misc | 1 - changelog.d/12468.misc | 1 - changelog.d/12472.misc | 1 - changelog.d/12474.misc | 1 - changelog.d/12475.doc | 1 - changelog.d/12476.bugfix | 1 - changelog.d/12478.misc | 1 - changelog.d/12483.misc | 1 - changelog.d/12492.doc | 1 - changelog.d/12495.doc | 1 - changelog.d/12496.bugfix | 1 - changelog.d/12497.misc | 1 - changelog.d/12501.doc | 1 - changelog.d/12510.bugfix | 1 - changelog.d/12511.misc | 1 - changelog.d/12514.misc | 1 - changelog.d/12519.misc | 1 - changelog.d/12520.bugfix | 1 - changelog.d/12522.bugfix | 1 - changelog.d/12527.doc | 2 - changelog.d/12528.misc | 1 - changelog.d/12529.misc | 1 - changelog.d/12533.doc | 1 - changelog.d/12543.misc | 1 - debian/changelog | 5 ++- pyproject.toml | 2 +- 54 files changed, 81 insertions(+), 55 deletions(-) delete mode 100644 changelog.d/11398.feature delete mode 100644 changelog.d/12213.bugfix delete mode 100644 changelog.d/12319.bugfix delete mode 100644 changelog.d/12337.feature delete mode 100644 changelog.d/12340.doc delete mode 100644 changelog.d/12344.removal delete mode 100644 changelog.d/12365.feature delete mode 100644 changelog.d/12368.doc delete mode 100644 changelog.d/12382.removal delete mode 100644 changelog.d/12394.misc delete mode 100644 changelog.d/12395.misc delete mode 100644 changelog.d/12399.misc delete mode 100644 changelog.d/12425.misc delete mode 100644 changelog.d/12427.feature delete mode 100644 changelog.d/12434.misc delete mode 100644 changelog.d/12438.misc delete mode 100644 changelog.d/12441.misc delete mode 100644 changelog.d/12445.misc delete mode 100644 changelog.d/12449.misc delete mode 100644 changelog.d/12450.misc delete mode 100644 changelog.d/12451.doc delete mode 100644 changelog.d/12454.misc delete mode 100644 changelog.d/12455.misc delete mode 100644 changelog.d/12457.doc delete mode 100644 changelog.d/12464.misc delete mode 100644 changelog.d/12465.feature delete mode 100644 changelog.d/12466.misc delete mode 100644 changelog.d/12467.misc delete mode 100644 changelog.d/12468.misc delete mode 100644 changelog.d/12472.misc delete mode 100644 changelog.d/12474.misc delete mode 100644 changelog.d/12475.doc delete mode 100644 changelog.d/12476.bugfix delete mode 100644 changelog.d/12478.misc delete mode 100644 changelog.d/12483.misc delete mode 100644 changelog.d/12492.doc delete mode 100644 changelog.d/12495.doc delete mode 100644 changelog.d/12496.bugfix delete mode 100644 changelog.d/12497.misc delete mode 100644 changelog.d/12501.doc delete mode 100644 changelog.d/12510.bugfix delete mode 100644 changelog.d/12511.misc delete mode 100644 changelog.d/12514.misc delete mode 100644 changelog.d/12519.misc delete mode 100644 changelog.d/12520.bugfix delete mode 100644 changelog.d/12522.bugfix delete mode 100644 changelog.d/12527.doc delete mode 100644 changelog.d/12528.misc delete mode 100644 changelog.d/12529.misc delete mode 100644 changelog.d/12533.doc delete mode 100644 changelog.d/12543.misc diff --git a/CHANGES.md b/CHANGES.md index a7d2529b557e..95f0b5ad3720 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,80 @@ +Synapse 1.58.0rc1 (2022-04-26) +============================== + +Features +-------- + +- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) +- Use poetry to manage Synapse's dependencies. ([\#12337](https://github.com/matrix-org/synapse/issues/12337)) +- Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) +- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) + + +Bugfixes +-------- + +- Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213)) +- Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319)) +- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) +- Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) +- Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) +- Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) +- Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522)) + + +Improved Documentation +---------------------- + +- Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340)) +- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368)) +- Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) +- Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) +- Strongly recommend `poetry` for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) +- Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) +- Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) +- Add an index to the configuration manual. ([\#12527](https://github.com/matrix-org/synapse/issues/12527)) +- Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533)) + + +Deprecations and Removals +------------------------- + +- The groups/communities feature in Synapse has been disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) +- Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) + + +Internal Changes +---------------- + +- Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) +- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) +- Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) +- Run twisted trunk CI job in the locked poetry environment. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)) +- Run lints under poetry in CI, and remove corresponding tox lint jobs. ([\#12434](https://github.com/matrix-org/synapse/issues/12434)) +- Run "main" trial tests under `poetry`. ([\#12438](https://github.com/matrix-org/synapse/issues/12438)) +- Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). ([\#12441](https://github.com/matrix-org/synapse/issues/12441)) +- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) +- Use `poetry` to manage the virtualenv in debian packages. ([\#12449](https://github.com/matrix-org/synapse/issues/12449)) +- Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) +- Limit length of device_id to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) +- Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455)) +- Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464)) +- Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466)) +- complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467)) +- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) +- Add a CI job which tests Synapse against the latest version of all dependencies. ([\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474)) +- Use poetry-core instead of setuptools to build wheels. ([\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514)) +- Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483)) +- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) +- Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511)) +- Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519)) +- Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528)) +- Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529)) +- Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) + + Synapse 1.57.1 (2022-04-20) =========================== diff --git a/changelog.d/11398.feature b/changelog.d/11398.feature deleted file mode 100644 index a910f4da1496..000000000000 --- a/changelog.d/11398.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. diff --git a/changelog.d/12213.bugfix b/changelog.d/12213.bugfix deleted file mode 100644 index 9278e3a9c163..000000000000 --- a/changelog.d/12213.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent a sync request from removing a user's busy presence status. diff --git a/changelog.d/12319.bugfix b/changelog.d/12319.bugfix deleted file mode 100644 index a50191feaaaf..000000000000 --- a/changelog.d/12319.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. diff --git a/changelog.d/12337.feature b/changelog.d/12337.feature deleted file mode 100644 index 6c4444c70722..000000000000 --- a/changelog.d/12337.feature +++ /dev/null @@ -1 +0,0 @@ -Use poetry to manage Synapse's dependencies. \ No newline at end of file diff --git a/changelog.d/12340.doc b/changelog.d/12340.doc deleted file mode 100644 index 8354f2259e0c..000000000000 --- a/changelog.d/12340.doc +++ /dev/null @@ -1 +0,0 @@ -Fix rendering of the documentation site when using the 'print' feature. diff --git a/changelog.d/12344.removal b/changelog.d/12344.removal deleted file mode 100644 index ecefa76d8ea5..000000000000 --- a/changelog.d/12344.removal +++ /dev/null @@ -1 +0,0 @@ -The groups/communities feature in Synapse has been disabled by default. diff --git a/changelog.d/12365.feature b/changelog.d/12365.feature deleted file mode 100644 index 642dea966c44..000000000000 --- a/changelog.d/12365.feature +++ /dev/null @@ -1 +0,0 @@ -Enable processing of device list updates asynchronously. diff --git a/changelog.d/12368.doc b/changelog.d/12368.doc deleted file mode 100644 index 62e4cb2c7ef5..000000000000 --- a/changelog.d/12368.doc +++ /dev/null @@ -1 +0,0 @@ -Add a manual documenting config file options. \ No newline at end of file diff --git a/changelog.d/12382.removal b/changelog.d/12382.removal deleted file mode 100644 index eb91186340f7..000000000000 --- a/changelog.d/12382.removal +++ /dev/null @@ -1 +0,0 @@ -Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). diff --git a/changelog.d/12394.misc b/changelog.d/12394.misc deleted file mode 100644 index 69109fcc37d3..000000000000 --- a/changelog.d/12394.misc +++ /dev/null @@ -1 +0,0 @@ -Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. diff --git a/changelog.d/12395.misc b/changelog.d/12395.misc deleted file mode 100644 index 0a2123b2942a..000000000000 --- a/changelog.d/12395.misc +++ /dev/null @@ -1 +0,0 @@ -Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). diff --git a/changelog.d/12399.misc b/changelog.d/12399.misc deleted file mode 100644 index cd2e09626d35..000000000000 --- a/changelog.d/12399.misc +++ /dev/null @@ -1 +0,0 @@ -Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. diff --git a/changelog.d/12425.misc b/changelog.d/12425.misc deleted file mode 100644 index 3b076be0bd52..000000000000 --- a/changelog.d/12425.misc +++ /dev/null @@ -1 +0,0 @@ -Run twisted trunk CI job in the locked poetry environment. diff --git a/changelog.d/12427.feature b/changelog.d/12427.feature deleted file mode 100644 index e6913c8c0973..000000000000 --- a/changelog.d/12427.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. diff --git a/changelog.d/12434.misc b/changelog.d/12434.misc deleted file mode 100644 index 88dab428d245..000000000000 --- a/changelog.d/12434.misc +++ /dev/null @@ -1 +0,0 @@ -Run lints under poetry in CI, and remove corresponding tox lint jobs. diff --git a/changelog.d/12438.misc b/changelog.d/12438.misc deleted file mode 100644 index f2c07a56da14..000000000000 --- a/changelog.d/12438.misc +++ /dev/null @@ -1 +0,0 @@ -Run "main" trial tests under `poetry`. diff --git a/changelog.d/12441.misc b/changelog.d/12441.misc deleted file mode 100644 index c2619f16540b..000000000000 --- a/changelog.d/12441.misc +++ /dev/null @@ -1 +0,0 @@ -Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). diff --git a/changelog.d/12445.misc b/changelog.d/12445.misc deleted file mode 100644 index 954248115a0d..000000000000 --- a/changelog.d/12445.misc +++ /dev/null @@ -1 +0,0 @@ -Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. \ No newline at end of file diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc deleted file mode 100644 index 03e08aace427..000000000000 --- a/changelog.d/12449.misc +++ /dev/null @@ -1 +0,0 @@ -Use `poetry` to manage the virtualenv in debian packages. diff --git a/changelog.d/12450.misc b/changelog.d/12450.misc deleted file mode 100644 index 4b1c8cba875f..000000000000 --- a/changelog.d/12450.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in the release script help string. diff --git a/changelog.d/12451.doc b/changelog.d/12451.doc deleted file mode 100644 index c8b23c128539..000000000000 --- a/changelog.d/12451.doc +++ /dev/null @@ -1 +0,0 @@ -Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. diff --git a/changelog.d/12454.misc b/changelog.d/12454.misc deleted file mode 100644 index cb7ff74b4c8f..000000000000 --- a/changelog.d/12454.misc +++ /dev/null @@ -1 +0,0 @@ -Limit length of device_id to less than 512 characters. diff --git a/changelog.d/12455.misc b/changelog.d/12455.misc deleted file mode 100644 index 9b19945673e4..000000000000 --- a/changelog.d/12455.misc +++ /dev/null @@ -1 +0,0 @@ -Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. diff --git a/changelog.d/12457.doc b/changelog.d/12457.doc deleted file mode 100644 index a4871622cf76..000000000000 --- a/changelog.d/12457.doc +++ /dev/null @@ -1 +0,0 @@ -Update worker documentation and replace old `federation_reader` with `generic_worker`. \ No newline at end of file diff --git a/changelog.d/12464.misc b/changelog.d/12464.misc deleted file mode 100644 index 7a8cc6ba512c..000000000000 --- a/changelog.d/12464.misc +++ /dev/null @@ -1 +0,0 @@ -Dockerfile-workers: reduce the amount we install in the image. diff --git a/changelog.d/12465.feature b/changelog.d/12465.feature deleted file mode 100644 index 642dea966c44..000000000000 --- a/changelog.d/12465.feature +++ /dev/null @@ -1 +0,0 @@ -Enable processing of device list updates asynchronously. diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc deleted file mode 100644 index b0c2c950fe55..000000000000 --- a/changelog.d/12466.misc +++ /dev/null @@ -1 +0,0 @@ -Dockerfile-workers: give the master its own log config. diff --git a/changelog.d/12467.misc b/changelog.d/12467.misc deleted file mode 100644 index fbf415f7072b..000000000000 --- a/changelog.d/12467.misc +++ /dev/null @@ -1 +0,0 @@ -complement-synapse-workers: factor out separate entry point script. diff --git a/changelog.d/12468.misc b/changelog.d/12468.misc deleted file mode 100644 index 3d5d25247f64..000000000000 --- a/changelog.d/12468.misc +++ /dev/null @@ -1 +0,0 @@ -Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. diff --git a/changelog.d/12472.misc b/changelog.d/12472.misc deleted file mode 100644 index ed306209cc42..000000000000 --- a/changelog.d/12472.misc +++ /dev/null @@ -1 +0,0 @@ -Add a CI job which tests Synapse against the latest version of all dependencies. diff --git a/changelog.d/12474.misc b/changelog.d/12474.misc deleted file mode 100644 index 5292108b3920..000000000000 --- a/changelog.d/12474.misc +++ /dev/null @@ -1 +0,0 @@ -Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). diff --git a/changelog.d/12475.doc b/changelog.d/12475.doc deleted file mode 100644 index f4481d0613c2..000000000000 --- a/changelog.d/12475.doc +++ /dev/null @@ -1 +0,0 @@ -Strongly recommend `poetry` for development. diff --git a/changelog.d/12476.bugfix b/changelog.d/12476.bugfix deleted file mode 100644 index 9ad6a71abd52..000000000000 --- a/changelog.d/12476.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. diff --git a/changelog.d/12478.misc b/changelog.d/12478.misc deleted file mode 100644 index 061a604a1e16..000000000000 --- a/changelog.d/12478.misc +++ /dev/null @@ -1 +0,0 @@ -Use poetry-core instead of setuptools to build wheels. diff --git a/changelog.d/12483.misc b/changelog.d/12483.misc deleted file mode 100644 index 88c6e3e465c8..000000000000 --- a/changelog.d/12483.misc +++ /dev/null @@ -1 +0,0 @@ -Fix grammatical error in federation error response when the room version of a room is unknown. diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc deleted file mode 100644 index 4a3e2f4f0698..000000000000 --- a/changelog.d/12492.doc +++ /dev/null @@ -1 +0,0 @@ -Add some example configurations for workers and update architectural diagram. diff --git a/changelog.d/12495.doc b/changelog.d/12495.doc deleted file mode 100644 index afa011167513..000000000000 --- a/changelog.d/12495.doc +++ /dev/null @@ -1 +0,0 @@ -Fix a broken link in `README.rst`. diff --git a/changelog.d/12496.bugfix b/changelog.d/12496.bugfix deleted file mode 100644 index a68df7c96a5f..000000000000 --- a/changelog.d/12496.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. diff --git a/changelog.d/12497.misc b/changelog.d/12497.misc deleted file mode 100644 index 17a661ec61b5..000000000000 --- a/changelog.d/12497.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a minor typo in the Debian changelogs generated by the release script. diff --git a/changelog.d/12501.doc b/changelog.d/12501.doc deleted file mode 100644 index 278193a69a31..000000000000 --- a/changelog.d/12501.doc +++ /dev/null @@ -1 +0,0 @@ -Add HAProxy delegation example with CORS headers to docs. diff --git a/changelog.d/12510.bugfix b/changelog.d/12510.bugfix deleted file mode 100644 index d5856e982a57..000000000000 --- a/changelog.d/12510.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. diff --git a/changelog.d/12511.misc b/changelog.d/12511.misc deleted file mode 100644 index a314bedfc4ba..000000000000 --- a/changelog.d/12511.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unnecessary configuration overrides in tests. diff --git a/changelog.d/12514.misc b/changelog.d/12514.misc deleted file mode 100644 index 061a604a1e16..000000000000 --- a/changelog.d/12514.misc +++ /dev/null @@ -1 +0,0 @@ -Use poetry-core instead of setuptools to build wheels. diff --git a/changelog.d/12519.misc b/changelog.d/12519.misc deleted file mode 100644 index 9c023d8e3ec6..000000000000 --- a/changelog.d/12519.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the relations code for clarity. diff --git a/changelog.d/12520.bugfix b/changelog.d/12520.bugfix deleted file mode 100644 index c73005fde837..000000000000 --- a/changelog.d/12520.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. \ No newline at end of file diff --git a/changelog.d/12522.bugfix b/changelog.d/12522.bugfix deleted file mode 100644 index 2220f05ceb75..000000000000 --- a/changelog.d/12522.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. diff --git a/changelog.d/12527.doc b/changelog.d/12527.doc deleted file mode 100644 index e6907321e769..000000000000 --- a/changelog.d/12527.doc +++ /dev/null @@ -1,2 +0,0 @@ -Add an index to the configuration manual. - diff --git a/changelog.d/12528.misc b/changelog.d/12528.misc deleted file mode 100644 index f64b5d24b005..000000000000 --- a/changelog.d/12528.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. diff --git a/changelog.d/12529.misc b/changelog.d/12529.misc deleted file mode 100644 index 542710874204..000000000000 --- a/changelog.d/12529.misc +++ /dev/null @@ -1 +0,0 @@ -Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. diff --git a/changelog.d/12533.doc b/changelog.d/12533.doc deleted file mode 100644 index 2c15488111cb..000000000000 --- a/changelog.d/12533.doc +++ /dev/null @@ -1 +0,0 @@ -Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. \ No newline at end of file diff --git a/changelog.d/12543.misc b/changelog.d/12543.misc deleted file mode 100644 index eed7a6973ca9..000000000000 --- a/changelog.d/12543.misc +++ /dev/null @@ -1 +0,0 @@ -Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". diff --git a/debian/changelog b/debian/changelog index 7727536a74ce..20e756f6dbc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -matrix-synapse-py3 (1.57.1+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. + * New Synapse release 1.58.0rc1. - -- Synapse Packaging team Wed, 30 Mar 2022 12:21:43 +0100 + -- Synapse Packaging team Tue, 26 Apr 2022 11:15:20 +0100 matrix-synapse-py3 (1.57.1) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index c7f3e20feda0..80e25cef1d0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.57.1" +version = "1.58.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From f987cdd80bf7c695dc977eb64614019a9ce73ff0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:32:57 +0100 Subject: [PATCH 060/263] Changelog update --- CHANGES.md | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 95f0b5ad3720..b9d477a6ebb3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,11 @@ Synapse 1.58.0rc1 (2022-04-26) Features -------- -- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) -- Use poetry to manage Synapse's dependencies. ([\#12337](https://github.com/matrix-org/synapse/issues/12337)) +- Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) +- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) - Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) +- Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) Bugfixes @@ -15,10 +16,10 @@ Bugfixes - Prevent a sync request from removing a user's busy presence status. ([\#12213](https://github.com/matrix-org/synapse/issues/12213)) - Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper. ([\#12319](https://github.com/matrix-org/synapse/issues/12319)) -- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) -- Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) -- Fix a long-standing bug where the image thumbanils embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) -- Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) +- Fix a long-standing bug which incorrectly caused `GET /_matrix/client/v3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original. ([\#12476](https://github.com/matrix-org/synapse/issues/12476)) +- Fix a bug introduced in Synapse 1.27.0 where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1, no matter how many extremities were deleted. ([\#12496](https://github.com/matrix-org/synapse/issues/12496)) +- Fix a long-standing bug where the image thumbnails embedded into email notifications were broken. ([\#12510](https://github.com/matrix-org/synapse/issues/12510)) +- Fix a bug in the implementation of [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`. ([\#12520](https://github.com/matrix-org/synapse/issues/12520)) - Fix a bug introduced in Synapse 0.99.3 which could cause Synapse to consume large amounts of RAM when back-paginating in a large room. ([\#12522](https://github.com/matrix-org/synapse/issues/12522)) @@ -26,21 +27,20 @@ Improved Documentation ---------------------- - Fix rendering of the documentation site when using the 'print' feature. ([\#12340](https://github.com/matrix-org/synapse/issues/12340)) -- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368)) +- Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527)) - Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) - Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) -- Strongly recommend `poetry` for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Strongly recommend Poetry for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) - Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) - Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) - Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) -- Add an index to the configuration manual. ([\#12527](https://github.com/matrix-org/synapse/issues/12527)) - Remove extraneous comma in User Admin API's device deletion section so that the example JSON is actually valid and works. Contributed by @olmari. ([\#12533](https://github.com/matrix-org/synapse/issues/12533)) Deprecations and Removals ------------------------- -- The groups/communities feature in Synapse has been disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) +- **The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))** - Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) @@ -48,31 +48,24 @@ Internal Changes ---------------- - Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) -- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) - Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) -- Run twisted trunk CI job in the locked poetry environment. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)) -- Run lints under poetry in CI, and remove corresponding tox lint jobs. ([\#12434](https://github.com/matrix-org/synapse/issues/12434)) -- Run "main" trial tests under `poetry`. ([\#12438](https://github.com/matrix-org/synapse/issues/12438)) -- Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644). ([\#12441](https://github.com/matrix-org/synapse/issues/12441)) -- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration. ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) -- Use `poetry` to manage the virtualenv in debian packages. ([\#12449](https://github.com/matrix-org/synapse/issues/12449)) +- Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) +- Run CI in the locked Poetry environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) - Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) -- Limit length of device_id to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) +- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) - Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development. ([\#12455](https://github.com/matrix-org/synapse/issues/12455)) +- Limit length of `device_id` to less than 512 characters. ([\#12454](https://github.com/matrix-org/synapse/issues/12454)) - Dockerfile-workers: reduce the amount we install in the image. ([\#12464](https://github.com/matrix-org/synapse/issues/12464)) - Dockerfile-workers: give the master its own log config. ([\#12466](https://github.com/matrix-org/synapse/issues/12466)) - complement-synapse-workers: factor out separate entry point script. ([\#12467](https://github.com/matrix-org/synapse/issues/12467)) -- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) -- Add a CI job which tests Synapse against the latest version of all dependencies. ([\#12472](https://github.com/matrix-org/synapse/issues/12472)) - Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314). ([\#12474](https://github.com/matrix-org/synapse/issues/12474)) -- Use poetry-core instead of setuptools to build wheels. ([\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514)) - Fix grammatical error in federation error response when the room version of a room is unknown. ([\#12483](https://github.com/matrix-org/synapse/issues/12483)) -- Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) - Remove unnecessary configuration overrides in tests. ([\#12511](https://github.com/matrix-org/synapse/issues/12511)) - Refactor the relations code for clarity. ([\#12519](https://github.com/matrix-org/synapse/issues/12519)) - Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`. ([\#12528](https://github.com/matrix-org/synapse/issues/12528)) +- Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s. ([\#12468](https://github.com/matrix-org/synapse/issues/12468)) - Handle cancellation in `EventsWorkerStore._get_events_from_cache_or_db`. ([\#12529](https://github.com/matrix-org/synapse/issues/12529)) -- Build debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) Synapse 1.57.1 (2022-04-20) From a54d9b0508c17a29f2f02c87054cd8990d476054 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:37:21 +0100 Subject: [PATCH 061/263] We don't require redbaron in the release script --- poetry.lock | 62 +------------------------------------------------- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 62 deletions(-) diff --git a/poetry.lock b/poetry.lock index 95c1afc077a7..8c7af1fa1eac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,11 +1,3 @@ -[[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "attrs" version = "21.4.0" @@ -49,17 +41,6 @@ six = "*" [package.extras] visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"] -[[package]] -name = "baron" -version = "0.10.1" -description = "Full Syntax Tree for python to make writing refactoring code a realist task" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -rply = "*" - [[package]] name = "bcrypt" version = "3.2.0" @@ -984,20 +965,6 @@ Pygments = ">=2.5.1" [package.extras] md = ["cmarkgfm (>=0.8.0)"] -[[package]] -name = "redbaron" -version = "0.9.2" -description = "Abstraction on top of baron, a FST for python to make writing refactoring code a realistic task" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -baron = ">=0.7" - -[package.extras] -notebook = ["pygments"] - [[package]] name = "requests" version = "2.27.1" @@ -1038,17 +1005,6 @@ python-versions = ">=3.7" [package.extras] idna2008 = ["idna"] -[[package]] -name = "rply" -version = "0.7.8" -description = "A pure Python Lex/Yacc that works with RPython" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -appdirs = "*" - [[package]] name = "secretstorage" version = "3.3.1" @@ -1597,13 +1553,9 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "964ad29eaf7fd02749a4e735818f3bc0ba729c2f4b9e3213f0daa02643508b16" +content-hash = "f482a4f594a165dfe01ce253a22510d5faf38647ab0dcebc35789350cafd9bf0" [metadata.files] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -1616,10 +1568,6 @@ automat = [ {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"}, {file = "Automat-20.2.0.tar.gz", hash = "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33"}, ] -baron = [ - {file = "baron-0.10.1-py2.py3-none-any.whl", hash = "sha256:befb33f4b9e832c7cd1e3cf0eafa6dd3cb6ed4cb2544245147c019936f4e0a8a"}, - {file = "baron-0.10.1.tar.gz", hash = "sha256:af822ad44d4eb425c8516df4239ac4fdba9fdb398ef77e4924cd7c9b4045bc2f"}, -] bcrypt = [ {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd"}, {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, @@ -2412,10 +2360,6 @@ readme-renderer = [ {file = "readme_renderer-33.0-py3-none-any.whl", hash = "sha256:f02cee0c4de9636b5a62b6be50c9742427ba1b956aad1d938bfb087d0d72ccdf"}, {file = "readme_renderer-33.0.tar.gz", hash = "sha256:e3b53bc84bd6af054e4cc1fe3567dc1ae19f554134221043a3f8c674e22209db"}, ] -redbaron = [ - {file = "redbaron-0.9.2-py2.py3-none-any.whl", hash = "sha256:d01032b6a848b5521a8d6ef72486315c2880f420956870cdd742e2b5a09b9bab"}, - {file = "redbaron-0.9.2.tar.gz", hash = "sha256:472d0739ca6b2240bb2278ae428604a75472c9c12e86c6321e8c016139c0132f"}, -] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, @@ -2428,10 +2372,6 @@ rfc3986 = [ {file = "rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd"}, {file = "rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"}, ] -rply = [ - {file = "rply-0.7.8-py2.py3-none-any.whl", hash = "sha256:28ffd11d656c48aeb8c508eb382acd6a0bd906662624b34388751732a27807e7"}, - {file = "rply-0.7.8.tar.gz", hash = "sha256:2a808ac25a4580a9991fc304d64434e299a8fc75760574492f242cbb5bb301c9"}, -] secretstorage = [ {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, diff --git a/pyproject.toml b/pyproject.toml index 80e25cef1d0e..cc4908fa2377 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,7 +270,6 @@ idna = ">=2.5" # The following are used by the release script click = "==8.1.0" -redbaron = "==0.9.2" GitPython = "==3.1.14" commonmark = "==0.9.1" pygithub = "==1.55" From 416604e3bc94ce9c555b7fe4363296998fb1e945 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:51:47 +0100 Subject: [PATCH 062/263] Another set of changelog updates --- CHANGES.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b9d477a6ebb3..58f0f0a13cf3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,13 @@ Synapse 1.58.0rc1 (2022-04-26) ============================== +As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. + Features -------- - Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) -- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) +- Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) - Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) - Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) @@ -30,7 +32,7 @@ Improved Documentation - Add a manual documenting config file options. ([\#12368](https://github.com/matrix-org/synapse/issues/12368), [\#12527](https://github.com/matrix-org/synapse/issues/12527)) - Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental. ([\#12451](https://github.com/matrix-org/synapse/issues/12451)) - Update worker documentation and replace old `federation_reader` with `generic_worker`. ([\#12457](https://github.com/matrix-org/synapse/issues/12457)) -- Strongly recommend Poetry for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) +- Strongly recommend [Poetry](https://python-poetry.org/) for development. ([\#12475](https://github.com/matrix-org/synapse/issues/12475)) - Add some example configurations for workers and update architectural diagram. ([\#12492](https://github.com/matrix-org/synapse/issues/12492)) - Fix a broken link in `README.rst`. ([\#12495](https://github.com/matrix-org/synapse/issues/12495)) - Add HAProxy delegation example with CORS headers to docs. ([\#12501](https://github.com/matrix-org/synapse/issues/12501)) @@ -50,7 +52,7 @@ Internal Changes - Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join. ([\#12394](https://github.com/matrix-org/synapse/issues/12394)) - Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive. ([\#12399](https://github.com/matrix-org/synapse/issues/12399)) - Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#12395](https://github.com/matrix-org/synapse/issues/12395)) -- Run CI in the locked Poetry environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425)), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) +- Run CI in the locked [Poetry](https://python-poetry.org/) environment, and remove corresponding `tox` jobs. ([\#12425](https://github.com/matrix-org/synapse/issues/12425), [\#12434](https://github.com/matrix-org/synapse/issues/12434), [\#12438](https://github.com/matrix-org/synapse/issues/12438), [\#12441](https://github.com/matrix-org/synapse/issues/12441), [\#12449](https://github.com/matrix-org/synapse/issues/12449), [\#12478](https://github.com/matrix-org/synapse/issues/12478), [\#12514](https://github.com/matrix-org/synapse/issues/12514), [\#12472](https://github.com/matrix-org/synapse/issues/12472)) - Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current iteration of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666). ([\#12445](https://github.com/matrix-org/synapse/issues/12445)) - Fix typo in the release script help string. ([\#12450](https://github.com/matrix-org/synapse/issues/12450)) - Fix a minor typo in the Debian changelogs generated by the release script. ([\#12497](https://github.com/matrix-org/synapse/issues/12497)) From 6b9e95015b39898011991a73051a71cb7dab3d1f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:53:37 +0100 Subject: [PATCH 063/263] Lint the release script --- scripts-dev/release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 725e8f0479c5..9d7c7c445fc0 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -31,7 +31,6 @@ import click import commonmark import git - from click.exceptions import ClickException from github import Github from packaging import version From ee1601e59d8a3de24914d4c08af4012daeb1689e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 11:59:10 +0100 Subject: [PATCH 064/263] Unbold deprecation: it is mentioned at the top --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 58f0f0a13cf3..d8ecb11e4e4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -42,7 +42,7 @@ Improved Documentation Deprecations and Removals ------------------------- -- **The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344))** +- The groups/communities feature in Synapse is now disabled by default. ([\#12344](https://github.com/matrix-org/synapse/issues/12344)) - Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#12382](https://github.com/matrix-org/synapse/issues/12382)) From 706456de1f63a4b9796592fd452da7ce30735274 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 26 Apr 2022 17:31:52 +0300 Subject: [PATCH 065/263] Mark Dockerfile as requiring BuildKit (#12541) Co-authored-by: David Robertson --- changelog.d/12541.docker | 1 + docker/Dockerfile | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/12541.docker diff --git a/changelog.d/12541.docker b/changelog.d/12541.docker new file mode 100644 index 000000000000..c3b9c31657cd --- /dev/null +++ b/changelog.d/12541.docker @@ -0,0 +1 @@ +Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. diff --git a/docker/Dockerfile b/docker/Dockerfile index 4523c60645bd..ccc6a9f77849 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # Dockerfile to build the matrixdotorg/synapse docker images. # # Note that it uses features which are only available in BuildKit - see From c48ab3734ee57fcf0841d814680e50e49948cafe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Apr 2022 15:48:16 +0100 Subject: [PATCH 066/263] Fix sending opentracing contexts to remote servers (#12555) --- changelog.d/12555.bugfix | 1 + synapse/storage/databases/main/devices.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12555.bugfix diff --git a/changelog.d/12555.bugfix b/changelog.d/12555.bugfix new file mode 100644 index 000000000000..3c721da1ee4e --- /dev/null +++ b/changelog.d/12555.bugfix @@ -0,0 +1 @@ +Fix sending opentracing contexts to whitelisted remote servers with device lists updates. Broken in v1.58.0rc1. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 318e4df376b2..59d223a9008c 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1776,7 +1776,17 @@ async def get_uncoverted_outbound_room_pokes( def get_uncoverted_outbound_room_pokes_txn(txn): txn.execute(sql, (limit,)) - return txn.fetchall() + + return [ + ( + user_id, + device_id, + room_id, + stream_id, + db_to_json(opentracing_context), + ) + for user_id, device_id, room_id, stream_id, opentracing_context in txn + ] return await self.db_pool.runInteraction( "get_uncoverted_outbound_room_pokes", get_uncoverted_outbound_room_pokes_txn From 6d89f1239ce5ebe83b085a2b8939b2b6ce461265 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 15:53:06 +0100 Subject: [PATCH 067/263] Comment out dodgy log-kv (#12554) --- changelog.d/12554.bugfix | 1 + synapse/handlers/device.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12554.bugfix diff --git a/changelog.d/12554.bugfix b/changelog.d/12554.bugfix new file mode 100644 index 000000000000..48c1146db52b --- /dev/null +++ b/changelog.d/12554.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 3c0fc756d46b..319836da2bee 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -683,9 +683,12 @@ async def _handle_new_device_update_async(self) -> None: self.federation_sender.send_device_messages( host, immediate=False ) - log_kv( - {"message": "sent device update to host", "host": host} - ) + # TODO: when called, this isn't in a logging context. + # This leads to log spam, sentry event spam, and massive + # memory usage. See #12552. + # log_kv( + # {"message": "sent device update to host", "host": host} + # ) if current_stream_id != stream_id: # Clear the set of hosts we've already sent to as we're From f59e3f4c900b3961e2b96592a67cd2f0821b00db Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Apr 2022 17:07:21 +0100 Subject: [PATCH 068/263] Mark remote device list updates as already handled (#12557) --- changelog.d/12557.misc | 1 + synapse/handlers/device.py | 5 +++-- synapse/storage/databases/main/devices.py | 3 ++- tests/storage/test_devices.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12557.misc diff --git a/changelog.d/12557.misc b/changelog.d/12557.misc new file mode 100644 index 000000000000..e4eb895ef5cb --- /dev/null +++ b/changelog.d/12557.misc @@ -0,0 +1 @@ +Reduce unnecessary work when handling remote device list updates. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 319836da2bee..a91b1ee4d5f4 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -505,8 +505,9 @@ async def notify_device_update( "device_list_key", position, users={user_id}, rooms=room_ids ) - # We may need to do some processing asynchronously. - self._handle_new_device_update_async() + # We may need to do some processing asynchronously for local user IDs. + if self.hs.is_mine_id(user_id): + self._handle_new_device_update_async() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 59d223a9008c..483dd804067a 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -1748,7 +1748,8 @@ def _add_device_outbound_room_poke_txn( device_id, room_id, stream_id, - False, + # We only need to calculate outbound pokes for local users + not self.hs.is_mine_id(user_id), encoded_context, ) for room_id in room_ids diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index ccc3893869db..bbf079b25b59 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -29,7 +29,7 @@ def add_device_change(self, user_id, device_ids, host): for device_id in device_ids: stream_id = self.get_success( self.store.add_device_change_to_streams( - "user_id", [device_id], ["!some:room"] + user_id, [device_id], ["!some:room"] ) ) From 6b64ee9ec7c09a322c38a2e537ad95a32f5ca5e2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:16:43 +0100 Subject: [PATCH 069/263] 1.58.0rc2 --- CHANGES.md | 18 ++++++++++++++++++ changelog.d/12554.bugfix | 1 - changelog.d/12555.bugfix | 1 - changelog.d/12557.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/12554.bugfix delete mode 100644 changelog.d/12555.bugfix delete mode 100644 changelog.d/12557.misc diff --git a/CHANGES.md b/CHANGES.md index d8ecb11e4e4f..b65cf00f4f49 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,21 @@ +Synapse 1.58.0rc2 (2022-04-26) +============================== + +This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handling device list updates. + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555)) + + +Internal Changes +---------------- + +- Reduce unnecessary work when handling remote device list updates. ([\#12557](https://github.com/matrix-org/synapse/issues/12557)) + + Synapse 1.58.0rc1 (2022-04-26) ============================== diff --git a/changelog.d/12554.bugfix b/changelog.d/12554.bugfix deleted file mode 100644 index 48c1146db52b..000000000000 --- a/changelog.d/12554.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. diff --git a/changelog.d/12555.bugfix b/changelog.d/12555.bugfix deleted file mode 100644 index 3c721da1ee4e..000000000000 --- a/changelog.d/12555.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sending opentracing contexts to whitelisted remote servers with device lists updates. Broken in v1.58.0rc1. diff --git a/changelog.d/12557.misc b/changelog.d/12557.misc deleted file mode 100644 index e4eb895ef5cb..000000000000 --- a/changelog.d/12557.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce unnecessary work when handling remote device list updates. diff --git a/debian/changelog b/debian/changelog index 20e756f6dbc2..5f1bf872bbcc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium + + * New Synapse release 1.58.0rc2. + + -- Synapse Packaging team Tue, 26 Apr 2022 17:14:56 +0100 + matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium * Use poetry to manage the bundled virtualenv included with this package. diff --git a/pyproject.toml b/pyproject.toml index cc4908fa2377..bdded7843440 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0rc1" +version = "1.58.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 56c9c6c4654f4c356fd266396e97f426c2afd8c6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:17:56 +0100 Subject: [PATCH 070/263] Credit Tulir's contribution in 1.58.0rc1 to Beeper, too --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b65cf00f4f49..b9b10bb0d69f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,7 +27,7 @@ Features - Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly. ([\#11398](https://github.com/matrix-org/synapse/issues/11398)) - Docker images and Debian packages from matrix.org now contain a locked set of Python dependencies, greatly improving build reproducibility. ([Board](https://github.com/orgs/matrix-org/projects/54), [\#11537](https://github.com/matrix-org/synapse/issues/11537)) - Enable processing of device list updates asynchronously. ([\#12365](https://github.com/matrix-org/synapse/issues/12365), [\#12465](https://github.com/matrix-org/synapse/issues/12465)) -- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) +- Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir @ Beeper. ([\#12427](https://github.com/matrix-org/synapse/issues/12427)) - Build Debian packages for Ubuntu 22.04 "Jammy Jellyfish". ([\#12543](https://github.com/matrix-org/synapse/issues/12543)) From 9cfecd2dc01e2e5c6ccb7b87f7beae83a98650f3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 26 Apr 2022 17:22:12 +0100 Subject: [PATCH 071/263] Adjust changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b9b10bb0d69f..1fbe0815dedf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ This release candidate fixes bugs related to Synapse 1.58.0rc1's logic for handl Bugfixes -------- -- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) +- Fix a bug introduced in Synapse 1.58.0rc1 where the main process could consume excessive amounts of CPU and memory while handling sentry logging failures. ([\#12554](https://github.com/matrix-org/synapse/issues/12554)) - Fix a bug introduced in Synapse 1.58.0rc1 where opentracing contexts were not correctly sent to whitelisted remote servers with device lists updates. ([\#12555](https://github.com/matrix-org/synapse/issues/12555)) From 63ba9ba38b47bc20074e2c83492c4d113cfe21b8 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Tue, 26 Apr 2022 20:14:21 +0100 Subject: [PATCH 072/263] Bound ephemeral events by key (#12544) Co-authored-by: Brad Murray Co-authored-by: Andrew Morgan --- changelog.d/12544.bugfix | 1 + synapse/handlers/appservice.py | 4 +- synapse/handlers/receipts.py | 4 +- tests/handlers/test_appservice.py | 82 +++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12544.bugfix diff --git a/changelog.d/12544.bugfix b/changelog.d/12544.bugfix new file mode 100644 index 000000000000..b5169cd8311a --- /dev/null +++ b/changelog.d/12544.bugfix @@ -0,0 +1 @@ +Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 1b5784050621..b3894666ccf5 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -416,7 +416,7 @@ async def _handle_typing( return typing async def _handle_receipts( - self, service: ApplicationService, new_token: Optional[int] + self, service: ApplicationService, new_token: int ) -> List[JsonDict]: """ Return the latest read receipts that the given application service should receive. @@ -447,7 +447,7 @@ async def _handle_receipts( receipts_source = self.event_sources.sources.receipt receipts, _ = await receipts_source.get_new_events_as( - service=service, from_key=from_key + service=service, from_key=from_key, to_key=new_token ) return receipts diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 6250bb3bdf2b..cfe860decc95 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -239,13 +239,14 @@ async def get_new_events( return events, to_key async def get_new_events_as( - self, from_key: int, service: ApplicationService + self, from_key: int, to_key: int, service: ApplicationService ) -> Tuple[List[JsonDict], int]: """Returns a set of new read receipt events that an appservice may be interested in. Args: from_key: the stream position at which events should be fetched from + to_key: the stream position up to which events should be fetched to service: The appservice which may be interested Returns: @@ -255,7 +256,6 @@ async def get_new_events_as( * The current read receipt stream token. """ from_key = int(from_key) - to_key = self.get_current_key() if from_key == to_key: return [], to_key diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 8c72cf6b308b..5b0cd1ab8608 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -411,6 +411,88 @@ def prepare(self, reactor, clock, hs): "exclusive_as_user", "password", self.exclusive_as_user_device_id ) + def test_sending_read_receipt_batches_to_application_services(self): + """Tests that a large batch of read receipts are sent correctly to + interested application services. + """ + # Register an application service that's interested in a certain user + # and room prefix + interested_appservice = self._register_application_service( + namespaces={ + ApplicationService.NS_USERS: [ + { + "regex": "@exclusive_as_user:.+", + "exclusive": True, + } + ], + ApplicationService.NS_ROOMS: [ + { + "regex": "!fakeroom_.*", + "exclusive": True, + } + ], + }, + ) + + # "Complete" a transaction. + # All this really does for us is make an entry in the application_services_state + # database table, which tracks the current stream_token per stream ID per AS. + self.get_success( + self.hs.get_datastores().main.complete_appservice_txn( + 0, + interested_appservice, + ) + ) + + # Now, pretend that we receive a large burst of read receipts (300 total) that + # all come in at once. + for i in range(300): + self.get_success( + # Insert a fake read receipt into the database + self.hs.get_datastores().main.insert_receipt( + # We have to use unique room ID + user ID combinations here, as the db query + # is an upsert. + room_id=f"!fakeroom_{i}:test", + receipt_type="m.read", + user_id=self.local_user, + event_ids=[f"$eventid_{i}"], + data={}, + ) + ) + + # Now notify the appservice handler that 300 read receipts have all arrived + # at once. What will it do! + # note: stream tokens start at 2 + for stream_token in range(2, 303): + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key="receipt_key", + new_token=stream_token, + users=[self.exclusive_as_user], + ) + ) + + # Using our txn send mock, we can see what the AS received. After iterating over every + # transaction, we'd like to see all 300 read receipts accounted for. + # No more, no less. + all_ephemeral_events = [] + for call in self.send_mock.call_args_list: + ephemeral_events = call[0][2] + all_ephemeral_events += ephemeral_events + + # Ensure that no duplicate events were sent + self.assertEqual(len(all_ephemeral_events), 300) + + # Check that the ephemeral event is a read receipt with the expected structure + latest_read_receipt = all_ephemeral_events[-1] + self.assertEqual(latest_read_receipt["type"], "m.receipt") + + event_id = list(latest_read_receipt["content"].keys())[0] + self.assertEqual( + latest_read_receipt["content"][event_id]["m.read"], {self.local_user: {}} + ) + @unittest.override_config( {"experimental_features": {"msc2409_to_device_messages_enabled": True}} ) From b76f1a4d5f918def1f643910939b80e9e035e07f Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 27 Apr 2022 14:05:00 +0200 Subject: [PATCH 073/263] Add some type hints to datastore (#12485) --- changelog.d/12485.misc | 1 + synapse/storage/databases/main/__init__.py | 21 +++-- synapse/storage/databases/main/appservice.py | 4 +- synapse/storage/databases/main/deviceinbox.py | 79 +++++++++++++------ synapse/storage/databases/main/devices.py | 51 +++++++----- .../storage/databases/main/group_server.py | 4 +- synapse/storage/databases/main/keys.py | 15 ++-- .../databases/main/media_repository.py | 13 ++- synapse/storage/databases/main/presence.py | 19 ++++- synapse/storage/databases/main/pusher.py | 49 ++++++++---- synapse/storage/databases/main/state.py | 4 +- synapse/storage/databases/main/ui_auth.py | 12 +-- 12 files changed, 188 insertions(+), 84 deletions(-) create mode 100644 changelog.d/12485.misc diff --git a/changelog.d/12485.misc b/changelog.d/12485.misc new file mode 100644 index 000000000000..e793d08e5e3f --- /dev/null +++ b/changelog.d/12485.misc @@ -0,0 +1 @@ +Add some type hints to datastore. \ No newline at end of file diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 951031af50b0..5895b892024c 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -15,12 +15,17 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple, cast from synapse.config.homeserver import HomeServerConfig -from synapse.storage.database import DatabasePool, LoggingDatabaseConnection +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.storage.databases.main.stats import UserSortOrder -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine +from synapse.storage.types import Cursor from synapse.storage.util.id_generators import ( IdGenerator, MultiWriterIdGenerator, @@ -266,7 +271,9 @@ async def get_users_paginate( A tuple of a list of mappings from user to information and a count of total users. """ - def get_users_paginate_txn(txn): + def get_users_paginate_txn( + txn: LoggingTransaction, + ) -> Tuple[List[JsonDict], int]: filters = [] args = [self.hs.config.server.server_name] @@ -301,7 +308,7 @@ def get_users_paginate_txn(txn): """ sql = "SELECT COUNT(*) as total_users " + sql_base txn.execute(sql, args) - count = txn.fetchone()[0] + count = cast(Tuple[int], txn.fetchone())[0] sql = f""" SELECT name, user_type, is_guest, admin, deactivated, shadow_banned, @@ -338,7 +345,9 @@ async def search_users(self, term: str) -> Optional[List[JsonDict]]: ) -def check_database_before_upgrade(cur, database_engine, config: HomeServerConfig): +def check_database_before_upgrade( + cur: Cursor, database_engine: BaseDatabaseEngine, config: HomeServerConfig +) -> None: """Called before upgrading an existing database to check that it is broadly sane compared with the configuration. """ diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index fa732edcca08..945707b0ecd5 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -14,7 +14,7 @@ # limitations under the License. import logging import re -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Pattern, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Pattern, Tuple, cast from synapse.appservice import ( ApplicationService, @@ -83,7 +83,7 @@ def get_max_as_txn_id(txn: Cursor) -> int: txn.execute( "SELECT COALESCE(max(txn_id), 0) FROM application_services_txns" ) - return txn.fetchone()[0] # type: ignore + return cast(Tuple[int], txn.fetchone())[0] self._as_txn_seq_gen = build_sequence_generator( db_conn, diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index b4a1b041b1f8..599b41838397 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -14,7 +14,17 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Set, Tuple, cast +from typing import ( + TYPE_CHECKING, + Collection, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + cast, +) from synapse.logging import issue9533_logger from synapse.logging.opentracing import log_kv, set_tag, trace @@ -118,7 +128,13 @@ def __init__( prefilled_cache=device_outbox_prefill, ) - def process_replication_rows(self, stream_name, instance_name, token, rows): + def process_replication_rows( + self, + stream_name: str, + instance_name: str, + token: int, + rows: Iterable[ToDeviceStream.ToDeviceStreamRow], + ) -> None: if stream_name == ToDeviceStream.NAME: # If replication is happening than postgres must be being used. assert isinstance(self._device_inbox_id_gen, MultiWriterIdGenerator) @@ -134,7 +150,7 @@ def process_replication_rows(self, stream_name, instance_name, token, rows): ) return super().process_replication_rows(stream_name, instance_name, token, rows) - def get_to_device_stream_token(self): + def get_to_device_stream_token(self) -> int: return self._device_inbox_id_gen.get_current_token() async def get_messages_for_user_devices( @@ -301,7 +317,9 @@ async def _get_device_messages( if not user_ids_to_query: return {}, to_stream_id - def get_device_messages_txn(txn: LoggingTransaction): + def get_device_messages_txn( + txn: LoggingTransaction, + ) -> Tuple[Dict[Tuple[str, str], List[JsonDict]], int]: # Build a query to select messages from any of the given devices that # are between the given stream id bounds. @@ -428,7 +446,7 @@ async def delete_messages_for_device( log_kv({"message": "No changes in cache since last check"}) return 0 - def delete_messages_for_device_txn(txn): + def delete_messages_for_device_txn(txn: LoggingTransaction) -> int: sql = ( "DELETE FROM device_inbox" " WHERE user_id = ? AND device_id = ?" @@ -455,15 +473,14 @@ def delete_messages_for_device_txn(txn): @trace async def get_new_device_msgs_for_remote( - self, destination, last_stream_id, current_stream_id, limit - ) -> Tuple[List[dict], int]: + self, destination: str, last_stream_id: int, current_stream_id: int, limit: int + ) -> Tuple[List[JsonDict], int]: """ Args: - destination(str): The name of the remote server. - last_stream_id(int|long): The last position of the device message stream + destination: The name of the remote server. + last_stream_id: The last position of the device message stream that the server sent up to. - current_stream_id(int|long): The current position of the device - message stream. + current_stream_id: The current position of the device message stream. Returns: A list of messages for the device and where in the stream the messages got to. """ @@ -485,7 +502,9 @@ async def get_new_device_msgs_for_remote( return [], last_stream_id @trace - def get_new_messages_for_remote_destination_txn(txn): + def get_new_messages_for_remote_destination_txn( + txn: LoggingTransaction, + ) -> Tuple[List[JsonDict], int]: sql = ( "SELECT stream_id, messages_json FROM device_federation_outbox" " WHERE destination = ?" @@ -527,7 +546,7 @@ async def delete_device_msgs_for_remote( up_to_stream_id: Where to delete messages up to. """ - def delete_messages_for_remote_destination_txn(txn): + def delete_messages_for_remote_destination_txn(txn: LoggingTransaction) -> None: sql = ( "DELETE FROM device_federation_outbox" " WHERE destination = ?" @@ -566,7 +585,9 @@ async def get_all_new_device_messages( if last_id == current_id: return [], current_id, False - def get_all_new_device_messages_txn(txn): + def get_all_new_device_messages_txn( + txn: LoggingTransaction, + ) -> Tuple[List[Tuple[int, tuple]], int, bool]: # We limit like this as we might have multiple rows per stream_id, and # we want to make sure we always get all entries for any stream_id # we return. @@ -607,8 +628,8 @@ def get_all_new_device_messages_txn(txn): @trace async def add_messages_to_device_inbox( self, - local_messages_by_user_then_device: dict, - remote_messages_by_destination: dict, + local_messages_by_user_then_device: Dict[str, Dict[str, JsonDict]], + remote_messages_by_destination: Dict[str, JsonDict], ) -> int: """Used to send messages from this server. @@ -624,7 +645,9 @@ async def add_messages_to_device_inbox( assert self._can_write_to_device - def add_messages_txn(txn, now_ms, stream_id): + def add_messages_txn( + txn: LoggingTransaction, now_ms: int, stream_id: int + ) -> None: # Add the local messages directly to the local inbox. self._add_messages_to_local_device_inbox_txn( txn, stream_id, local_messages_by_user_then_device @@ -677,11 +700,16 @@ def add_messages_txn(txn, now_ms, stream_id): return self._device_inbox_id_gen.get_current_token() async def add_messages_from_remote_to_device_inbox( - self, origin: str, message_id: str, local_messages_by_user_then_device: dict + self, + origin: str, + message_id: str, + local_messages_by_user_then_device: Dict[str, Dict[str, JsonDict]], ) -> int: assert self._can_write_to_device - def add_messages_txn(txn, now_ms, stream_id): + def add_messages_txn( + txn: LoggingTransaction, now_ms: int, stream_id: int + ) -> None: # Check if we've already inserted a matching message_id for that # origin. This can happen if the origin doesn't receive our # acknowledgement from the first time we received the message. @@ -727,8 +755,11 @@ def add_messages_txn(txn, now_ms, stream_id): return stream_id def _add_messages_to_local_device_inbox_txn( - self, txn, stream_id, messages_by_user_then_device - ): + self, + txn: LoggingTransaction, + stream_id: int, + messages_by_user_then_device: Dict[str, Dict[str, JsonDict]], + ) -> None: assert self._can_write_to_device local_by_user_then_device = {} @@ -840,8 +871,10 @@ def __init__( self._remove_dead_devices_from_device_inbox, ) - async def _background_drop_index_device_inbox(self, progress, batch_size): - def reindex_txn(conn): + async def _background_drop_index_device_inbox( + self, progress: JsonDict, batch_size: int + ) -> int: + def reindex_txn(conn: LoggingDatabaseConnection) -> None: txn = conn.cursor() txn.execute("DROP INDEX IF EXISTS device_inbox_stream_id") txn.close() diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 483dd804067a..2df4dd4ed423 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -25,6 +25,7 @@ Optional, Set, Tuple, + cast, ) from synapse.api.errors import Codes, StoreError @@ -136,7 +137,9 @@ async def count_devices_by_users(self, user_ids: Optional[List[str]] = None) -> Number of devices of this users. """ - def count_devices_by_users_txn(txn, user_ids): + def count_devices_by_users_txn( + txn: LoggingTransaction, user_ids: List[str] + ) -> int: sql = """ SELECT count(*) FROM devices @@ -149,7 +152,7 @@ def count_devices_by_users_txn(txn, user_ids): ) txn.execute(sql + clause, args) - return txn.fetchone()[0] + return cast(Tuple[int], txn.fetchone())[0] if not user_ids: return 0 @@ -468,7 +471,7 @@ def _get_device_updates_by_remote_txn( """ txn.execute(sql, (destination, from_stream_id, now_stream_id, limit)) - return list(txn) + return cast(List[Tuple[str, str, int, Optional[str]]], txn.fetchall()) async def _get_device_update_edus_by_remote( self, @@ -549,7 +552,7 @@ async def _get_device_update_edus_by_remote( async def _get_last_device_update_for_remote_user( self, destination: str, user_id: str, from_stream_id: int ) -> int: - def f(txn): + def f(txn: LoggingTransaction) -> int: prev_sent_id_sql = """ SELECT coalesce(max(stream_id), 0) as stream_id FROM device_lists_outbound_last_success @@ -767,7 +770,7 @@ async def get_users_whose_devices_changed( if not user_ids_to_check: return set() - def _get_users_whose_devices_changed_txn(txn): + def _get_users_whose_devices_changed_txn(txn: LoggingTransaction) -> Set[str]: changes = set() stream_id_where_clause = "stream_id > ?" @@ -966,7 +969,9 @@ async def mark_remote_user_device_cache_as_valid(self, user_id: str) -> None: async def mark_remote_user_device_list_as_unsubscribed(self, user_id: str) -> None: """Mark that we no longer track device lists for remote user.""" - def _mark_remote_user_device_list_as_unsubscribed_txn(txn): + def _mark_remote_user_device_list_as_unsubscribed_txn( + txn: LoggingTransaction, + ) -> None: self.db_pool.simple_delete_txn( txn, table="device_lists_remote_extremeties", @@ -1004,7 +1009,7 @@ async def get_dehydrated_device( ) def _store_dehydrated_device_txn( - self, txn, user_id: str, device_id: str, device_data: str + self, txn: LoggingTransaction, user_id: str, device_id: str, device_data: str ) -> Optional[str]: old_device_id = self.db_pool.simple_select_one_onecol_txn( txn, @@ -1081,7 +1086,7 @@ async def _prune_old_outbound_device_pokes( """ yesterday = self._clock.time_msec() - prune_age - def _prune_txn(txn): + def _prune_txn(txn: LoggingTransaction) -> None: # look for (user, destination) pairs which have an update older than # the cutoff. # @@ -1204,8 +1209,10 @@ def __init__( "drop_device_lists_outbound_last_success_non_unique_idx", ) - async def _drop_device_list_streams_non_unique_indexes(self, progress, batch_size): - def f(conn): + async def _drop_device_list_streams_non_unique_indexes( + self, progress: JsonDict, batch_size: int + ) -> int: + def f(conn: LoggingDatabaseConnection) -> None: txn = conn.cursor() txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") @@ -1217,7 +1224,9 @@ def f(conn): ) return 1 - async def _remove_duplicate_outbound_pokes(self, progress, batch_size): + async def _remove_duplicate_outbound_pokes( + self, progress: JsonDict, batch_size: int + ) -> int: # for some reason, we have accumulated duplicate entries in # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less # efficient. @@ -1230,7 +1239,7 @@ async def _remove_duplicate_outbound_pokes(self, progress, batch_size): {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, ) - def _txn(txn): + def _txn(txn: LoggingTransaction) -> int: clause, args = make_tuple_comparison_clause( [(x, last_row[x]) for x in KEY_COLS] ) @@ -1602,7 +1611,9 @@ async def add_device_change_to_streams( context = get_active_span_text_map() - def add_device_changes_txn(txn, stream_ids): + def add_device_changes_txn( + txn: LoggingTransaction, stream_ids: List[int] + ) -> None: self._add_device_change_to_stream_txn( txn, user_id, @@ -1635,8 +1646,8 @@ def _add_device_change_to_stream_txn( txn: LoggingTransaction, user_id: str, device_ids: Collection[str], - stream_ids: List[str], - ): + stream_ids: List[int], + ) -> None: txn.call_after( self._device_list_stream_cache.entity_has_changed, user_id, @@ -1720,7 +1731,7 @@ def _add_device_outbound_room_poke_txn( user_id: str, device_ids: Iterable[str], room_ids: Collection[str], - stream_ids: List[str], + stream_ids: List[int], context: Dict[str, str], ) -> None: """Record the user in the room has updated their device.""" @@ -1775,7 +1786,9 @@ async def get_uncoverted_outbound_room_pokes( LIMIT ? """ - def get_uncoverted_outbound_room_pokes_txn(txn): + def get_uncoverted_outbound_room_pokes_txn( + txn: LoggingTransaction, + ) -> List[Tuple[str, str, str, int, Optional[Dict[str, str]]]]: txn.execute(sql, (limit,)) return [ @@ -1808,7 +1821,9 @@ async def add_device_list_outbound_pokes( Marks the associated row in `device_lists_changes_in_room` as handled. """ - def add_device_list_outbound_pokes_txn(txn, stream_ids: List[int]): + def add_device_list_outbound_pokes_txn( + txn: LoggingTransaction, stream_ids: List[int] + ) -> None: if hosts: self._add_device_outbound_poke_to_stream_txn( txn, diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py index 0aef121d8348..04efad9e9a39 100644 --- a/synapse/storage/databases/main/group_server.py +++ b/synapse/storage/databases/main/group_server.py @@ -522,7 +522,9 @@ async def get_joined_groups(self, user_id: str) -> List[str]: desc="get_joined_groups", ) - async def get_all_groups_for_user(self, user_id, now_token) -> List[JsonDict]: + async def get_all_groups_for_user( + self, user_id: str, now_token: int + ) -> List[JsonDict]: def _get_all_groups_for_user_txn(txn: LoggingTransaction) -> List[JsonDict]: sql = """ SELECT group_id, type, membership, u.content diff --git a/synapse/storage/databases/main/keys.py b/synapse/storage/databases/main/keys.py index 6990f3ed1d27..0a19f607bda1 100644 --- a/synapse/storage/databases/main/keys.py +++ b/synapse/storage/databases/main/keys.py @@ -15,11 +15,12 @@ import itertools import logging -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple from signedjson.key import decode_verify_key_bytes from synapse.storage._base import SQLBaseStore +from synapse.storage.database import LoggingTransaction from synapse.storage.keys import FetchKeyResult from synapse.storage.types import Cursor from synapse.util.caches.descriptors import cached, cachedList @@ -35,7 +36,9 @@ class KeyStore(SQLBaseStore): """Persistence for signature verification keys""" @cached() - def _get_server_verify_key(self, server_name_and_key_id): + def _get_server_verify_key( + self, server_name_and_key_id: Tuple[str, str] + ) -> FetchKeyResult: raise NotImplementedError() @cachedList( @@ -179,19 +182,21 @@ async def store_server_keys_json( async def get_server_keys_json( self, server_keys: Iterable[Tuple[str, Optional[str], Optional[str]]] - ) -> Dict[Tuple[str, Optional[str], Optional[str]], List[dict]]: + ) -> Dict[Tuple[str, Optional[str], Optional[str]], List[Dict[str, Any]]]: """Retrieve the key json for a list of server_keys and key ids. If no keys are found for a given server, key_id and source then that server, key_id, and source triplet entry will be an empty list. The JSON is returned as a byte array so that it can be efficiently used in an HTTP response. Args: - server_keys (list): List of (server_name, key_id, source) triplets. + server_keys: List of (server_name, key_id, source) triplets. Returns: A mapping from (server_name, key_id, source) triplets to a list of dicts """ - def _get_server_keys_json_txn(txn): + def _get_server_keys_json_txn( + txn: LoggingTransaction, + ) -> Dict[Tuple[str, Optional[str], Optional[str]], List[Dict[str, Any]]]: results = {} for server_name, key_id, from_server in server_keys: keyvalues = {"server_name": server_name} diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 322ed0539014..40ac377ca95e 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -388,7 +388,14 @@ def get_url_cache_txn(txn: LoggingTransaction) -> Optional[Dict[str, Any]]: return await self.db_pool.runInteraction("get_url_cache", get_url_cache_txn) async def store_url_cache( - self, url, response_code, etag, expires_ts, og, media_id, download_ts + self, + url: str, + response_code: int, + etag: Optional[str], + expires_ts: int, + og: Optional[str], + media_id: str, + download_ts: int, ) -> None: await self.db_pool.simple_insert( "local_media_repository_url_cache", @@ -441,7 +448,7 @@ async def store_local_thumbnail( ) async def get_cached_remote_media( - self, origin, media_id: str + self, origin: str, media_id: str ) -> Optional[Dict[str, Any]]: return await self.db_pool.simple_select_one( "remote_media_cache", @@ -608,7 +615,7 @@ async def get_remote_media_before(self, before_ts: int) -> List[Dict[str, str]]: ) async def delete_remote_media(self, media_origin: str, media_id: str) -> None: - def delete_remote_media_txn(txn): + def delete_remote_media_txn(txn: LoggingTransaction) -> None: self.db_pool.simple_delete_txn( txn, "remote_media_cache", diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index d3c4611686f8..b47c511450b9 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, Dict, Iterable, List, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Tuple, cast from synapse.api.presence import PresenceState, UserPresenceState from synapse.replication.tcp.streams import PresenceStream @@ -103,7 +103,9 @@ def __init__( prefilled_cache=presence_cache_prefill, ) - async def update_presence(self, presence_states) -> Tuple[int, int]: + async def update_presence( + self, presence_states: List[UserPresenceState] + ) -> Tuple[int, int]: assert self._can_persist_presence stream_ordering_manager = self._presence_id_gen.get_next_mult( @@ -121,7 +123,10 @@ async def update_presence(self, presence_states) -> Tuple[int, int]: return stream_orderings[-1], self._presence_id_gen.get_current_token() def _update_presence_txn( - self, txn: LoggingTransaction, stream_orderings, presence_states + self, + txn: LoggingTransaction, + stream_orderings: List[int], + presence_states: List[UserPresenceState], ) -> None: for stream_id, state in zip(stream_orderings, presence_states): txn.call_after( @@ -405,7 +410,13 @@ def take_presence_startup_info(self) -> List[UserPresenceState]: self._presence_on_startup = [] return active_on_startup - def process_replication_rows(self, stream_name, instance_name, token, rows) -> None: + def process_replication_rows( + self, + stream_name: str, + instance_name: str, + token: int, + rows: Iterable[Any], + ) -> None: if stream_name == PresenceStream.NAME: self._presence_id_gen.advance(instance_name, token) for row in rows: diff --git a/synapse/storage/databases/main/pusher.py b/synapse/storage/databases/main/pusher.py index cf64cd63a46f..91286c9b65c6 100644 --- a/synapse/storage/databases/main/pusher.py +++ b/synapse/storage/databases/main/pusher.py @@ -14,11 +14,25 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Tuple +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + cast, +) from synapse.push import PusherConfig, ThrottleParams from synapse.storage._base import SQLBaseStore, db_to_json -from synapse.storage.database import DatabasePool, LoggingDatabaseConnection +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.storage.util.id_generators import StreamIdGenerator from synapse.types import JsonDict from synapse.util import json_encoder @@ -117,7 +131,7 @@ async def get_pushers_by(self, keyvalues: Dict[str, Any]) -> Iterator[PusherConf return self._decode_pushers_rows(ret) async def get_all_pushers(self) -> Iterator[PusherConfig]: - def get_pushers(txn): + def get_pushers(txn: LoggingTransaction) -> Iterator[PusherConfig]: txn.execute("SELECT * FROM pushers") rows = self.db_pool.cursor_to_dict(txn) @@ -152,7 +166,9 @@ async def get_all_updated_pushers_rows( if last_id == current_id: return [], current_id, False - def get_all_updated_pushers_rows_txn(txn): + def get_all_updated_pushers_rows_txn( + txn: LoggingTransaction, + ) -> Tuple[List[Tuple[int, tuple]], int, bool]: sql = """ SELECT id, user_name, app_id, pushkey FROM pushers @@ -160,10 +176,13 @@ def get_all_updated_pushers_rows_txn(txn): ORDER BY id ASC LIMIT ? """ txn.execute(sql, (last_id, current_id, limit)) - updates = [ - (stream_id, (user_name, app_id, pushkey, False)) - for stream_id, user_name, app_id, pushkey in txn - ] + updates = cast( + List[Tuple[int, tuple]], + [ + (stream_id, (user_name, app_id, pushkey, False)) + for stream_id, user_name, app_id, pushkey in txn + ], + ) sql = """ SELECT stream_id, user_id, app_id, pushkey @@ -192,12 +211,12 @@ def get_all_updated_pushers_rows_txn(txn): ) @cached(num_args=1, max_entries=15000) - async def get_if_user_has_pusher(self, user_id: str): + async def get_if_user_has_pusher(self, user_id: str) -> None: # This only exists for the cachedList decorator raise NotImplementedError() async def update_pusher_last_stream_ordering( - self, app_id, pushkey, user_id, last_stream_ordering + self, app_id: str, pushkey: str, user_id: str, last_stream_ordering: int ) -> None: await self.db_pool.simple_update_one( "pushers", @@ -291,7 +310,7 @@ async def _remove_deactivated_pushers(self, progress: dict, batch_size: int) -> last_user = progress.get("last_user", "") - def _delete_pushers(txn) -> int: + def _delete_pushers(txn: LoggingTransaction) -> int: sql = """ SELECT name FROM users @@ -339,7 +358,7 @@ async def _remove_stale_pushers(self, progress: dict, batch_size: int) -> int: last_pusher = progress.get("last_pusher", 0) - def _delete_pushers(txn) -> int: + def _delete_pushers(txn: LoggingTransaction) -> int: sql = """ SELECT p.id, access_token FROM pushers AS p @@ -396,7 +415,7 @@ async def _remove_deleted_email_pushers( last_pusher = progress.get("last_pusher", 0) - def _delete_pushers(txn) -> int: + def _delete_pushers(txn: LoggingTransaction) -> int: sql = """ SELECT p.id, p.user_name, p.app_id, p.pushkey @@ -502,7 +521,7 @@ async def add_pusher( async def delete_pusher_by_app_id_pushkey_user_id( self, app_id: str, pushkey: str, user_id: str ) -> None: - def delete_pusher_txn(txn, stream_id): + def delete_pusher_txn(txn: LoggingTransaction, stream_id: int) -> None: self._invalidate_cache_and_stream( # type: ignore[attr-defined] txn, self.get_if_user_has_pusher, (user_id,) ) @@ -547,7 +566,7 @@ async def delete_all_pushers_for_user(self, user_id: str) -> None: # account. pushers = list(await self.get_pushers_by_user_id(user_id)) - def delete_pushers_txn(txn, stream_ids): + def delete_pushers_txn(txn: LoggingTransaction, stream_ids: List[int]) -> None: self._invalidate_cache_and_stream( # type: ignore[attr-defined] txn, self.get_if_user_has_pusher, (user_id,) ) diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index e653841fe503..5e340bd03df4 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -370,10 +370,10 @@ async def update_state_for_partial_state_event( def _update_state_for_partial_state_event_txn( self, - txn, + txn: LoggingTransaction, event: EventBase, context: EventContext, - ): + ) -> None: # we shouldn't have any outliers here assert not event.internal_metadata.is_outlier() diff --git a/synapse/storage/databases/main/ui_auth.py b/synapse/storage/databases/main/ui_auth.py index 2d339b60083b..f38bedbbcdae 100644 --- a/synapse/storage/databases/main/ui_auth.py +++ b/synapse/storage/databases/main/ui_auth.py @@ -131,7 +131,7 @@ async def mark_ui_auth_stage_complete( session_id: str, stage_type: str, result: Union[str, bool, JsonDict], - ): + ) -> None: """ Mark a session stage as completed. @@ -200,7 +200,9 @@ async def set_ui_auth_clientdict( desc="set_ui_auth_client_dict", ) - async def set_ui_auth_session_data(self, session_id: str, key: str, value: Any): + async def set_ui_auth_session_data( + self, session_id: str, key: str, value: Any + ) -> None: """ Store a key-value pair into the sessions data associated with this request. This data is stored server-side and cannot be modified by @@ -223,7 +225,7 @@ async def set_ui_auth_session_data(self, session_id: str, key: str, value: Any): def _set_ui_auth_session_data_txn( self, txn: LoggingTransaction, session_id: str, key: str, value: Any - ): + ) -> None: # Get the current value. result = cast( Dict[str, Any], @@ -275,7 +277,7 @@ async def add_user_agent_ip_to_ui_auth_session( session_id: str, user_agent: str, ip: str, - ): + ) -> None: """Add the given user agent / IP to the tracking table""" await self.db_pool.simple_upsert( table="ui_auth_sessions_ips", @@ -318,7 +320,7 @@ async def delete_old_ui_auth_sessions(self, expiration_time: int) -> None: def _delete_old_ui_auth_sessions_txn( self, txn: LoggingTransaction, expiration_time: int - ): + ) -> None: # Get the expired sessions. sql = "SELECT session_id FROM ui_auth_sessions WHERE creation_time <= ?" txn.execute(sql, [expiration_time]) From e8d1ec0e92da96a01b6c723fcdb4eac27f801e87 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 27 Apr 2022 13:57:53 +0100 Subject: [PATCH 074/263] Add option to enable token registration without requiring 3pids (#12526) --- changelog.d/12526.feature | 1 + docs/sample_config.yaml | 6 ++++++ synapse/config/registration.py | 9 +++++++++ synapse/handlers/ui_auth/checkers.py | 4 +++- synapse/rest/client/register.py | 7 ++++++- 5 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12526.feature diff --git a/changelog.d/12526.feature b/changelog.d/12526.feature new file mode 100644 index 000000000000..c01596282c9f --- /dev/null +++ b/changelog.d/12526.feature @@ -0,0 +1 @@ +Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index b8d8c0dbf0a1..67184c6b1ae1 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1323,6 +1323,12 @@ oembed: # #registration_requires_token: true +# Allow users to submit a token during registration to bypass any required 3pid +# steps configured in `registrations_require_3pid`. +# Defaults to false, requiring that registration tokens (if enabled) complete a 3pid flow. +# +#enable_registration_token_3pid_bypass: false + # If set, allows registration of standard or admin accounts by anyone who # has the shared secret, even if registration is otherwise disabled. # diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 39e9acb62a90..70eb7e6a9778 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -43,6 +43,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.registration_requires_token = config.get( "registration_requires_token", False ) + self.enable_registration_token_3pid_bypasss = config.get( + "enable_registration_token_3pid_bypasss", False + ) self.registration_shared_secret = config.get("registration_shared_secret") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) @@ -309,6 +312,12 @@ def generate_config_section( # #registration_requires_token: true + # Allow users to submit a token during registration to bypass any required 3pid + # steps configured in `registrations_require_3pid`. + # Defaults to false, requiring that registration tokens (if enabled) complete a 3pid flow. + # + #enable_registration_token_3pid_bypass: false + # If set, allows registration of standard or admin accounts by anyone who # has the shared secret, even if registration is otherwise disabled. # diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py index 472b029af3f6..e2a441066d1c 100644 --- a/synapse/handlers/ui_auth/checkers.py +++ b/synapse/handlers/ui_auth/checkers.py @@ -256,7 +256,9 @@ class RegistrationTokenAuthChecker(UserInteractiveAuthChecker): def __init__(self, hs: "HomeServer"): super().__init__(hs) self.hs = hs - self._enabled = bool(hs.config.registration.registration_requires_token) + self._enabled = bool( + hs.config.registration.registration_requires_token + ) or bool(hs.config.registration.enable_registration_token_3pid_bypasss) self.store = hs.get_datastores().main def is_enabled(self) -> bool: diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 70baf50fa473..13ef6b35a066 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -929,6 +929,10 @@ def _calculate_registration_flows( # always let users provide both MSISDN & email flows.append([LoginType.MSISDN, LoginType.EMAIL_IDENTITY]) + # Add a flow that doesn't require any 3pids, if the config requests it. + if config.registration.enable_registration_token_3pid_bypasss: + flows.append([LoginType.REGISTRATION_TOKEN]) + # Prepend m.login.terms to all flows if we're requiring consent if config.consent.user_consent_at_registration: for flow in flows: @@ -942,7 +946,8 @@ def _calculate_registration_flows( # Prepend registration token to all flows if we're requiring a token if config.registration.registration_requires_token: for flow in flows: - flow.insert(0, LoginType.REGISTRATION_TOKEN) + if LoginType.REGISTRATION_TOKEN not in flow: + flow.insert(0, LoginType.REGISTRATION_TOKEN) return flows From 8a23bde82352f18d7d6f098e3f3f0ce7099efb86 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 27 Apr 2022 09:00:07 -0400 Subject: [PATCH 075/263] Consistently use collections.abc.Mapping to check frozendict. (#12564) --- changelog.d/12564.misc | 1 + synapse/events/utils.py | 7 ++++--- synapse/handlers/relations.py | 4 ++-- synapse/storage/databases/main/state.py | 5 ++--- synapse/util/frozenutils.py | 3 ++- 5 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelog.d/12564.misc diff --git a/changelog.d/12564.misc b/changelog.d/12564.misc new file mode 100644 index 000000000000..207c3224645f --- /dev/null +++ b/changelog.d/12564.misc @@ -0,0 +1 @@ +Consistently check if an object is a `frozendict`. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index f8d3ba5456a5..a6c48308b310 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -27,7 +27,6 @@ ) import attr -from frozendict import frozendict from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.errors import Codes, SynapseError @@ -204,7 +203,9 @@ def _copy_field(src: JsonDict, dst: JsonDict, field: List[str]) -> None: key_to_move = field.pop(-1) sub_dict = src for sub_field in field: # e.g. sub_field => "content" - if sub_field in sub_dict and type(sub_dict[sub_field]) in [dict, frozendict]: + if sub_field in sub_dict and isinstance( + sub_dict[sub_field], collections.abc.Mapping + ): sub_dict = sub_dict[sub_field] else: return @@ -622,7 +623,7 @@ def validate_canonicaljson(value: Any) -> None: # Note that Infinity, -Infinity, and NaN are also considered floats. raise SynapseError(400, "Bad JSON value: float", Codes.BAD_JSON) - elif isinstance(value, (dict, frozendict)): + elif isinstance(value, collections.abc.Mapping): for v in value.values(): validate_canonicaljson(v) diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index 5efb5612736f..b5dc9f74b372 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import collections.abc import logging from typing import ( TYPE_CHECKING, @@ -24,7 +25,6 @@ ) import attr -from frozendict import frozendict from synapse.api.constants import RelationTypes from synapse.api.errors import SynapseError @@ -380,7 +380,7 @@ async def get_bundled_aggregations( # Do not bundle aggregations for an event which represents an edit or an # annotation. It does not make sense for them to have related events. relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, (dict, frozendict)): + if isinstance(relates_to, collections.abc.Mapping): relation_type = relates_to.get("rel_type") if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): continue diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 5e340bd03df4..18ae8aee295d 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import collections.abc import logging from typing import TYPE_CHECKING, Collection, Dict, Iterable, Optional, Set, Tuple -from frozendict import frozendict - from synapse.api.constants import EventTypes, Membership from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion @@ -160,7 +159,7 @@ async def get_room_predecessor(self, room_id: str) -> Optional[JsonMapping]: predecessor = create_event.content.get("predecessor", None) # Ensure the key is a dictionary - if not isinstance(predecessor, (dict, frozendict)): + if not isinstance(predecessor, collections.abc.Mapping): return None # The keys must be strings since the data is JSON. diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 9c405eb4d763..7223af1a36e7 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import collections.abc from typing import Any from frozendict import frozendict @@ -35,7 +36,7 @@ def freeze(o: Any) -> Any: def unfreeze(o: Any) -> Any: - if isinstance(o, (dict, frozendict)): + if isinstance(o, collections.abc.Mapping): return {k: unfreeze(v) for k, v in o.items()} if isinstance(o, (bytes, str)): From 646324437543c096e737777c81b4fe4b45c3e1a7 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 27 Apr 2022 14:03:44 +0100 Subject: [PATCH 076/263] Remove unused `# type: ignore`s (#12531) Over time we've begun to use newer versions of mypy, typeshed, stub packages---and of course we've improved our own annotations. This makes some type ignore comments no longer necessary. I have removed them. There was one exception: a module that imports `select.epoll`. The ignore is redundant on Linux, but I've kept it ignored for those of us who work on the source tree using not-Linux. (#11771) I'm more interested in the config line which enforces this. I want unused ignores to be reported, because I think it's useful feedback when annotating to know when you've fixed a problem you had to previously ignore. * Installing extras before typechecking Lacking an easy way to install all extras generically, let's bite the bullet and make install the hand-maintained `all` extra before typechecking. Now that https://github.com/matrix-org/backend-meta/pull/6 is merged to the release/v1 branch. --- .github/workflows/tests.yml | 8 ++------ changelog.d/12531.misc | 1 + mypy.ini | 6 ++++++ stubs/sortedcontainers/sorteddict.pyi | 4 +--- synapse/app/_base.py | 4 ++-- synapse/config/server.py | 6 ++---- synapse/federation/federation_server.py | 4 ++-- synapse/federation/transport/client.py | 10 +++++----- synapse/handlers/auth.py | 2 +- synapse/handlers/oidc.py | 2 +- synapse/handlers/search.py | 6 +++--- synapse/http/server.py | 4 ++-- synapse/module_api/__init__.py | 4 +++- .../databases/main/monthly_active_users.py | 6 +++--- synapse/storage/prepare_database.py | 4 ++-- synapse/util/caches/ttlcache.py | 2 +- tests/handlers/test_register.py | 9 +++------ tests/module_api/test_account_data_manager.py | 17 +++++++++++------ .../replication/test_federation_sender_shard.py | 8 ++++---- tests/test_utils/__init__.py | 8 ++++---- tests/test_utils/logging_setup.py | 2 +- 21 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 changelog.d/12531.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cad4cb6d77b3..efa35b71df18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,13 +20,9 @@ jobs: - run: scripts-dev/config-lint.sh lint: - # This does a vanilla `poetry install` - no extras. I'm slightly anxious - # that we might skip some typechecks on code that uses extras. However, - # I think the right way to fix this is to mark any extras needed for - # typechecking as development dependencies. To detect this, we ought to - # turn up mypy's strictness: disallow unknown imports and be accept fewer - # uses of `Any`. uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" + with: + typechecking-extras: "all" lint-crlf: runs-on: ubuntu-latest diff --git a/changelog.d/12531.misc b/changelog.d/12531.misc new file mode 100644 index 000000000000..412fc9b6dc70 --- /dev/null +++ b/changelog.d/12531.misc @@ -0,0 +1 @@ +Remove unused `# type: ignore`s. diff --git a/mypy.ini b/mypy.ini index a663bf69752a..280f1e898ec1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,6 +7,7 @@ show_error_codes = True show_traceback = True mypy_path = stubs warn_unreachable = True +warn_unused_ignores = True local_partial_types = True no_implicit_optional = True @@ -134,6 +135,11 @@ disallow_untyped_defs = True [mypy-synapse.metrics.*] disallow_untyped_defs = True +[mypy-synapse.metrics._reactor_metrics] +# This module imports select.epoll. That exists on Linux, but doesn't on macOS. +# See https://github.com/matrix-org/synapse/pull/11771. +warn_unused_ignores = False + [mypy-synapse.module_api.*] disallow_untyped_defs = True diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index e18d617281ab..3a4f9c307685 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -115,9 +115,7 @@ class SortedKeysView(KeysView[_KT_co], Sequence[_KT_co]): def __getitem__(self, index: slice) -> List[_KT_co]: ... def __delitem__(self, index: Union[int, slice]) -> None: ... -class SortedItemsView( # type: ignore - ItemsView[_KT_co, _VT_co], Sequence[Tuple[_KT_co, _VT_co]] -): +class SortedItemsView(ItemsView[_KT_co, _VT_co], Sequence[Tuple[_KT_co, _VT_co]]): def __iter__(self) -> Iterator[Tuple[_KT_co, _VT_co]]: ... @overload def __getitem__(self, index: int) -> Tuple[_KT_co, _VT_co]: ... diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 37321f913399..d28b87a3f4d8 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -48,7 +48,6 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.threadpool import ThreadPool -import synapse from synapse.api.constants import MAX_PDU_SIZE from synapse.app import check_bind_error from synapse.app.phone_stats_home import start_phone_stats_home @@ -60,6 +59,7 @@ from synapse.events.third_party_rules import load_legacy_third_party_event_rules from synapse.handlers.auth import load_legacy_password_auth_providers from synapse.logging.context import PreserveLoggingContext +from synapse.logging.opentracing import init_tracer from synapse.metrics import install_gc_manager, register_threadpool from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.metrics.jemalloc import setup_jemalloc_stats @@ -431,7 +431,7 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: refresh_certificate(hs) # Start the tracer - synapse.logging.opentracing.init_tracer(hs) # type: ignore[attr-defined] # noqa + init_tracer(hs) # noqa # Instantiate the modules so they can register their web resources to the module API # before we start the listeners. diff --git a/synapse/config/server.py b/synapse/config/server.py index d771045b522a..b6cd32641688 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -186,7 +186,7 @@ def generate_ip_set( class HttpResourceConfig: names: List[str] = attr.ib( factory=list, - validator=attr.validators.deep_iterable(attr.validators.in_(KNOWN_RESOURCES)), # type: ignore + validator=attr.validators.deep_iterable(attr.validators.in_(KNOWN_RESOURCES)), ) compress: bool = attr.ib( default=False, @@ -231,9 +231,7 @@ class ManholeConfig: class LimitRemoteRoomsConfig: enabled: bool = attr.ib(validator=attr.validators.instance_of(bool), default=False) complexity: Union[float, int] = attr.ib( - validator=attr.validators.instance_of( - (float, int) # type: ignore[arg-type] # noqa - ), + validator=attr.validators.instance_of((float, int)), # noqa default=1.0, ) complexity_error: str = attr.ib( diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index beab1227b84c..884b5d60b4f9 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -268,8 +268,8 @@ async def on_incoming_transaction( transaction_id=transaction_id, destination=destination, origin=origin, - origin_server_ts=transaction_data.get("origin_server_ts"), # type: ignore - pdus=transaction_data.get("pdus"), # type: ignore + origin_server_ts=transaction_data.get("origin_server_ts"), # type: ignore[arg-type] + pdus=transaction_data.get("pdus"), edus=transaction_data.get("edus"), ) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 1421050b9a53..9ce06dfa28ba 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -229,21 +229,21 @@ async def send_transaction( """ logger.debug( "send_data dest=%s, txid=%s", - transaction.destination, # type: ignore - transaction.transaction_id, # type: ignore + transaction.destination, + transaction.transaction_id, ) - if transaction.destination == self.server_name: # type: ignore + if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") # FIXME: This is only used by the tests. The actual json sent is # generated by the json_data_callback. json_data = transaction.get_dict() - path = _create_v1_path("/send/%s", transaction.transaction_id) # type: ignore + path = _create_v1_path("/send/%s", transaction.transaction_id) return await self.client.put_json( - transaction.destination, # type: ignore + transaction.destination, path=path, data=json_data, json_data_callback=json_data_callback, diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 86991d26ce79..22678d486d73 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -481,7 +481,7 @@ async def check_ui_auth( sid = authdict["session"] # Convert the URI and method to strings. - uri = request.uri.decode("utf-8") # type: ignore + uri = request.uri.decode("utf-8") method = request.method.decode("utf-8") # If there's no session ID, create a new session. diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 724b9cfcb4bb..f6ffb7d18d91 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -966,7 +966,7 @@ async def oidc_response_to_user_attributes(failures: int) -> UserAttributes: "Mapping provider does not support de-duplicating Matrix IDs" ) - attributes = await self._user_mapping_provider.map_user_attributes( # type: ignore + attributes = await self._user_mapping_provider.map_user_attributes( userinfo, token ) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 102dd4b57dea..5619f8f50e03 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -357,7 +357,7 @@ async def _search( itertools.chain( # The events_before and events_after for each context. itertools.chain.from_iterable( - itertools.chain(context["events_before"], context["events_after"]) # type: ignore[arg-type] + itertools.chain(context["events_before"], context["events_after"]) for context in contexts.values() ), # The returned events. @@ -373,10 +373,10 @@ async def _search( for context in contexts.values(): context["events_before"] = self._event_serializer.serialize_events( - context["events_before"], time_now, bundle_aggregations=aggregations # type: ignore[arg-type] + context["events_before"], time_now, bundle_aggregations=aggregations ) context["events_after"] = self._event_serializer.serialize_events( - context["events_after"], time_now, bundle_aggregations=aggregations # type: ignore[arg-type] + context["events_after"], time_now, bundle_aggregations=aggregations ) results = [ diff --git a/synapse/http/server.py b/synapse/http/server.py index 31ca84188975..b8a7a0f5df60 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -295,7 +295,7 @@ async def _async_render(self, request: SynapseRequest) -> Optional[Tuple[int, An if isawaitable(raw_callback_return): callback_return = await raw_callback_return else: - callback_return = raw_callback_return # type: ignore + callback_return = raw_callback_return return callback_return @@ -469,7 +469,7 @@ async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]: if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)): callback_return = await raw_callback_return else: - callback_return = raw_callback_return # type: ignore + callback_return = raw_callback_return return callback_return diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 8f9e62927465..7e6e23db7342 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -109,6 +109,7 @@ from synapse.types import ( DomainSpecificString, JsonDict, + JsonMapping, Requester, StateMap, UserID, @@ -151,6 +152,7 @@ "PRESENCE_ALL_USERS", "LoginResponse", "JsonDict", + "JsonMapping", "EventBase", "StateMap", "ProfileInfo", @@ -1419,7 +1421,7 @@ def _validate_user_id(self, user_id: str) -> None: f"{user_id} is not local to this homeserver; can't access account data for remote users." ) - async def get_global(self, user_id: str, data_type: str) -> Optional[JsonDict]: + async def get_global(self, user_id: str, data_type: str) -> Optional[JsonMapping]: """ Gets some global account data, of a specified type, for the specified user. diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index 4f1c22c71b0a..5beb8f1d4bc8 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -232,10 +232,10 @@ def _reap_users(txn: LoggingTransaction, reserved_users: List[str]) -> None: # is racy. # Have resolved to invalidate the whole cache for now and do # something about it if and when the perf becomes significant - self._invalidate_all_cache_and_stream( # type: ignore[attr-defined] + self._invalidate_all_cache_and_stream( txn, self.user_last_seen_monthly_active ) - self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ()) # type: ignore[attr-defined] + self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ()) reserved_users = await self.get_registered_reserved_users() await self.db_pool.runInteraction( @@ -363,7 +363,7 @@ async def populate_monthly_active_users(self, user_id: str) -> None: if self._limit_usage_by_mau or self._mau_stats_only: # Trial users and guests should not be included as part of MAU group - is_guest = await self.is_guest(user_id) # type: ignore[attr-defined] + is_guest = await self.is_guest(user_id) if is_guest: return is_trial = await self.is_trial_user(user_id) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index e3153d1a4aa9..546d6bae6e56 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -501,11 +501,11 @@ def _upgrade_existing_database( if hasattr(module, "run_create"): logger.info("Running %s:run_create", relative_path) - module.run_create(cur, database_engine) # type: ignore + module.run_create(cur, database_engine) if not is_empty and hasattr(module, "run_upgrade"): logger.info("Running %s:run_upgrade", relative_path) - module.run_upgrade(cur, database_engine, config=config) # type: ignore + module.run_upgrade(cur, database_engine, config=config) elif ext == ".pyc" or file_name == "__pycache__": # Sometimes .pyc files turn up anyway even though we've # disabled their generation; e.g. from distribution package diff --git a/synapse/util/caches/ttlcache.py b/synapse/util/caches/ttlcache.py index 0b9ac26b6949..f6b3ee31e4b8 100644 --- a/synapse/util/caches/ttlcache.py +++ b/synapse/util/caches/ttlcache.py @@ -107,7 +107,7 @@ def get_with_expiry(self, key: KT) -> Tuple[VT, float, float]: self._metrics.inc_hits() return e.value, e.expiry_time, e.ttl - def pop(self, key: KT, default: T = SENTINEL) -> Union[VT, T]: # type: ignore + def pop(self, key: KT, default: T = SENTINEL) -> Union[VT, T]: """Remove a value from the cache If key is in the cache, remove it and return its value, else return default. diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 45fd30cf4369..b6ba19c73903 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -193,8 +193,7 @@ def test_mau_limits_when_disabled(self): @override_config({"limit_usage_by_mau": True}) def test_get_or_create_user_mau_not_blocked(self): - # Type ignore: mypy doesn't like us assigning to methods. - self.store.count_monthly_users = Mock( # type: ignore[assignment] + self.store.count_monthly_users = Mock( return_value=make_awaitable(self.hs.config.server.max_mau_value - 1) ) # Ensure does not throw exception @@ -202,8 +201,7 @@ def test_get_or_create_user_mau_not_blocked(self): @override_config({"limit_usage_by_mau": True}) def test_get_or_create_user_mau_blocked(self): - # Type ignore: mypy doesn't like us assigning to methods. - self.store.get_monthly_active_count = Mock( # type: ignore[assignment] + self.store.get_monthly_active_count = Mock( return_value=make_awaitable(self.lots_of_users) ) self.get_failure( @@ -211,8 +209,7 @@ def test_get_or_create_user_mau_blocked(self): ResourceLimitError, ) - # Type ignore: mypy doesn't like us assigning to methods. - self.store.get_monthly_active_count = Mock( # type: ignore[assignment] + self.store.get_monthly_active_count = Mock( return_value=make_awaitable(self.hs.config.server.max_mau_value) ) self.get_failure( diff --git a/tests/module_api/test_account_data_manager.py b/tests/module_api/test_account_data_manager.py index bec018d9e702..89009bea8c32 100644 --- a/tests/module_api/test_account_data_manager.py +++ b/tests/module_api/test_account_data_manager.py @@ -11,8 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.test.proto_helpers import MemoryReactor + from synapse.api.errors import SynapseError from synapse.rest import admin +from synapse.server import HomeServer +from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -22,7 +26,9 @@ class ModuleApiTestCase(HomeserverTestCase): admin.register_servlets, ] - def prepare(self, reactor, clock, homeserver) -> None: + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: self._store = homeserver.get_datastores().main self._module_api = homeserver.get_module_api() self._account_data_mgr = self._module_api.account_data_manager @@ -91,7 +97,7 @@ def test_get_global_no_mutability(self) -> None: ) with self.assertRaises(TypeError): # This throws an exception because it's a frozen dict. - the_data["wombat"] = False + the_data["wombat"] = False # type: ignore[index] def test_put_global(self) -> None: """ @@ -143,15 +149,14 @@ def test_put_global_validation(self) -> None: with self.assertRaises(TypeError): # The account data type must be a string. self.get_success_or_raise( - self._module_api.account_data_manager.put_global( - self.user_id, 42, {} # type: ignore - ) + self._module_api.account_data_manager.put_global(self.user_id, 42, {}) # type: ignore[arg-type] ) with self.assertRaises(TypeError): # The account data dict must be a dict. + # noinspection PyTypeChecker self.get_success_or_raise( self._module_api.account_data_manager.put_global( - self.user_id, "test.data", 42 # type: ignore + self.user_id, "test.data", 42 # type: ignore[arg-type] ) ) diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index ba1a63c0d667..6104a55aa1a9 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -102,8 +102,8 @@ def test_send_event_sharded(self): for i in range(20): server_name = "other_server_%d" % (i,) room = self.create_room_with_remote_server(user, token, server_name) - mock_client1.reset_mock() # type: ignore[attr-defined] - mock_client2.reset_mock() # type: ignore[attr-defined] + mock_client1.reset_mock() + mock_client2.reset_mock() self.create_and_send_event(room, UserID.from_string(user)) self.replicate() @@ -167,8 +167,8 @@ def test_send_typing_sharded(self): for i in range(20): server_name = "other_server_%d" % (i,) room = self.create_room_with_remote_server(user, token, server_name) - mock_client1.reset_mock() # type: ignore[attr-defined] - mock_client2.reset_mock() # type: ignore[attr-defined] + mock_client1.reset_mock() + mock_client2.reset_mock() self.get_success( typing_handler.started_typing( diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index f05a373aa083..0d0d6faf0d3a 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -52,7 +52,7 @@ def make_awaitable(result: TV) -> Awaitable[TV]: This uses Futures as they can be awaited multiple times so can be returned to multiple callers. """ - future = Future() # type: ignore + future: Future[TV] = Future() future.set_result(result) return future @@ -69,7 +69,7 @@ def setup_awaitable_errors() -> Callable[[], None]: # State shared between unraisablehook and check_for_unraisable_exceptions. unraisable_exceptions = [] - orig_unraisablehook = sys.unraisablehook # type: ignore + orig_unraisablehook = sys.unraisablehook def unraisablehook(unraisable): unraisable_exceptions.append(unraisable.exc_value) @@ -78,11 +78,11 @@ def cleanup(): """ A method to be used as a clean-up that fails a test-case if there are any new unraisable exceptions. """ - sys.unraisablehook = orig_unraisablehook # type: ignore + sys.unraisablehook = orig_unraisablehook if unraisable_exceptions: raise unraisable_exceptions.pop() - sys.unraisablehook = unraisablehook # type: ignore + sys.unraisablehook = unraisablehook return cleanup diff --git a/tests/test_utils/logging_setup.py b/tests/test_utils/logging_setup.py index 51a197a8c621..9228454c9e78 100644 --- a/tests/test_utils/logging_setup.py +++ b/tests/test_utils/logging_setup.py @@ -27,7 +27,7 @@ class ToTwistedHandler(logging.Handler): def emit(self, record): log_entry = self.format(record) log_level = record.levelname.lower().replace("warning", "warn") - self.tx_log.emit( # type: ignore + self.tx_log.emit( twisted.logger.LogLevel.levelWithName(log_level), "{entry}", entry=log_entry ) From 30c8e7e408322967e5beb2a64ef5f796cb8df226 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 27 Apr 2022 14:10:31 +0100 Subject: [PATCH 077/263] Make `scripts-dev` pass `mypy --disallow-untyped-defs` (#12356) Not enforced in config yet. One day. --- changelog.d/12356.misc | 1 + mypy.ini | 10 +++-- poetry.lock | 25 ++++++++++--- pyproject.toml | 4 +- scripts-dev/build_debian_packages.py | 21 +++++++---- scripts-dev/federation_client.py | 27 +++++++++----- scripts-dev/mypy_synapse_plugin.py | 4 +- scripts-dev/release.py | 55 +++++++++++++++++----------- scripts-dev/sign_json.py | 2 +- 9 files changed, 96 insertions(+), 53 deletions(-) create mode 100644 changelog.d/12356.misc diff --git a/changelog.d/12356.misc b/changelog.d/12356.misc new file mode 100644 index 000000000000..43e192910602 --- /dev/null +++ b/changelog.d/12356.misc @@ -0,0 +1 @@ +Fix scripts-dev to pass typechecking. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 280f1e898ec1..ef28216418a5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -24,10 +24,6 @@ files = # https://docs.python.org/3/library/re.html#re.X exclude = (?x) ^( - |scripts-dev/build_debian_packages.py - |scripts-dev/federation_client.py - |scripts-dev/release.py - |synapse/storage/databases/__init__.py |synapse/storage/databases/main/cache.py |synapse/storage/databases/main/devices.py @@ -308,6 +304,9 @@ ignore_missing_imports = True [mypy-pympler.*] ignore_missing_imports = True +[mypy-redbaron.*] +ignore_missing_imports = True + [mypy-rust_python_jaeger_reporter.*] ignore_missing_imports = True @@ -323,6 +322,9 @@ ignore_missing_imports = True [mypy-signedjson.*] ignore_missing_imports = True +[mypy-srvlookup.*] +ignore_missing_imports = True + [mypy-treq.*] ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 8c7af1fa1eac..e27a44989cd4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -309,14 +309,15 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.14" -description = "Python Git Library" +version = "3.1.27" +description = "GitPython is a python library used to interact with Git repositories" category = "dev" optional = false -python-versions = ">=3.4" +python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} [[package]] name = "hiredis" @@ -1315,6 +1316,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-commonmark" +version = "0.9.2" +description = "Typing stubs for commonmark" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-cryptography" version = "3.3.15" @@ -1553,7 +1562,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "f482a4f594a165dfe01ce253a22510d5faf38647ab0dcebc35789350cafd9bf0" +content-hash = "3825cef058b8c9f520ef4b7acb92519be95db9a663a61c2e89a5fe431ed55655" [metadata.files] attrs = [ @@ -1766,8 +1775,8 @@ gitdb = [ {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, ] gitpython = [ - {file = "GitPython-3.1.14-py3-none-any.whl", hash = "sha256:3283ae2fba31c913d857e12e5ba5f9a7772bbc064ae2bb09efafa71b0dd4939b"}, - {file = "GitPython-3.1.14.tar.gz", hash = "sha256:be27633e7509e58391f10207cd32b2a6cf5b908f92d9cd30da2e514e1137af61"}, + {file = "GitPython-3.1.27-py3-none-any.whl", hash = "sha256:5b68b000463593e05ff2b261acff0ff0972df8ab1b70d3cdbd41b546c8b8fc3d"}, + {file = "GitPython-3.1.27.tar.gz", hash = "sha256:1c885ce809e8ba2d88a29befeb385fcea06338d3640712b59ca623c220bb5704"}, ] hiredis = [ {file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"}, @@ -2588,6 +2597,10 @@ types-bleach = [ {file = "types-bleach-4.1.4.tar.gz", hash = "sha256:2d30c2c4fb6854088ac636471352c9a51bf6c089289800d2a8060820a01cd43a"}, {file = "types_bleach-4.1.4-py3-none-any.whl", hash = "sha256:edffe173ed6d7b6f3543036a96204a9319c3bf6c3645917b14274e43f000cc9b"}, ] +types-commonmark = [ + {file = "types-commonmark-0.9.2.tar.gz", hash = "sha256:b894b67750c52fd5abc9a40a9ceb9da4652a391d75c1b480bba9cef90f19fc86"}, + {file = "types_commonmark-0.9.2-py3-none-any.whl", hash = "sha256:56f20199a1f9a2924443211a0ef97f8b15a8a956a7f4e9186be6950bf38d6d02"}, +] types-cryptography = [ {file = "types-cryptography-3.3.15.tar.gz", hash = "sha256:a7983a75a7b88a18f88832008f0ef140b8d1097888ec1a0824ec8fb7e105273b"}, {file = "types_cryptography-3.3.15-py3-none-any.whl", hash = "sha256:d9b0dd5465d7898d400850e7f35e5518aa93a7e23d3e11757cd81b4777089046"}, diff --git a/pyproject.toml b/pyproject.toml index bdded7843440..e6f2dc16cd16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -251,6 +251,7 @@ flake8 = "*" mypy = "==0.931" mypy-zope = "==0.3.5" types-bleach = ">=4.1.0" +types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" types-opentracing = ">=2.4.2" types-Pillow = ">=8.3.4" @@ -270,7 +271,8 @@ idna = ">=2.5" # The following are used by the release script click = "==8.1.0" -GitPython = "==3.1.14" +# GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. +GitPython = ">=3.1.20" commonmark = "==0.9.1" pygithub = "==1.55" # The following are executed as commands by the release script. diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index e3e68786867e..38564893e95b 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -17,7 +17,8 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor -from typing import Optional, Sequence +from types import FrameType +from typing import Collection, Optional, Sequence, Set DISTS = ( "debian:buster", # oldstable: EOL 2022-08 @@ -41,15 +42,17 @@ class Builder(object): def __init__( - self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None + self, + redirect_stdout: bool = False, + docker_build_args: Optional[Sequence[str]] = None, ): self.redirect_stdout = redirect_stdout self._docker_build_args = tuple(docker_build_args or ()) - self.active_containers = set() + self.active_containers: Set[str] = set() self._lock = threading.Lock() self._failed = False - def run_build(self, dist, skip_tests=False): + def run_build(self, dist: str, skip_tests: bool = False) -> None: """Build deb for a single distribution""" if self._failed: @@ -63,7 +66,7 @@ def run_build(self, dist, skip_tests=False): self._failed = True raise - def _inner_build(self, dist, skip_tests=False): + def _inner_build(self, dist: str, skip_tests: bool = False) -> None: tag = dist.split(":", 1)[1] # Make the dir where the debs will live. @@ -138,7 +141,7 @@ def _inner_build(self, dist, skip_tests=False): stdout.close() print("Completed build of %s" % (dist,)) - def kill_containers(self): + def kill_containers(self) -> None: with self._lock: active = list(self.active_containers) @@ -156,8 +159,10 @@ def kill_containers(self): self.active_containers.remove(c) -def run_builds(builder, dists, jobs=1, skip_tests=False): - def sig(signum, _frame): +def run_builds( + builder: Builder, dists: Collection[str], jobs: int = 1, skip_tests: bool = False +) -> None: + def sig(signum: int, _frame: Optional[FrameType]) -> None: print("Caught SIGINT") builder.kill_containers() diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 079d2f5ed061..763dd02c477e 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -38,7 +38,7 @@ import base64 import json import sys -from typing import Any, Optional +from typing import Any, Dict, Optional, Tuple from urllib import parse as urlparse import requests @@ -47,13 +47,14 @@ import srvlookup import yaml from requests.adapters import HTTPAdapter +from urllib3 import HTTPConnectionPool # uncomment the following to enable debug logging of http requests # from httplib import HTTPConnection # HTTPConnection.debuglevel = 1 -def encode_base64(input_bytes): +def encode_base64(input_bytes: bytes) -> str: """Encode bytes as a base64 string without any padding.""" input_len = len(input_bytes) @@ -63,7 +64,7 @@ def encode_base64(input_bytes): return output_string -def encode_canonical_json(value): +def encode_canonical_json(value: object) -> bytes: return json.dumps( value, # Encode code-points outside of ASCII as UTF-8 rather than \u escapes @@ -130,7 +131,7 @@ def request( sig, destination, ) - authorization_headers.append(header.encode("ascii")) + authorization_headers.append(header) print("Authorization: %s" % header, file=sys.stderr) dest = "matrix://%s%s" % (destination, path) @@ -139,7 +140,10 @@ def request( s = requests.Session() s.mount("matrix://", MatrixConnectionAdapter()) - headers = {"Host": destination, "Authorization": authorization_headers[0]} + headers: Dict[str, str] = { + "Host": destination, + "Authorization": authorization_headers[0], + } if method == "POST": headers["Content-Type"] = "application/json" @@ -154,7 +158,7 @@ def request( ) -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="Signs and sends a federation request to a matrix homeserver" ) @@ -212,6 +216,7 @@ def main(): if not args.server_name or not args.signing_key: read_args_from_config(args) + assert isinstance(args.signing_key, str) algorithm, version, key_base64 = args.signing_key.split() key = signedjson.key.decode_signing_key_base64(algorithm, version, key_base64) @@ -233,7 +238,7 @@ def main(): print("") -def read_args_from_config(args): +def read_args_from_config(args: argparse.Namespace) -> None: with open(args.config, "r") as fh: config = yaml.safe_load(fh) @@ -250,7 +255,7 @@ def read_args_from_config(args): class MatrixConnectionAdapter(HTTPAdapter): @staticmethod - def lookup(s, skip_well_known=False): + def lookup(s: str, skip_well_known: bool = False) -> Tuple[str, int]: if s[-1] == "]": # ipv6 literal (with no port) return s, 8448 @@ -276,7 +281,7 @@ def lookup(s, skip_well_known=False): return s, 8448 @staticmethod - def get_well_known(server_name): + def get_well_known(server_name: str) -> Optional[str]: uri = "https://%s/.well-known/matrix/server" % (server_name,) print("fetching %s" % (uri,), file=sys.stderr) @@ -299,7 +304,9 @@ def get_well_known(server_name): print("Invalid response from %s: %s" % (uri, e), file=sys.stderr) return None - def get_connection(self, url, proxies=None): + def get_connection( + self, url: str, proxies: Optional[Dict[str, str]] = None + ) -> HTTPConnectionPool: parsed = urlparse.urlparse(url) (host, port) = self.lookup(parsed.netloc) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 1217e148747d..c775865212ee 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -16,7 +16,7 @@ can crop up, e.g the cache descriptors. """ -from typing import Callable, Optional +from typing import Callable, Optional, Type from mypy.nodes import ARG_NAMED_OPT from mypy.plugin import MethodSigContext, Plugin @@ -94,7 +94,7 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType: return signature -def plugin(version: str): +def plugin(version: str) -> Type[SynapsePlugin]: # This is the entry point of the plugin, and let's us deal with the fact # that the mypy plugin interface is *not* stable by looking at the version # string. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 9d7c7c445fc0..14f3f3a45d8a 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -25,7 +25,7 @@ import urllib.request from os import path from tempfile import TemporaryDirectory -from typing import List, Optional +from typing import Any, List, Optional, cast import attr import click @@ -36,7 +36,9 @@ from packaging import version -def run_until_successful(command, *args, **kwargs): +def run_until_successful( + command: str, *args: Any, **kwargs: Any +) -> subprocess.CompletedProcess: while True: completed_process = subprocess.run(command, *args, **kwargs) exit_code = completed_process.returncode @@ -50,7 +52,7 @@ def run_until_successful(command, *args, **kwargs): @click.group() -def cli(): +def cli() -> None: """An interactive script to walk through the parts of creating a release. Requires the dev dependencies be installed, which can be done via: @@ -81,7 +83,7 @@ def cli(): @cli.command() -def prepare(): +def prepare() -> None: """Do the initial stages of creating a release, including creating release branch, updating changelog and pushing to GitHub. """ @@ -161,7 +163,9 @@ def prepare(): click.get_current_context().abort() # Switch to the release branch. - parsed_new_version: version.Version = version.parse(new_version) + # Cast safety: parse() won't return a version.LegacyVersion from our + # version string format. + parsed_new_version = cast(version.Version, version.parse(new_version)) # We assume for debian changelogs that we only do RCs or full releases. assert not parsed_new_version.is_devrelease @@ -176,7 +180,6 @@ def prepare(): # If the release branch only exists on the remote we check it out # locally. repo.git.checkout(release_branch_name) - release_branch = repo.active_branch else: # If a branch doesn't exist we create one. We ask which one branch it # should be based off, defaulting to sensible values depending on the @@ -198,13 +201,15 @@ def prepare(): click.get_current_context().abort() # Check out the base branch and ensure it's up to date - repo.head.reference = base_branch + repo.head.set_reference(base_branch, "check out the base branch") repo.head.reset(index=True, working_tree=True) if not base_branch.is_remote(): update_branch(repo) # Create the new release branch - release_branch = repo.create_head(release_branch_name, commit=base_branch) + # Type ignore will no longer be needed after GitPython 3.1.28. + # See https://github.com/gitpython-developers/GitPython/pull/1419 + repo.create_head(release_branch_name, commit=base_branch) # type: ignore[arg-type] # Switch to the release branch and ensure it's up to date. repo.git.checkout(release_branch_name) @@ -265,7 +270,7 @@ def prepare(): @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"]) -def tag(gh_token: Optional[str]): +def tag(gh_token: Optional[str]) -> None: """Tags the release and generates a draft GitHub release""" # Make sure we're in a git repo. @@ -293,7 +298,12 @@ def tag(gh_token: Optional[str]): click.echo_via_pager(changes) if click.confirm("Edit text?", default=False): - changes = click.edit(changes, require_save=False) + edited_changes = click.edit(changes, require_save=False) + # This assert is for mypy's benefit. click's docs are a little unclear, but + # when `require_save=False`, not saving the temp file in the editor returns + # the original string. + assert edited_changes is not None + changes = edited_changes repo.create_tag(tag_name, message=changes, sign=True) @@ -347,7 +357,7 @@ def tag(gh_token: Optional[str]): @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) -def publish(gh_token: str): +def publish(gh_token: str) -> None: """Publish release.""" # Make sure we're in a git repo. @@ -390,7 +400,7 @@ def publish(gh_token: str): @cli.command() -def upload(): +def upload() -> None: """Upload release to pypi.""" current_version = get_package_version() @@ -418,7 +428,7 @@ def upload(): @cli.command() -def announce(): +def announce() -> None: """Generate markdown to announce the release.""" current_version = get_package_version() @@ -461,18 +471,19 @@ def get_package_version() -> version.Version: def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]: """Find the branch/ref, looking first locally then in the remote.""" - if ref_name in repo.refs: - return repo.refs[ref_name] + if ref_name in repo.references: + return repo.references[ref_name] elif ref_name in repo.remote().refs: return repo.remote().refs[ref_name] else: return None -def update_branch(repo: git.Repo): +def update_branch(repo: git.Repo) -> None: """Ensure branch is up to date if it has a remote""" - if repo.active_branch.tracking_branch(): - repo.git.merge(repo.active_branch.tracking_branch().name) + tracking_branch = repo.active_branch.tracking_branch() + if tracking_branch: + repo.git.merge(tracking_branch.name) def get_changes_for_version(wanted_version: version.Version) -> str: @@ -536,7 +547,9 @@ class VersionSection: return "\n".join(version_changelog) -def generate_and_write_changelog(current_version: version.Version, new_version: str): +def generate_and_write_changelog( + current_version: version.Version, new_version: str +) -> None: # We do this by getting a draft so that we can edit it before writing to the # changelog. result = run_until_successful( @@ -558,8 +571,8 @@ def generate_and_write_changelog(current_version: version.Version, new_version: f.write(existing_content) # Remove all the news fragments - for f in glob.iglob("changelog.d/*.*"): - os.remove(f) + for filename in glob.iglob("changelog.d/*.*"): + os.remove(filename) if __name__ == "__main__": diff --git a/scripts-dev/sign_json.py b/scripts-dev/sign_json.py index 945954310610..bb217799fbcc 100755 --- a/scripts-dev/sign_json.py +++ b/scripts-dev/sign_json.py @@ -27,7 +27,7 @@ from synapse.util import json_encoder -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="""Adds a signature to a JSON object. From d743b25c8f0c96516d9c374ee946e93e937e2c5b Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 27 Apr 2022 14:39:41 +0100 Subject: [PATCH 078/263] Use supervisord to supervise Postgres and Caddy in the Complement image. (#12480) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/12480.misc | 1 + docker/Dockerfile-workers | 3 +++ docker/complement/SynapseWorkers.Dockerfile | 5 ++++- .../conf-workers/caddy.supervisord.conf | 7 +++++++ .../conf-workers/postgres.supervisord.conf | 16 ++++++++++++++++ .../start-complement-synapse-workers.sh | 6 ------ docker/conf/log.config | 4 ---- docker/configure_workers_and_start.py | 2 +- docker/prefix-log | 12 ++++++++++++ 9 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 changelog.d/12480.misc create mode 100644 docker/complement/conf-workers/caddy.supervisord.conf create mode 100644 docker/complement/conf-workers/postgres.supervisord.conf create mode 100755 docker/prefix-log diff --git a/changelog.d/12480.misc b/changelog.d/12480.misc new file mode 100644 index 000000000000..18a85e7b1579 --- /dev/null +++ b/changelog.d/12480.misc @@ -0,0 +1 @@ +Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. \ No newline at end of file diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 9ccb2b22a750..24b03585f9a2 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -20,6 +20,9 @@ RUN rm /etc/nginx/sites-enabled/default # Copy Synapse worker, nginx and supervisord configuration template files COPY ./docker/conf-workers/* /conf/ +# Copy a script to prefix log lines with the supervisor program name +COPY ./docker/prefix-log /usr/local/bin/ + # Expose nginx listener port EXPOSE 8080/tcp diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 65df2d114d58..9a4438e7303b 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -34,13 +34,16 @@ WORKDIR /data # Copy the caddy config COPY conf-workers/caddy.complement.json /root/caddy.json +COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf +COPY conf-workers/caddy.supervisord.conf /etc/supervisor/conf.d/caddy.conf + # Copy the entrypoint COPY conf-workers/start-complement-synapse-workers.sh / # Expose caddy's listener ports EXPOSE 8008 8448 -ENTRYPOINT /start-complement-synapse-workers.sh +ENTRYPOINT ["/start-complement-synapse-workers.sh"] # Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ diff --git a/docker/complement/conf-workers/caddy.supervisord.conf b/docker/complement/conf-workers/caddy.supervisord.conf new file mode 100644 index 000000000000..d9ddb51dac46 --- /dev/null +++ b/docker/complement/conf-workers/caddy.supervisord.conf @@ -0,0 +1,7 @@ +[program:caddy] +command=/usr/local/bin/prefix-log /root/caddy run --config /root/caddy.json +autorestart=unexpected +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf-workers/postgres.supervisord.conf new file mode 100644 index 000000000000..5608342d1a9e --- /dev/null +++ b/docker/complement/conf-workers/postgres.supervisord.conf @@ -0,0 +1,16 @@ +[program:postgres] +command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground + +# Lower priority number = starts first +priority=1 + +autorestart=unexpected +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +# Use 'Fast Shutdown' mode which aborts current transactions and closes connections quickly. +# (Default (TERM) is 'Smart Shutdown' which stops accepting new connections but +# lets existing connections close gracefully.) +stopsignal=INT diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh index 2c1e05bd6221..b9a6b55bbe8e 100755 --- a/docker/complement/conf-workers/start-complement-synapse-workers.sh +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -12,12 +12,6 @@ function log { # Replace the server name in the caddy config sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json -log "starting postgres" -pg_ctlcluster 13 main start - -log "starting caddy" -/root/caddy start --config /root/caddy.json - # Set the server name of the homeserver export SYNAPSE_SERVER_NAME=${SERVER_NAME} diff --git a/docker/conf/log.config b/docker/conf/log.config index 7a216a36a046..dc8c70befd40 100644 --- a/docker/conf/log.config +++ b/docker/conf/log.config @@ -2,11 +2,7 @@ version: 1 formatters: precise: -{% if worker_name %} - format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -{% else %} format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -{% endif %} handlers: {% if LOG_FILE_PATH %} diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3bda6c300ba8..33fc20d2182e 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -171,7 +171,7 @@ # Templates for sections that may be inserted multiple times in config files SUPERVISORD_PROCESS_CONFIG_BLOCK = """ [program:synapse_{name}] -command=/usr/local/bin/python -m {app} \ +command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \ --config-path="{config_path}" \ --config-path=/conf/workers/shared.yaml \ --config-path=/conf/workers/{name}.yaml diff --git a/docker/prefix-log b/docker/prefix-log new file mode 100755 index 000000000000..0e26a4f19d33 --- /dev/null +++ b/docker/prefix-log @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Prefixes all lines on stdout and stderr with the process name (as determined by +# the SUPERVISOR_PROCESS_NAME env var, which is automatically set by Supervisor). +# +# Usage: +# prefix-log command [args...] +# + +exec 1> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&1) +exec 2> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&2) +exec "$@" From 5ef673de4f0bf991402ee29235741a91a7cc9b02 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 27 Apr 2022 15:55:33 +0200 Subject: [PATCH 079/263] Add a module API to allow modules to edit push rule actions (#12406) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/12406.feature | 1 + synapse/handlers/push_rules.py | 138 ++++++++++++++++++++ synapse/module_api/__init__.py | 64 +++++++++ synapse/module_api/errors.py | 4 + synapse/rest/client/push_rule.py | 112 +++------------- synapse/server.py | 5 + synapse/storage/databases/main/push_rule.py | 15 +-- tests/module_api/test_api.py | 84 +++++++++++- 8 files changed, 319 insertions(+), 104 deletions(-) create mode 100644 changelog.d/12406.feature create mode 100644 synapse/handlers/push_rules.py diff --git a/changelog.d/12406.feature b/changelog.d/12406.feature new file mode 100644 index 000000000000..e345afdee729 --- /dev/null +++ b/changelog.d/12406.feature @@ -0,0 +1 @@ +Add a module API to allow modules to change actions for existing push rules of local users. diff --git a/synapse/handlers/push_rules.py b/synapse/handlers/push_rules.py new file mode 100644 index 000000000000..2599160bcc00 --- /dev/null +++ b/synapse/handlers/push_rules.py @@ -0,0 +1,138 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING, List, Optional, Union + +import attr + +from synapse.api.errors import SynapseError, UnrecognizedRequestError +from synapse.push.baserules import BASE_RULE_IDS +from synapse.storage.push_rule import RuleNotFoundException +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RuleSpec: + scope: str + template: str + rule_id: str + attr: Optional[str] + + +class PushRulesHandler: + """A class to handle changes in push rules for users.""" + + def __init__(self, hs: "HomeServer"): + self._notifier = hs.get_notifier() + self._main_store = hs.get_datastores().main + + async def set_rule_attr( + self, user_id: str, spec: RuleSpec, val: Union[bool, JsonDict] + ) -> None: + """Set an attribute (enabled or actions) on an existing push rule. + + Notifies listeners (e.g. sync handler) of the change. + + Args: + user_id: the user for which to modify the push rule. + spec: the spec of the push rule to modify. + val: the value to change the attribute to. + + Raises: + RuleNotFoundException if the rule being modified doesn't exist. + SynapseError(400) if the value is malformed. + UnrecognizedRequestError if the attribute to change is unknown. + InvalidRuleException if we're trying to change the actions on a rule but + the provided actions aren't compliant with the spec. + """ + if spec.attr not in ("enabled", "actions"): + # for the sake of potential future expansion, shouldn't report + # 404 in the case of an unknown request so check it corresponds to + # a known attribute first. + raise UnrecognizedRequestError() + + namespaced_rule_id = f"global/{spec.template}/{spec.rule_id}" + rule_id = spec.rule_id + is_default_rule = rule_id.startswith(".") + if is_default_rule: + if namespaced_rule_id not in BASE_RULE_IDS: + raise RuleNotFoundException("Unknown rule %r" % (namespaced_rule_id,)) + if spec.attr == "enabled": + if isinstance(val, dict) and "enabled" in val: + val = val["enabled"] + if not isinstance(val, bool): + # Legacy fallback + # This should *actually* take a dict, but many clients pass + # bools directly, so let's not break them. + raise SynapseError(400, "Value for 'enabled' must be boolean") + await self._main_store.set_push_rule_enabled( + user_id, namespaced_rule_id, val, is_default_rule + ) + elif spec.attr == "actions": + if not isinstance(val, dict): + raise SynapseError(400, "Value must be a dict") + actions = val.get("actions") + if not isinstance(actions, list): + raise SynapseError(400, "Value for 'actions' must be dict") + check_actions(actions) + rule_id = spec.rule_id + is_default_rule = rule_id.startswith(".") + if is_default_rule: + if namespaced_rule_id not in BASE_RULE_IDS: + raise RuleNotFoundException( + "Unknown rule %r" % (namespaced_rule_id,) + ) + await self._main_store.set_push_rule_actions( + user_id, namespaced_rule_id, actions, is_default_rule + ) + else: + raise UnrecognizedRequestError() + + self.notify_user(user_id) + + def notify_user(self, user_id: str) -> None: + """Notify listeners about a push rule change. + + Args: + user_id: the user ID the change is for. + """ + stream_id = self._main_store.get_max_push_rules_stream_id() + self._notifier.on_new_event("push_rules_key", stream_id, users=[user_id]) + + +def check_actions(actions: List[Union[str, JsonDict]]) -> None: + """Check if the given actions are spec compliant. + + Args: + actions: the actions to check. + + Raises: + InvalidRuleException if the rules aren't compliant with the spec. + """ + if not isinstance(actions, list): + raise InvalidRuleException("No actions found") + + for a in actions: + if a in ["notify", "dont_notify", "coalesce"]: + pass + elif isinstance(a, dict) and "set_tweak" in a: + pass + else: + raise InvalidRuleException("Unrecognised action %s" % a) + + +class InvalidRuleException(Exception): + pass diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 7e6e23db7342..834fe1b62c77 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -82,6 +82,7 @@ ON_LOGGED_OUT_CALLBACK, AuthHandler, ) +from synapse.handlers.push_rules import RuleSpec, check_actions from synapse.http.client import SimpleHttpClient from synapse.http.server import ( DirectServeHtmlResource, @@ -195,6 +196,7 @@ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None: self._clock: Clock = hs.get_clock() self._registration_handler = hs.get_registration_handler() self._send_email_handler = hs.get_send_email_handler() + self._push_rules_handler = hs.get_push_rules_handler() self.custom_template_dir = hs.config.server.custom_template_directory try: @@ -1352,6 +1354,68 @@ async def store_remote_3pid_association( """ await self._store.add_user_bound_threepid(user_id, medium, address, id_server) + def check_push_rule_actions( + self, actions: List[Union[str, Dict[str, str]]] + ) -> None: + """Checks if the given push rule actions are valid according to the Matrix + specification. + + See https://spec.matrix.org/v1.2/client-server-api/#actions for the list of valid + actions. + + Added in Synapse v1.58.0. + + Args: + actions: the actions to check. + + Raises: + synapse.module_api.errors.InvalidRuleException if the actions are invalid. + """ + check_actions(actions) + + async def set_push_rule_action( + self, + user_id: str, + scope: str, + kind: str, + rule_id: str, + actions: List[Union[str, Dict[str, str]]], + ) -> None: + """Changes the actions of an existing push rule for the given user. + + See https://spec.matrix.org/v1.2/client-server-api/#push-rules for more + information about push rules and their syntax. + + Can only be called on the main process. + + Added in Synapse v1.58.0. + + Args: + user_id: the user for which to change the push rule's actions. + scope: the push rule's scope, currently only "global" is allowed. + kind: the push rule's kind. + rule_id: the push rule's identifier. + actions: the actions to run when the rule's conditions match. + + Raises: + RuntimeError if this method is called on a worker or `scope` is invalid. + synapse.module_api.errors.RuleNotFoundException if the rule being modified + can't be found. + synapse.module_api.errors.InvalidRuleException if the actions are invalid. + """ + if self.worker_app is not None: + raise RuntimeError("module tried to change push rule actions on a worker") + + if scope != "global": + raise RuntimeError( + "invalid scope %s, only 'global' is currently allowed" % scope + ) + + spec = RuleSpec(scope, kind, rule_id, "actions") + await self._push_rules_handler.set_rule_attr( + user_id, spec, {"actions": actions} + ) + class PublicRoomListManager: """Contains methods for adding to, removing from and querying whether a room diff --git a/synapse/module_api/errors.py b/synapse/module_api/errors.py index 1db900e41f64..e58e0e60feab 100644 --- a/synapse/module_api/errors.py +++ b/synapse/module_api/errors.py @@ -20,10 +20,14 @@ SynapseError, ) from synapse.config._base import ConfigError +from synapse.handlers.push_rules import InvalidRuleException +from synapse.storage.push_rule import RuleNotFoundException __all__ = [ "InvalidClientCredentialsError", "RedirectException", "SynapseError", "ConfigError", + "InvalidRuleException", + "RuleNotFoundException", ] diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index a93f6fd5e0a8..b98640b14ac5 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, List, Optional, Sequence, Tuple, Union - -import attr +from typing import TYPE_CHECKING, List, Sequence, Tuple, Union from synapse.api.errors import ( NotFoundError, @@ -22,6 +20,7 @@ SynapseError, UnrecognizedRequestError, ) +from synapse.handlers.push_rules import InvalidRuleException, RuleSpec, check_actions from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -29,7 +28,6 @@ parse_string, ) from synapse.http.site import SynapseRequest -from synapse.push.baserules import BASE_RULE_IDS from synapse.push.clientformat import format_push_rules_for_user from synapse.push.rulekinds import PRIORITY_CLASS_MAP from synapse.rest.client._base import client_patterns @@ -40,14 +38,6 @@ from synapse.server import HomeServer -@attr.s(slots=True, frozen=True, auto_attribs=True) -class RuleSpec: - scope: str - template: str - rule_id: str - attr: Optional[str] - - class PushRuleRestServlet(RestServlet): PATTERNS = client_patterns("/(?Ppushrules/.*)$", v1=True) SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( @@ -60,6 +50,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self._is_worker = hs.config.worker.worker_app is not None + self._push_rules_handler = hs.get_push_rules_handler() async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if self._is_worker: @@ -81,8 +72,13 @@ async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic user_id = requester.user.to_string() if spec.attr: - await self.set_rule_attr(user_id, spec, content) - self.notify_user(user_id) + try: + await self._push_rules_handler.set_rule_attr(user_id, spec, content) + except InvalidRuleException as e: + raise SynapseError(400, "Invalid actions: %s" % e) + except RuleNotFoundException: + raise NotFoundError("Unknown rule") + return 200, {} if spec.rule_id.startswith("."): @@ -98,23 +94,23 @@ async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic before = parse_string(request, "before") if before: - before = _namespaced_rule_id(spec, before) + before = f"global/{spec.template}/{before}" after = parse_string(request, "after") if after: - after = _namespaced_rule_id(spec, after) + after = f"global/{spec.template}/{after}" try: await self.store.add_push_rule( user_id=user_id, - rule_id=_namespaced_rule_id_from_spec(spec), + rule_id=f"global/{spec.template}/{spec.rule_id}", priority_class=priority_class, conditions=conditions, actions=actions, before=before, after=after, ) - self.notify_user(user_id) + self._push_rules_handler.notify_user(user_id) except InconsistentRuleException as e: raise SynapseError(400, str(e)) except RuleNotFoundException as e: @@ -133,11 +129,11 @@ async def on_DELETE( requester = await self.auth.get_user_by_req(request) user_id = requester.user.to_string() - namespaced_rule_id = _namespaced_rule_id_from_spec(spec) + namespaced_rule_id = f"global/{spec.template}/{spec.rule_id}" try: await self.store.delete_push_rule(user_id, namespaced_rule_id) - self.notify_user(user_id) + self._push_rules_handler.notify_user(user_id) return 200, {} except StoreError as e: if e.code == 404: @@ -172,55 +168,6 @@ async def on_GET(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic else: raise UnrecognizedRequestError() - def notify_user(self, user_id: str) -> None: - stream_id = self.store.get_max_push_rules_stream_id() - self.notifier.on_new_event("push_rules_key", stream_id, users=[user_id]) - - async def set_rule_attr( - self, user_id: str, spec: RuleSpec, val: Union[bool, JsonDict] - ) -> None: - if spec.attr not in ("enabled", "actions"): - # for the sake of potential future expansion, shouldn't report - # 404 in the case of an unknown request so check it corresponds to - # a known attribute first. - raise UnrecognizedRequestError() - - namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - rule_id = spec.rule_id - is_default_rule = rule_id.startswith(".") - if is_default_rule: - if namespaced_rule_id not in BASE_RULE_IDS: - raise NotFoundError("Unknown rule %s" % (namespaced_rule_id,)) - if spec.attr == "enabled": - if isinstance(val, dict) and "enabled" in val: - val = val["enabled"] - if not isinstance(val, bool): - # Legacy fallback - # This should *actually* take a dict, but many clients pass - # bools directly, so let's not break them. - raise SynapseError(400, "Value for 'enabled' must be boolean") - await self.store.set_push_rule_enabled( - user_id, namespaced_rule_id, val, is_default_rule - ) - elif spec.attr == "actions": - if not isinstance(val, dict): - raise SynapseError(400, "Value must be a dict") - actions = val.get("actions") - if not isinstance(actions, list): - raise SynapseError(400, "Value for 'actions' must be dict") - _check_actions(actions) - namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - rule_id = spec.rule_id - is_default_rule = rule_id.startswith(".") - if is_default_rule: - if namespaced_rule_id not in BASE_RULE_IDS: - raise SynapseError(404, "Unknown rule %r" % (namespaced_rule_id,)) - await self.store.set_push_rule_actions( - user_id, namespaced_rule_id, actions, is_default_rule - ) - else: - raise UnrecognizedRequestError() - def _rule_spec_from_path(path: Sequence[str]) -> RuleSpec: """Turn a sequence of path components into a rule spec @@ -291,24 +238,11 @@ def _rule_tuple_from_request_object( raise InvalidRuleException("No actions found") actions = req_obj["actions"] - _check_actions(actions) + check_actions(actions) return conditions, actions -def _check_actions(actions: List[Union[str, JsonDict]]) -> None: - if not isinstance(actions, list): - raise InvalidRuleException("No actions found") - - for a in actions: - if a in ["notify", "dont_notify", "coalesce"]: - pass - elif isinstance(a, dict) and "set_tweak" in a: - pass - else: - raise InvalidRuleException("Unrecognised action") - - def _filter_ruleset_with_path(ruleset: JsonDict, path: List[str]) -> JsonDict: if path == []: raise UnrecognizedRequestError( @@ -357,17 +291,5 @@ def _priority_class_from_spec(spec: RuleSpec) -> int: return pc -def _namespaced_rule_id_from_spec(spec: RuleSpec) -> str: - return _namespaced_rule_id(spec, spec.rule_id) - - -def _namespaced_rule_id(spec: RuleSpec, rule_id: str) -> str: - return "global/%s/%s" % (spec.template, rule_id) - - -class InvalidRuleException(Exception): - pass - - def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: PushRuleRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index 37c72bd83a9f..d49c76518a8d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -91,6 +91,7 @@ WorkerPresenceHandler, ) from synapse.handlers.profile import ProfileHandler +from synapse.handlers.push_rules import PushRulesHandler from synapse.handlers.read_marker import ReadMarkerHandler from synapse.handlers.receipts import ReceiptsHandler from synapse.handlers.register import RegistrationHandler @@ -810,6 +811,10 @@ def get_external_cache(self) -> ExternalCache: def get_account_handler(self) -> AccountHandler: return AccountHandler(self) + @cache_in_self + def get_push_rules_handler(self) -> PushRulesHandler: + return PushRulesHandler(self) + @cache_in_self def get_outbound_redis_connection(self) -> "ConnectionHandler": """ diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 92539f5d41b1..eb85bbd39243 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -16,7 +16,7 @@ import logging from typing import TYPE_CHECKING, Dict, List, Tuple, Union -from synapse.api.errors import NotFoundError, StoreError +from synapse.api.errors import StoreError from synapse.push.baserules import list_with_base_rules from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker from synapse.storage._base import SQLBaseStore, db_to_json @@ -618,7 +618,7 @@ async def set_push_rule_enabled( are always stored in the database `push_rules` table). Raises: - NotFoundError if the rule does not exist. + RuleNotFoundException if the rule does not exist. """ async with self._push_rules_stream_id_gen.get_next() as stream_id: event_stream_ordering = self._stream_id_gen.get_current_token() @@ -668,8 +668,7 @@ def _set_push_rule_enabled_txn( ) txn.execute(sql, (user_id, rule_id)) if txn.fetchone() is None: - # needed to set NOT_FOUND code. - raise NotFoundError("Push rule does not exist.") + raise RuleNotFoundException("Push rule does not exist.") self.db_pool.simple_upsert_txn( txn, @@ -698,9 +697,6 @@ async def set_push_rule_actions( """ Sets the `actions` state of a push rule. - Will throw NotFoundError if the rule does not exist; the Code for this - is NOT_FOUND. - Args: user_id: the user ID of the user who wishes to enable/disable the rule e.g. '@tina:example.org' @@ -712,6 +708,9 @@ async def set_push_rule_actions( is_default_rule: True if and only if this is a server-default rule. This skips the check for existence (as only user-created rules are always stored in the database `push_rules` table). + + Raises: + RuleNotFoundException if the rule does not exist. """ actions_json = json_encoder.encode(actions) @@ -744,7 +743,7 @@ def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering): except StoreError as serr: if serr.code == 404: # this sets the NOT_FOUND error Code - raise NotFoundError("Push rule does not exist") + raise RuleNotFoundException("Push rule does not exist") else: raise diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 9fd5d59c55b2..8bc84aaaca44 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -19,8 +19,9 @@ from synapse.events import EventBase from synapse.federation.units import Transaction from synapse.handlers.presence import UserPresenceState +from synapse.handlers.push_rules import InvalidRuleException from synapse.rest import admin -from synapse.rest.client import login, presence, profile, room +from synapse.rest.client import login, notifications, presence, profile, room from synapse.types import create_requester from tests.events.test_presence_router import send_presence_update, sync_presence @@ -38,6 +39,7 @@ class ModuleApiTestCase(HomeserverTestCase): room.register_servlets, presence.register_servlets, profile.register_servlets, + notifications.register_servlets, ] def prepare(self, reactor, clock, homeserver): @@ -553,6 +555,86 @@ def test_get_room_state(self): self.assertEqual(state[("org.matrix.test", "")].state_key, "") self.assertEqual(state[("org.matrix.test", "")].content, {}) + def test_set_push_rules_action(self) -> None: + """Test that a module can change the actions of an existing push rule for a user.""" + + # Create a room with 2 users in it. Push rules must not match if the user is the + # event's sender, so we need one user to send messages and one user to receive + # notifications. + user_id = self.register_user("user", "password") + tok = self.login("user", "password") + + room_id = self.helper.create_room_as(user_id, is_public=True, tok=tok) + + user_id2 = self.register_user("user2", "password") + tok2 = self.login("user2", "password") + self.helper.join(room_id, user_id2, tok=tok2) + + # Register a 3rd user and join them to the room, so that we don't accidentally + # trigger 1:1 push rules. + user_id3 = self.register_user("user3", "password") + tok3 = self.login("user3", "password") + self.helper.join(room_id, user_id3, tok=tok3) + + # Send a message as the second user and check that it notifies. + res = self.helper.send(room_id=room_id, body="here's a message", tok=tok2) + event_id = res["event_id"] + + channel = self.make_request( + "GET", + "/notifications", + access_token=tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.assertEqual(len(channel.json_body["notifications"]), 1, channel.json_body) + self.assertEqual( + channel.json_body["notifications"][0]["event"]["event_id"], + event_id, + channel.json_body, + ) + + # Change the .m.rule.message actions to not notify on new messages. + self.get_success( + defer.ensureDeferred( + self.module_api.set_push_rule_action( + user_id=user_id, + scope="global", + kind="underride", + rule_id=".m.rule.message", + actions=["dont_notify"], + ) + ) + ) + + # Send another message as the second user and check that the number of + # notifications didn't change. + self.helper.send(room_id=room_id, body="here's another message", tok=tok2) + + channel = self.make_request( + "GET", + "/notifications?from=", + access_token=tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(len(channel.json_body["notifications"]), 1, channel.json_body) + + def test_check_push_rules_actions(self) -> None: + """Test that modules can check whether a list of push rules actions are spec + compliant. + """ + with self.assertRaises(InvalidRuleException): + self.module_api.check_push_rule_actions(["foo"]) + + with self.assertRaises(InvalidRuleException): + self.module_api.check_push_rule_actions({"foo": "bar"}) + + self.module_api.check_push_rule_actions(["notify"]) + + self.module_api.check_push_rule_actions( + [{"set_tweak": "sound", "value": "default"}] + ) + class ModuleApiWorkerTestCase(BaseMultiWorkerStreamTestCase): """For testing ModuleApi functionality in a multi-worker setup""" From 78b99de7c206b106340e12cdee0af9aa246bd5ad Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 27 Apr 2022 14:58:26 +0100 Subject: [PATCH 080/263] Prefer `make_awaitable` over `defer.succeed` in tests (#12505) When configuring the return values of mocks, prefer awaitables from `make_awaitable` over `defer.succeed`. `Deferred`s are only awaitable once, so it is inappropriate for a mock to return the same `Deferred` multiple times. Also update `run_in_background` to support functions that return arbitrary awaitables. Signed-off-by: Sean Quah --- changelog.d/12505.misc | 1 + synapse/logging/context.py | 26 +++++++++----- tests/federation/test_federation_client.py | 2 +- tests/federation/test_federation_sender.py | 2 +- tests/handlers/test_e2e_keys.py | 7 ++-- tests/handlers/test_password_providers.py | 34 +++++++++---------- tests/handlers/test_typing.py | 6 ++-- tests/handlers/test_user_directory.py | 6 ++-- tests/rest/client/test_presence.py | 4 +-- tests/rest/client/test_rooms.py | 7 ++-- tests/rest/client/test_transactions.py | 7 ++-- .../test_resource_limits_server_notices.py | 28 +++++++-------- tests/storage/test_monthly_active_users.py | 9 +++-- tests/test_federation.py | 2 +- 14 files changed, 72 insertions(+), 69 deletions(-) create mode 100644 changelog.d/12505.misc diff --git a/changelog.d/12505.misc b/changelog.d/12505.misc new file mode 100644 index 000000000000..a691d7962f89 --- /dev/null +++ b/changelog.d/12505.misc @@ -0,0 +1 @@ +Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 88cd8a9e1c39..fd9cb979208a 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -722,6 +722,11 @@ def nested_logging_context(suffix: str) -> LoggingContext: R = TypeVar("R") +async def _unwrap_awaitable(awaitable: Awaitable[R]) -> R: + """Unwraps an arbitrary awaitable by awaiting it.""" + return await awaitable + + @overload def preserve_fn( # type: ignore[misc] f: Callable[P, Awaitable[R]], @@ -802,17 +807,20 @@ def run_in_background( # type: ignore[misc] # by synchronous exceptions, so let's turn them into Failures. return defer.fail() + # `res` may be a coroutine, `Deferred`, some other kind of awaitable, or a plain + # value. Convert it to a `Deferred`. if isinstance(res, typing.Coroutine): + # Wrap the coroutine in a `Deferred`. res = defer.ensureDeferred(res) - - # At this point we should have a Deferred, if not then f was a synchronous - # function, wrap it in a Deferred for consistency. - if not isinstance(res, defer.Deferred): - # `res` is not a `Deferred` and not a `Coroutine`. - # There are no other types of `Awaitable`s we expect to encounter in Synapse. - assert not isinstance(res, Awaitable) - - return defer.succeed(res) + elif isinstance(res, defer.Deferred): + pass + elif isinstance(res, Awaitable): + # `res` is probably some kind of completed awaitable, such as a `DoneAwaitable` + # or `Future` from `make_awaitable`. + res = defer.ensureDeferred(_unwrap_awaitable(res)) + else: + # `res` is a plain value. Wrap it in a `Deferred`. + res = defer.succeed(res) if res.called and not res.paused: # The function should have maintained the logcontext, so we can diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index ec8864dafe37..268a48d7ba5f 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -83,7 +83,7 @@ def test_get_room_state(self): ) # mock up the response, and have the agent return it - self._mock_agent.request.return_value = defer.succeed( + self._mock_agent.request.side_effect = lambda *args, **kwargs: defer.succeed( _mock_response( { "pdus": [ diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 91f982518e69..6b26353d5e93 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -226,7 +226,7 @@ def test_dont_send_device_updates_for_remote_users(self): # Send the server a device list EDU for the other user, this will cause # it to try and resync the device lists. self.hs.get_federation_transport_client().query_user_devices.return_value = ( - defer.succeed( + make_awaitable( { "stream_id": "1", "user_id": "@user2:host2", diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 8c74ed1fcffc..1e6ad4b663e9 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -19,7 +19,6 @@ from parameterized import parameterized from signedjson import key as key, sign as sign -from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms @@ -704,7 +703,7 @@ def test_query_devices_remote_no_sync(self) -> None: remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" self.hs.get_federation_client().query_client_keys = mock.Mock( - return_value=defer.succeed( + return_value=make_awaitable( { "device_keys": {remote_user_id: {}}, "master_keys": { @@ -777,14 +776,14 @@ def test_query_devices_remote_sync(self) -> None: # Pretend we're sharing a room with the user we're querying. If not, # `_query_devices_for_destination` will return early. self.store.get_rooms_for_user = mock.Mock( - return_value=defer.succeed({"some_room_id"}) + return_value=make_awaitable({"some_room_id"}) ) remote_master_key = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" remote_self_signing_key = "QeIiFEjluPBtI7WQdG365QKZcFs9kqmHir6RBD0//nQ" self.hs.get_federation_client().query_user_devices = mock.Mock( - return_value=defer.succeed( + return_value=make_awaitable( { "user_id": remote_user_id, "stream_id": 1, diff --git a/tests/handlers/test_password_providers.py b/tests/handlers/test_password_providers.py index d401fda93855..addf14fa2ba0 100644 --- a/tests/handlers/test_password_providers.py +++ b/tests/handlers/test_password_providers.py @@ -17,8 +17,6 @@ from typing import Any, Type, Union from unittest.mock import Mock -from twisted.internet import defer - import synapse from synapse.api.constants import LoginType from synapse.api.errors import Codes @@ -190,7 +188,7 @@ def password_only_auth_provider_login_test_body(self): self.assertEqual(flows, [{"type": "m.login.password"}] + ADDITIONAL_LOGIN_FLOWS) # check_password must return an awaitable - mock_password_provider.check_password.return_value = defer.succeed(True) + mock_password_provider.check_password.return_value = make_awaitable(True) channel = self._send_password_login("u", "p") self.assertEqual(channel.code, 200, channel.result) self.assertEqual("@u:test", channel.json_body["user_id"]) @@ -226,13 +224,13 @@ def password_only_auth_provider_ui_auth_test_body(self): self.get_success(module_api.register_user("u")) # log in twice, to get two devices - mock_password_provider.check_password.return_value = defer.succeed(True) + mock_password_provider.check_password.return_value = make_awaitable(True) tok1 = self.login("u", "p") self.login("u", "p", device_id="dev2") mock_password_provider.reset_mock() # have the auth provider deny the request to start with - mock_password_provider.check_password.return_value = defer.succeed(False) + mock_password_provider.check_password.return_value = make_awaitable(False) # make the initial request which returns a 401 session = self._start_delete_device_session(tok1, "dev2") @@ -246,7 +244,7 @@ def password_only_auth_provider_ui_auth_test_body(self): mock_password_provider.reset_mock() # Finally, check the request goes through when we allow it - mock_password_provider.check_password.return_value = defer.succeed(True) + mock_password_provider.check_password.return_value = make_awaitable(True) channel = self._authed_delete_device(tok1, "dev2", session, "u", "p") self.assertEqual(channel.code, 200) mock_password_provider.check_password.assert_called_once_with("@u:test", "p") @@ -260,7 +258,7 @@ def local_user_fallback_login_test_body(self): self.register_user("localuser", "localpass") # check_password must return an awaitable - mock_password_provider.check_password.return_value = defer.succeed(False) + mock_password_provider.check_password.return_value = make_awaitable(False) channel = self._send_password_login("u", "p") self.assertEqual(channel.code, 403, channel.result) @@ -277,7 +275,7 @@ def local_user_fallback_ui_auth_test_body(self): self.register_user("localuser", "localpass") # have the auth provider deny the request - mock_password_provider.check_password.return_value = defer.succeed(False) + mock_password_provider.check_password.return_value = make_awaitable(False) # log in twice, to get two devices tok1 = self.login("localuser", "localpass") @@ -320,7 +318,7 @@ def no_local_user_fallback_login_test_body(self): self.register_user("localuser", "localpass") # check_password must return an awaitable - mock_password_provider.check_password.return_value = defer.succeed(False) + mock_password_provider.check_password.return_value = make_awaitable(False) channel = self._send_password_login("localuser", "localpass") self.assertEqual(channel.code, 403) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") @@ -342,7 +340,7 @@ def no_local_user_fallback_ui_auth_test_body(self): self.register_user("localuser", "localpass") # allow login via the auth provider - mock_password_provider.check_password.return_value = defer.succeed(True) + mock_password_provider.check_password.return_value = make_awaitable(True) # log in twice, to get two devices tok1 = self.login("localuser", "p") @@ -359,7 +357,7 @@ def no_local_user_fallback_ui_auth_test_body(self): mock_password_provider.check_password.assert_not_called() # now try deleting with the local password - mock_password_provider.check_password.return_value = defer.succeed(False) + mock_password_provider.check_password.return_value = make_awaitable(False) channel = self._authed_delete_device( tok1, "dev2", session, "localuser", "localpass" ) @@ -413,7 +411,7 @@ def custom_auth_provider_login_test_body(self): self.assertEqual(channel.code, 400, channel.result) mock_password_provider.check_auth.assert_not_called() - mock_password_provider.check_auth.return_value = defer.succeed( + mock_password_provider.check_auth.return_value = make_awaitable( ("@user:bz", None) ) channel = self._send_login("test.login_type", "u", test_field="y") @@ -427,7 +425,7 @@ def custom_auth_provider_login_test_body(self): # try a weird username. Again, it's unclear what we *expect* to happen # in these cases, but at least we can guard against the API changing # unexpectedly - mock_password_provider.check_auth.return_value = defer.succeed( + mock_password_provider.check_auth.return_value = make_awaitable( ("@ MALFORMED! :bz", None) ) channel = self._send_login("test.login_type", " USER🙂NAME ", test_field=" abc ") @@ -477,7 +475,7 @@ def custom_auth_provider_ui_auth_test_body(self): mock_password_provider.reset_mock() # right params, but authing as the wrong user - mock_password_provider.check_auth.return_value = defer.succeed( + mock_password_provider.check_auth.return_value = make_awaitable( ("@user:bz", None) ) body["auth"]["test_field"] = "foo" @@ -490,7 +488,7 @@ def custom_auth_provider_ui_auth_test_body(self): mock_password_provider.reset_mock() # and finally, succeed - mock_password_provider.check_auth.return_value = defer.succeed( + mock_password_provider.check_auth.return_value = make_awaitable( ("@localuser:test", None) ) channel = self._delete_device(tok1, "dev2", body) @@ -508,9 +506,9 @@ def test_custom_auth_provider_callback(self): self.custom_auth_provider_callback_test_body() def custom_auth_provider_callback_test_body(self): - callback = Mock(return_value=defer.succeed(None)) + callback = Mock(return_value=make_awaitable(None)) - mock_password_provider.check_auth.return_value = defer.succeed( + mock_password_provider.check_auth.return_value = make_awaitable( ("@user:bz", callback) ) channel = self._send_login("test.login_type", "u", test_field="y") @@ -646,7 +644,7 @@ def password_custom_auth_password_disabled_ui_auth_test_body(self): login is disabled""" # register the user and log in twice via the test login type to get two devices, self.register_user("localuser", "localpass") - mock_password_provider.check_auth.return_value = defer.succeed( + mock_password_provider.check_auth.return_value = make_awaitable( ("@localuser:test", None) ) channel = self._send_login("test.login_type", "localuser", test_field="") diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index ffd5c4cb938c..5f2e26a5fce7 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -65,11 +65,11 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: # we mock out the keyring so as to skip the authentication check on the # federation API call. mock_keyring = Mock(spec=["verify_json_for_server"]) - mock_keyring.verify_json_for_server.return_value = defer.succeed(True) + mock_keyring.verify_json_for_server.return_value = make_awaitable(True) # we mock out the federation client too mock_federation_client = Mock(spec=["put_json"]) - mock_federation_client.put_json.return_value = defer.succeed((200, "OK")) + mock_federation_client.put_json.return_value = make_awaitable((200, "OK")) # the tests assume that we are starting at unix time 1000 reactor.pump((1000,)) @@ -98,7 +98,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.datastore = hs.get_datastores().main self.datastore.get_destination_retry_timings = Mock( - return_value=defer.succeed(None) + return_value=make_awaitable(None) ) self.datastore.get_device_updates_by_remote = Mock( diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index c6e501c7be56..96e2e3039ba8 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -15,7 +15,6 @@ from unittest.mock import Mock, patch from urllib.parse import quote -from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin @@ -30,6 +29,7 @@ from tests import unittest from tests.storage.test_user_directory import GetUserDirectoryTables +from tests.test_utils import make_awaitable from tests.test_utils.event_injection import inject_member_event from tests.unittest import override_config @@ -439,7 +439,7 @@ def test_handle_user_deactivated_support_user(self) -> None: ) ) - mock_remove_from_user_dir = Mock(return_value=defer.succeed(None)) + mock_remove_from_user_dir = Mock(return_value=make_awaitable(None)) with patch.object( self.store, "remove_from_user_dir", mock_remove_from_user_dir ): @@ -454,7 +454,7 @@ def test_handle_user_deactivated_regular_user(self) -> None: self.store.register_user(user_id=r_user_id, password_hash=None) ) - mock_remove_from_user_dir = Mock(return_value=defer.succeed(None)) + mock_remove_from_user_dir = Mock(return_value=make_awaitable(None)) with patch.object( self.store, "remove_from_user_dir", mock_remove_from_user_dir ): diff --git a/tests/rest/client/test_presence.py b/tests/rest/client/test_presence.py index 0abe378fe4d6..b3738a03046a 100644 --- a/tests/rest/client/test_presence.py +++ b/tests/rest/client/test_presence.py @@ -14,7 +14,6 @@ from http import HTTPStatus from unittest.mock import Mock -from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor from synapse.handlers.presence import PresenceHandler @@ -24,6 +23,7 @@ from synapse.util import Clock from tests import unittest +from tests.test_utils import make_awaitable class PresenceTestCase(unittest.HomeserverTestCase): @@ -37,7 +37,7 @@ class PresenceTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: presence_handler = Mock(spec=PresenceHandler) - presence_handler.set_state.return_value = defer.succeed(None) + presence_handler.set_state.return_value = make_awaitable(None) hs = self.setup_test_homeserver( "red", diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 6ff79b9e2eed..9443daa0560a 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -22,7 +22,6 @@ from unittest.mock import Mock, call from urllib import parse as urlparse -from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin @@ -1426,9 +1425,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: def test_simple(self) -> None: "Simple test for searching rooms over federation" - self.federation_client.get_public_rooms.side_effect = lambda *a, **k: defer.succeed( # type: ignore[attr-defined] - {} - ) + self.federation_client.get_public_rooms.return_value = make_awaitable({}) # type: ignore[attr-defined] search_filter = {"generic_search_term": "foobar"} @@ -1456,7 +1453,7 @@ def test_fallback(self) -> None: # with a 404, when using search filters. self.federation_client.get_public_rooms.side_effect = ( # type: ignore[attr-defined] HttpResponseException(404, "Not Found", b""), - defer.succeed({}), + make_awaitable({}), ) search_filter = {"generic_search_term": "foobar"} diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 8d8251b2ac99..21a1ca2a6885 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -22,6 +22,7 @@ from synapse.util import Clock from tests import unittest +from tests.test_utils import make_awaitable from tests.utils import MockClock @@ -38,7 +39,7 @@ def setUp(self) -> None: @defer.inlineCallbacks def test_executes_given_function(self): - cb = Mock(return_value=defer.succeed(self.mock_http_response)) + cb = Mock(return_value=make_awaitable(self.mock_http_response)) res = yield self.cache.fetch_or_execute( self.mock_key, cb, "some_arg", keyword="arg" ) @@ -47,7 +48,7 @@ def test_executes_given_function(self): @defer.inlineCallbacks def test_deduplicates_based_on_key(self): - cb = Mock(return_value=defer.succeed(self.mock_http_response)) + cb = Mock(return_value=make_awaitable(self.mock_http_response)) for i in range(3): # invoke multiple times res = yield self.cache.fetch_or_execute( self.mock_key, cb, "some_arg", keyword="arg", changing_args=i @@ -130,7 +131,7 @@ def cb(): @defer.inlineCallbacks def test_cleans_up(self): - cb = Mock(return_value=defer.succeed(self.mock_http_response)) + cb = Mock(return_value=make_awaitable(self.mock_http_response)) yield self.cache.fetch_or_execute(self.mock_key, cb, "an arg") # should NOT have cleaned up yet self.clock.advance_time_msec(CLEANUP_PERIOD_MS / 2) diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index 02b96c9e6ecf..9ee9509d3a96 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -14,8 +14,6 @@ from unittest.mock import Mock -from twisted.internet import defer - from synapse.api.constants import EventTypes, LimitBlockingTypes, ServerNoticeMsgType from synapse.api.errors import ResourceLimitError from synapse.rest import admin @@ -68,16 +66,16 @@ def prepare(self, reactor, clock, hs): return_value=make_awaitable(1000) ) self._rlsn._server_notices_manager.send_notice = Mock( - return_value=defer.succeed(Mock()) + return_value=make_awaitable(Mock()) ) self._send_notice = self._rlsn._server_notices_manager.send_notice self.user_id = "@user_id:test" self._rlsn._server_notices_manager.get_or_create_notice_room_for_user = Mock( - return_value=defer.succeed("!something:localhost") + return_value=make_awaitable("!something:localhost") ) - self._rlsn._store.add_tag_to_room = Mock(return_value=defer.succeed(None)) + self._rlsn._store.add_tag_to_room = Mock(return_value=make_awaitable(None)) self._rlsn._store.get_tags_for_room = Mock(return_value=make_awaitable({})) @override_config({"hs_disabled": True}) @@ -95,7 +93,7 @@ def test_maybe_send_server_notice_to_user_flag_off(self): def test_maybe_send_server_notice_to_user_remove_blocked_notice(self): """Test when user has blocked notice, but should have it removed""" - self._rlsn._auth.check_auth_blocking = Mock(return_value=defer.succeed(None)) + self._rlsn._auth.check_auth_blocking = Mock(return_value=make_awaitable(None)) mock_event = Mock( type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType} ) @@ -111,7 +109,8 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self): Test when user has blocked notice, but notice ought to be there (NOOP) """ self._rlsn._auth.check_auth_blocking = Mock( - return_value=defer.succeed(None), side_effect=ResourceLimitError(403, "foo") + return_value=make_awaitable(None), + side_effect=ResourceLimitError(403, "foo"), ) mock_event = Mock( @@ -130,7 +129,8 @@ def test_maybe_send_server_notice_to_user_add_blocked_notice(self): Test when user does not have blocked notice, but should have one """ self._rlsn._auth.check_auth_blocking = Mock( - return_value=defer.succeed(None), side_effect=ResourceLimitError(403, "foo") + return_value=make_awaitable(None), + side_effect=ResourceLimitError(403, "foo"), ) self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id)) @@ -141,7 +141,7 @@ def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self): """ Test when user does not have blocked notice, nor should they (NOOP) """ - self._rlsn._auth.check_auth_blocking = Mock(return_value=defer.succeed(None)) + self._rlsn._auth.check_auth_blocking = Mock(return_value=make_awaitable(None)) self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id)) @@ -152,7 +152,7 @@ def test_maybe_send_server_notice_to_user_not_in_mau_cohort(self): Test when user is not part of the MAU cohort - this should not ever happen - but ... """ - self._rlsn._auth.check_auth_blocking = Mock(return_value=defer.succeed(None)) + self._rlsn._auth.check_auth_blocking = Mock(return_value=make_awaitable(None)) self._rlsn._store.user_last_seen_monthly_active = Mock( return_value=make_awaitable(None) ) @@ -167,7 +167,7 @@ def test_maybe_send_server_notice_when_alerting_suppressed_room_unblocked(self): an alert message is not sent into the room """ self._rlsn._auth.check_auth_blocking = Mock( - return_value=defer.succeed(None), + return_value=make_awaitable(None), side_effect=ResourceLimitError( 403, "foo", limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER ), @@ -182,7 +182,7 @@ def test_check_hs_disabled_unaffected_by_mau_alert_suppression(self): Test that when a server is disabled, that MAU limit alerting is ignored. """ self._rlsn._auth.check_auth_blocking = Mock( - return_value=defer.succeed(None), + return_value=make_awaitable(None), side_effect=ResourceLimitError( 403, "foo", limit_type=LimitBlockingTypes.HS_DISABLED ), @@ -199,14 +199,14 @@ def test_maybe_send_server_notice_when_alerting_suppressed_room_blocked(self): is suppressed that the room is returned to an unblocked state. """ self._rlsn._auth.check_auth_blocking = Mock( - return_value=defer.succeed(None), + return_value=make_awaitable(None), side_effect=ResourceLimitError( 403, "foo", limit_type=LimitBlockingTypes.MONTHLY_ACTIVE_USER ), ) self._rlsn._server_notices_manager.__is_room_currently_blocked = Mock( - return_value=defer.succeed((True, [])) + return_value=make_awaitable((True, [])) ) mock_event = Mock( diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 60c8d3759481..0fbf46567091 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -14,7 +14,6 @@ from typing import Any, Dict, List from unittest.mock import Mock -from twisted.internet import defer from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import UserTypes @@ -259,10 +258,10 @@ def test_populate_monthly_users_is_guest(self): def test_populate_monthly_users_should_update(self): self.store.upsert_monthly_active_user = Mock(return_value=make_awaitable(None)) # type: ignore[assignment] - self.store.is_trial_user = Mock(return_value=defer.succeed(False)) # type: ignore[assignment] + self.store.is_trial_user = Mock(return_value=make_awaitable(False)) # type: ignore[assignment] self.store.user_last_seen_monthly_active = Mock( - return_value=defer.succeed(None) + return_value=make_awaitable(None) ) d = self.store.populate_monthly_active_users("user_id") self.get_success(d) @@ -272,9 +271,9 @@ def test_populate_monthly_users_should_update(self): def test_populate_monthly_users_should_not_update(self): self.store.upsert_monthly_active_user = Mock(return_value=make_awaitable(None)) # type: ignore[assignment] - self.store.is_trial_user = Mock(return_value=defer.succeed(False)) # type: ignore[assignment] + self.store.is_trial_user = Mock(return_value=make_awaitable(False)) # type: ignore[assignment] self.store.user_last_seen_monthly_active = Mock( - return_value=defer.succeed(self.hs.get_clock().time_msec()) + return_value=make_awaitable(self.hs.get_clock().time_msec()) ) d = self.store.populate_monthly_active_users("user_id") diff --git a/tests/test_federation.py b/tests/test_federation.py index c39816de855d..0cbef70bfa57 100644 --- a/tests/test_federation.py +++ b/tests/test_federation.py @@ -233,7 +233,7 @@ def test_cross_signing_keys_retry(self): # Register mock device list retrieval on the federation client. federation_client = self.homeserver.get_federation_client() federation_client.query_user_devices = Mock( - return_value=succeed( + return_value=make_awaitable( { "user_id": remote_user_id, "stream_id": 1, From ce6ecdd4b4939fd99418bc949b40c01d39480489 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 28 Apr 2022 11:27:42 +0100 Subject: [PATCH 081/263] Allow unused ignores in "bleeding edge" CI Where "bleeding edge" means the Twisted Trunk and Latest Deps jobs. Follow up from #12531. Resolves #12574. --- .github/workflows/latest_deps.yml | 2 ++ .github/workflows/twisted_trunk.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 1a61d179d905..41da1c74088c 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -38,6 +38,8 @@ jobs: # `pip install matrix-synapse[all]` as closely as possible. - run: poetry update --no-dev - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) + - name: Remove warn_unused_ignores from mypy config + run: sed '/warn_unused_ignores = True/d' -i mypy.ini - run: poetry run mypy trial: runs-on: ubuntu-latest diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 8fc1affb7746..5f0671f3503a 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -24,6 +24,8 @@ jobs: poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" + - name: Remove warn_unused_ignores from mypy config + run: sed '/warn_unused_ignores = True/d' -i mypy.ini - run: poetry run mypy trial: From f282d5fc1185dde3f9ec31c49b630cff962545d7 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 28 Apr 2022 11:29:13 +0100 Subject: [PATCH 082/263] Use `--extras all` in latest deps mypy CI Twisted trunk job already does this. Missed in #12531. --- .github/workflows/latest_deps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 41da1c74088c..c537a5a60f9f 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -32,6 +32,7 @@ jobs: with: python-version: "3.x" poetry-version: "1.2.0b1" + extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt # Upgrade all runtime dependencies only. This is intended to mimic a fresh From 5a320baa45b8e826e52bdd6cadadfad727ab0357 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 28 Apr 2022 11:31:26 +0100 Subject: [PATCH 083/263] changelog --- changelog.d/12576.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12576.misc diff --git a/changelog.d/12576.misc b/changelog.d/12576.misc new file mode 100644 index 000000000000..71022c86337f --- /dev/null +++ b/changelog.d/12576.misc @@ -0,0 +1 @@ +Allow unused `#type: ignore` comments in bleeding edge CI jobs. From 5d3509dfda607323892f1cc96d7421612816e803 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 28 Apr 2022 11:32:50 +0100 Subject: [PATCH 084/263] Revert accidental direct-to-develop commits. This reverts commit 5a320baa45b8e826e52bdd6cadadfad727ab0357. This reverts commit f282d5fc1185dde3f9ec31c49b630cff962545d7. This reverts commit ce6ecdd4b4939fd99418bc949b40c01d39480489. --- .github/workflows/latest_deps.yml | 3 --- .github/workflows/twisted_trunk.yml | 2 -- changelog.d/12576.misc | 1 - 3 files changed, 6 deletions(-) delete mode 100644 changelog.d/12576.misc diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index c537a5a60f9f..1a61d179d905 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -32,15 +32,12 @@ jobs: with: python-version: "3.x" poetry-version: "1.2.0b1" - extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt # Upgrade all runtime dependencies only. This is intended to mimic a fresh # `pip install matrix-synapse[all]` as closely as possible. - run: poetry update --no-dev - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) - - name: Remove warn_unused_ignores from mypy config - run: sed '/warn_unused_ignores = True/d' -i mypy.ini - run: poetry run mypy trial: runs-on: ubuntu-latest diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 5f0671f3503a..8fc1affb7746 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -24,8 +24,6 @@ jobs: poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" - - name: Remove warn_unused_ignores from mypy config - run: sed '/warn_unused_ignores = True/d' -i mypy.ini - run: poetry run mypy trial: diff --git a/changelog.d/12576.misc b/changelog.d/12576.misc deleted file mode 100644 index 71022c86337f..000000000000 --- a/changelog.d/12576.misc +++ /dev/null @@ -1 +0,0 @@ -Allow unused `#type: ignore` comments in bleeding edge CI jobs. From 629aa517436c89192815bbd05a4f6ee193b9e839 Mon Sep 17 00:00:00 2001 From: "DeepBlueV7.X" Date: Thu, 28 Apr 2022 12:54:46 +0000 Subject: [PATCH 085/263] Add linebreak to pipx install quote in README (#12579) --- README.rst | 4 ++-- changelog.d/12579.doc | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12579.doc diff --git a/README.rst b/README.rst index d71d73367931..80201d4eb155 100644 --- a/README.rst +++ b/README.rst @@ -296,8 +296,8 @@ directory of your choice:: Synapse has a number of external dependencies. We maintain a fixed development environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend - pip install --user pipx - pipx install poetry + | pip install --user pipx + | pipx install poetry as described `here `_. (See `poetry's installation docs ` diff --git a/changelog.d/12579.doc b/changelog.d/12579.doc new file mode 100644 index 000000000000..bcec5fe1af24 --- /dev/null +++ b/changelog.d/12579.doc @@ -0,0 +1 @@ +Add missing linebreak to pipx install instructions. From 0b684b59e5ef761575830e9bcdfebcb13c6d2132 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:49:50 +0100 Subject: [PATCH 086/263] Fix logging of incorrect status codes for disconnected requests (#12580) The status code of requests must always be set, regardless of client disconnection, otherwise they will always be logged as 200!. Broken for `respond_with_json` in f48792eec43f893f4f893ffdcbf00f8958b6f6b5. Broken for `respond_with_json_bytes` in 3e58ce72b42f2ae473c1e76a967548cd6fa7e2e6. Broken for `respond_with_html_bytes` in ea26e9a98b0541fc886a1cb826a38352b7599dbe. Signed-off-by: Sean Quah --- changelog.d/12580.bugfix | 1 + synapse/http/server.py | 12 +++++++++--- tests/handlers/test_cas.py | 14 +++++++++++++- tests/handlers/test_saml.py | 14 +++++++++++++- 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12580.bugfix diff --git a/changelog.d/12580.bugfix b/changelog.d/12580.bugfix new file mode 100644 index 000000000000..bedce405e2ae --- /dev/null +++ b/changelog.d/12580.bugfix @@ -0,0 +1 @@ +Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing. diff --git a/synapse/http/server.py b/synapse/http/server.py index b8a7a0f5df60..1cf49830e89b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -683,6 +683,9 @@ def respond_with_json( Returns: twisted.web.server.NOT_DONE_YET if the request is still active. """ + # The response code must always be set, for logging purposes. + request.setResponseCode(code) + # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -697,7 +700,6 @@ def respond_with_json( else: encoder = _encode_json_bytes - request.setResponseCode(code) request.setHeader(b"Content-Type", b"application/json") request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") @@ -728,13 +730,15 @@ def respond_with_json_bytes( Returns: twisted.web.server.NOT_DONE_YET if the request is still active. """ + # The response code must always be set, for logging purposes. + request.setResponseCode(code) + if request._disconnected: logger.warning( "Not sending response to request %s, already disconnected.", request ) return None - request.setResponseCode(code) request.setHeader(b"Content-Type", b"application/json") request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") @@ -840,6 +844,9 @@ def respond_with_html_bytes(request: Request, code: int, html_bytes: bytes) -> N code: The HTTP response code. html_bytes: The HTML bytes to use as the response body. """ + # The response code must always be set, for logging purposes. + request.setResponseCode(code) + # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -849,7 +856,6 @@ def respond_with_html_bytes(request: Request, code: int, html_bytes: bytes) -> N ) return None - request.setResponseCode(code) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) diff --git a/tests/handlers/test_cas.py b/tests/handlers/test_cas.py index a54aa29cf177..751025c5dae0 100644 --- a/tests/handlers/test_cas.py +++ b/tests/handlers/test_cas.py @@ -201,4 +201,16 @@ def test_required_attributes(self) -> None: def _mock_request(): """Returns a mock which will stand in as a SynapseRequest""" - return Mock(spec=["getClientIP", "getHeader", "_disconnected"]) + mock = Mock( + spec=[ + "finish", + "getClientIP", + "getHeader", + "setHeader", + "setResponseCode", + "write", + ] + ) + # `_disconnected` musn't be another `Mock`, otherwise it will be truthy. + mock._disconnected = False + return mock diff --git a/tests/handlers/test_saml.py b/tests/handlers/test_saml.py index 8d4404eda10d..e2f0f90ef114 100644 --- a/tests/handlers/test_saml.py +++ b/tests/handlers/test_saml.py @@ -349,4 +349,16 @@ def test_attribute_requirements(self) -> None: def _mock_request(): """Returns a mock which will stand in as a SynapseRequest""" - return Mock(spec=["getClientIP", "getHeader", "_disconnected"]) + mock = Mock( + spec=[ + "finish", + "getClientIP", + "getHeader", + "setHeader", + "setResponseCode", + "write", + ] + ) + # `_disconnected` musn't be another `Mock`, otherwise it will be truthy. + mock._disconnected = False + return mock From 0d9eaa19fd4f934d2a249bc8fb0191fbac30dd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 28 Apr 2022 13:34:12 -0400 Subject: [PATCH 087/263] Use constants for receipt types in tests. (#12582) --- changelog.d/12582.misc | 1 + tests/handlers/test_receipts.py | 38 +++++++++---------- .../slave/storage/test_receipts.py | 11 ++++-- tests/rest/client/test_sync.py | 3 +- 4 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 changelog.d/12582.misc diff --git a/changelog.d/12582.misc b/changelog.d/12582.misc new file mode 100644 index 000000000000..5fa9c9afe86e --- /dev/null +++ b/changelog.d/12582.misc @@ -0,0 +1 @@ +Use constants for read-receipts in tests. diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index 5081b97573a0..65ab7db0c811 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -15,7 +15,7 @@ from typing import List -from synapse.api.constants import ReadReceiptEventFields +from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes from synapse.types import JsonDict from tests import unittest @@ -35,7 +35,7 @@ def test_filters_out_hidden_receipt(self): { "content": { "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@rikj:jki.re": { "ts": 1436451550453, "hidden": True, @@ -56,7 +56,7 @@ def test_does_not_filter_out_our_hidden_receipt(self): { "content": { "$1435641916hfgh4394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@me:server.org": { "ts": 1436451550453, "hidden": True, @@ -72,7 +72,7 @@ def test_does_not_filter_out_our_hidden_receipt(self): { "content": { "$1435641916hfgh4394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@me:server.org": { "ts": 1436451550453, ReadReceiptEventFields.MSC2285_HIDDEN: True, @@ -92,7 +92,7 @@ def test_filters_out_hidden_receipt_and_ignores_rest(self): { "content": { "$1dgdgrd5641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@rikj:jki.re": { "ts": 1436451550453, "hidden": True, @@ -111,7 +111,7 @@ def test_filters_out_hidden_receipt_and_ignores_rest(self): { "content": { "$1dgdgrd5641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -130,7 +130,7 @@ def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self): { "content": { "$14356419edgd14394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@rikj:jki.re": { "ts": 1436451550453, "hidden": True, @@ -138,7 +138,7 @@ def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self): } }, "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -153,7 +153,7 @@ def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self): { "content": { "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -171,9 +171,9 @@ def test_handles_missing_content_of_m_read(self): [ { "content": { - "$14356419ggffg114394fHBLK:matrix.org": {"m.read": {}}, + "$14356419ggffg114394fHBLK:matrix.org": {ReceiptTypes.READ: {}}, "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -187,9 +187,9 @@ def test_handles_missing_content_of_m_read(self): [ { "content": { - "$14356419ggffg114394fHBLK:matrix.org": {"m.read": {}}, + "$14356419ggffg114394fHBLK:matrix.org": {ReceiptTypes.READ: {}}, "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -209,7 +209,7 @@ def test_handles_empty_event(self): "content": { "$143564gdfg6114394fHBLK:matrix.org": {}, "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -225,7 +225,7 @@ def test_handles_empty_event(self): "content": { "$143564gdfg6114394fHBLK:matrix.org": {}, "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -244,7 +244,7 @@ def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(sel { "content": { "$14356419edgd14394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@rikj:jki.re": { "ts": 1436451550453, "hidden": True, @@ -258,7 +258,7 @@ def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(sel { "content": { "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -273,7 +273,7 @@ def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(sel { "content": { "$1435641916114394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, } @@ -297,7 +297,7 @@ def test_handles_string_data(self): { "content": { "$14356419edgd14394fHBLK:matrix.org": { - "m.read": { + ReceiptTypes.READ: { "@rikj:jki.re": "string", } }, diff --git a/tests/replication/slave/storage/test_receipts.py b/tests/replication/slave/storage/test_receipts.py index f47d94f690f9..de19e75b9dec 100644 --- a/tests/replication/slave/storage/test_receipts.py +++ b/tests/replication/slave/storage/test_receipts.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.api.constants import ReceiptTypes from synapse.replication.slave.storage.receipts import SlavedReceiptsStore from ._base import BaseSlavedStoreTestCase @@ -26,9 +27,13 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase): STORE_TYPE = SlavedReceiptsStore def test_receipt(self): - self.check("get_receipts_for_user", [USER_ID, "m.read"], {}) + self.check("get_receipts_for_user", [USER_ID, ReceiptTypes.READ], {}) self.get_success( - self.master_store.insert_receipt(ROOM_ID, "m.read", USER_ID, [EVENT_ID], {}) + self.master_store.insert_receipt( + ROOM_ID, ReceiptTypes.READ, USER_ID, [EVENT_ID], {} + ) ) self.replicate() - self.check("get_receipts_for_user", [USER_ID, "m.read"], {ROOM_ID: EVENT_ID}) + self.check( + "get_receipts_for_user", [USER_ID, ReceiptTypes.READ], {ROOM_ID: EVENT_ID} + ) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 773c16a54ce6..cb765455c16d 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -24,6 +24,7 @@ EventContentFields, EventTypes, ReadReceiptEventFields, + ReceiptTypes, RelationTypes, ) from synapse.rest.client import devices, knock, login, read_marker, receipts, room, sync @@ -560,7 +561,7 @@ def test_unread_counts(self) -> None: self._check_unread_count(1) # Send a read receipt to tell the server we've read the latest event. - body = json.dumps({"m.read": res["event_id"]}).encode("utf8") + body = json.dumps({ReceiptTypes.READ: res["event_id"]}).encode("utf8") channel = self.make_request( "POST", "/rooms/%s/read_markers" % self.room_id, From 3ae56d125c95fe6919cdf7892b9e6a6cdba651a4 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 28 Apr 2022 13:58:58 -0400 Subject: [PATCH 088/263] Improve the docstrings for the receipts store. (#12581) --- changelog.d/12581.misc | 1 + synapse/storage/databases/main/receipts.py | 56 ++++++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12581.misc diff --git a/changelog.d/12581.misc b/changelog.d/12581.misc new file mode 100644 index 000000000000..38d40b262b25 --- /dev/null +++ b/changelog.d/12581.misc @@ -0,0 +1 @@ +Improve docstrings for the receipts store. diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 332e901dda40..7d96f4fedabf 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -122,10 +122,21 @@ async def get_users_with_read_receipts_in_room(self, room_id: str) -> Set[str]: receipts = await self.get_receipts_for_room(room_id, ReceiptTypes.READ) return {r["user_id"] for r in receipts} - @cached(num_args=2) + @cached() async def get_receipts_for_room( self, room_id: str, receipt_type: str ) -> List[Dict[str, Any]]: + """ + Fetch the event IDs for the latest receipt for all users in a room with the given receipt type. + + Args: + room_id: The room ID to fetch the receipt for. + receipt_type: The receipt type to fetch. + + Returns: + A list of dictionaries, one for each user ID. Each dictionary + contains a user ID and the event ID of that user's latest receipt. + """ return await self.db_pool.simple_select_list( table="receipts_linearized", keyvalues={"room_id": room_id, "receipt_type": receipt_type}, @@ -133,10 +144,21 @@ async def get_receipts_for_room( desc="get_receipts_for_room", ) - @cached(num_args=3) + @cached() async def get_last_receipt_event_id_for_user( self, user_id: str, room_id: str, receipt_type: str ) -> Optional[str]: + """ + Fetch the event ID for the latest receipt in a room with the given receipt type. + + Args: + user_id: The user to fetch receipts for. + room_id: The room ID to fetch the receipt for. + receipt_type: The receipt type to fetch. + + Returns: + The event ID of the latest receipt, if one exists; otherwise `None`. + """ return await self.db_pool.simple_select_one_onecol( table="receipts_linearized", keyvalues={ @@ -149,10 +171,23 @@ async def get_last_receipt_event_id_for_user( allow_none=True, ) - @cached(num_args=2) + @cached() async def get_receipts_for_user( self, user_id: str, receipt_type: str ) -> Dict[str, str]: + """ + Fetch the event IDs for the latest receipts sent by the given user. + + Args: + user_id: The user to fetch receipts for. + receipt_type: The receipt type to fetch. + + Returns: + A map of room ID to the event ID of the latest receipt for that room. + + If the user has not sent a receipt to a room then it will not appear + in the returned dictionary. + """ rows = await self.db_pool.simple_select_list( table="receipts_linearized", keyvalues={"user_id": user_id, "receipt_type": receipt_type}, @@ -165,6 +200,17 @@ async def get_receipts_for_user( async def get_receipts_for_user_with_orderings( self, user_id: str, receipt_type: str ) -> JsonDict: + """ + Fetch receipts for all rooms that the given user is joined to. + + Args: + user_id: The user to fetch receipts for. + receipt_type: The receipt type to fetch. + + Returns: + A map of room ID to the latest receipt information. + """ + def f(txn: LoggingTransaction) -> List[Tuple[str, str, int, int]]: sql = ( "SELECT rl.room_id, rl.event_id," @@ -241,7 +287,7 @@ async def get_linearized_receipts_for_room( return await self._get_linearized_receipts_for_room(room_id, to_key, from_key) - @cached(num_args=3, tree=True) + @cached(tree=True) async def _get_linearized_receipts_for_room( self, room_id: str, to_key: int, from_key: Optional[int] = None ) -> List[JsonDict]: @@ -541,7 +587,7 @@ def insert_linearized_receipt_txn( data: JsonDict, stream_id: int, ) -> Optional[int]: - """Inserts a read-receipt into the database if it's newer than the current RR + """Inserts a receipt into the database if it's newer than the current one. Returns: None if the RR is older than the current RR From 57fac2a2347e0b386385868cdd1ae435364062b5 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 29 Apr 2022 17:57:23 +0100 Subject: [PATCH 089/263] Allow unused ignores in "bleeding edge" CI (#12576) * Allow unused ignores in "bleeding edge" CI Where "bleeding edge" means the Twisted Trunk and Latest Deps jobs. Follow up from #12531. Resolves #12574. * Use `--extras all` in latest deps mypy CI Twisted trunk job already does this. Missed in #12531. * changelog --- .github/workflows/latest_deps.yml | 3 +++ .github/workflows/twisted_trunk.yml | 2 ++ changelog.d/12576.misc | 1 + 3 files changed, 6 insertions(+) create mode 100644 changelog.d/12576.misc diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 1a61d179d905..c537a5a60f9f 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -32,12 +32,15 @@ jobs: with: python-version: "3.x" poetry-version: "1.2.0b1" + extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt # Upgrade all runtime dependencies only. This is intended to mimic a fresh # `pip install matrix-synapse[all]` as closely as possible. - run: poetry update --no-dev - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true) + - name: Remove warn_unused_ignores from mypy config + run: sed '/warn_unused_ignores = True/d' -i mypy.ini - run: poetry run mypy trial: runs-on: ubuntu-latest diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 8fc1affb7746..5f0671f3503a 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -24,6 +24,8 @@ jobs: poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk poetry install --no-interaction --extras "all test" + - name: Remove warn_unused_ignores from mypy config + run: sed '/warn_unused_ignores = True/d' -i mypy.ini - run: poetry run mypy trial: diff --git a/changelog.d/12576.misc b/changelog.d/12576.misc new file mode 100644 index 000000000000..71022c86337f --- /dev/null +++ b/changelog.d/12576.misc @@ -0,0 +1 @@ +Allow unused `#type: ignore` comments in bleeding edge CI jobs. From 8d156ec0ba17d848581f18aa40ebfd76dda763d4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 29 Apr 2022 22:05:18 +0100 Subject: [PATCH 090/263] Remove special-case for `twisted` logger (#12589) This was originally added when we first added a `MemoryHandler` to the default log config back in https://github.com/matrix-org/synapse/pull/8040, to ensure that we didn't explode with an infinite loop if there was an error formatting the logs. Since then, we made additional improvements to logging which make this workaround redundant. In particular: * we no longer attempt to log un-UTF8-decodable byte sequences, which were the most likely cause of an error in the first place. * https://github.com/matrix-org/synapse/pull/8268 ensures that in the unlikely case that there *is* an error, it won't cause an infinite loop. --- changelog.d/12589.misc | 1 + docs/sample_log_config.yaml | 7 ------- synapse/config/logger.py | 7 ------- 3 files changed, 1 insertion(+), 14 deletions(-) create mode 100644 changelog.d/12589.misc diff --git a/changelog.d/12589.misc b/changelog.d/12589.misc new file mode 100644 index 000000000000..d362828d2e59 --- /dev/null +++ b/changelog.d/12589.misc @@ -0,0 +1 @@ +Remove special-case for `twisted` logger from default log config. diff --git a/docs/sample_log_config.yaml b/docs/sample_log_config.yaml index 2485ad25edfc..3065a0e2d986 100644 --- a/docs/sample_log_config.yaml +++ b/docs/sample_log_config.yaml @@ -62,13 +62,6 @@ loggers: # information such as access tokens. level: INFO - twisted: - # We send the twisted logging directly to the file handler, - # to work around https://github.com/matrix-org/synapse/issues/3471 - # when using "buffer" logger. Use "console" to log to stderr instead. - handlers: [file] - propagate: false - root: level: INFO diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 99db9e1e3910..470b8b44929c 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -110,13 +110,6 @@ # information such as access tokens. level: INFO - twisted: - # We send the twisted logging directly to the file handler, - # to work around https://github.com/matrix-org/synapse/issues/3471 - # when using "buffer" logger. Use "console" to log to stderr instead. - handlers: [file] - propagate: false - root: level: INFO From 8f5d2823dff26d4c8756f1db348b3b4908ef27f5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 3 May 2022 10:53:09 +0100 Subject: [PATCH 091/263] 1.58.0 --- CHANGES.md | 6 ++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1fbe0815dedf..f0fc4e5bae0f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 1.58.0 (2022-05-03) +=========================== + +No significant changes since 1.58.0rc2. + + Synapse 1.58.0rc2 (2022-04-26) ============================== diff --git a/debian/changelog b/debian/changelog index 5f1bf872bbcc..53b2387776f5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.58.0) stable; urgency=medium + + * New Synapse release 1.58.0. + + -- Synapse Packaging team Tue, 03 May 2022 10:52:58 +0100 + matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium * New Synapse release 1.58.0rc2. diff --git a/pyproject.toml b/pyproject.toml index bdded7843440..f0f029f0169f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0rc2" +version = "1.58.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 7e6598bcf6cf63b4d04c27b9696ed8429b6ddff6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 3 May 2022 10:54:20 +0100 Subject: [PATCH 092/263] Move groups/communities deprecation notice to 1.58.0 heading --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f0fc4e5bae0f..31f156127424 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.58.0 (2022-05-03) =========================== +As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. + No significant changes since 1.58.0rc2. @@ -25,8 +27,6 @@ Internal Changes Synapse 1.58.0rc1 (2022-04-26) ============================== -As of this release, the groups/communities feature in Synapse is now disabled by default. See [\#11584](https://github.com/matrix-org/synapse/issues/11584) for details. As mentioned in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1580), this feature will be removed in Synapse 1.61. - Features -------- From 01dcf7532d7863b2721156a4817716a8628fe73f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 3 May 2022 11:03:20 +0100 Subject: [PATCH 093/263] Prune mypy ignore_missing_imports list (#12608) --- changelog.d/12608.misc | 1 + mypy.ini | 53 ++---------------------------------------- 2 files changed, 3 insertions(+), 51 deletions(-) create mode 100644 changelog.d/12608.misc diff --git a/changelog.d/12608.misc b/changelog.d/12608.misc new file mode 100644 index 000000000000..38272118fbe8 --- /dev/null +++ b/changelog.d/12608.misc @@ -0,0 +1 @@ +Remove redundant lines of config from `mypy.ini`. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index ef28216418a5..78699e318704 100644 --- a/mypy.ini +++ b/mypy.ini @@ -241,98 +241,49 @@ disallow_untyped_defs = True [mypy-authlib.*] ignore_missing_imports = True -[mypy-bcrypt] -ignore_missing_imports = True - [mypy-canonicaljson] ignore_missing_imports = True [mypy-constantly] ignore_missing_imports = True -[mypy-daemonize] -ignore_missing_imports = True - -[mypy-h11] -ignore_missing_imports = True - -[mypy-hiredis] -ignore_missing_imports = True - -[mypy-hyperlink] -ignore_missing_imports = True - [mypy-ijson.*] ignore_missing_imports = True -[mypy-importlib_metadata.*] -ignore_missing_imports = True - -[mypy-jaeger_client.*] -ignore_missing_imports = True - -[mypy-josepy.*] -ignore_missing_imports = True - -[mypy-jwt.*] -ignore_missing_imports = True - [mypy-lxml] ignore_missing_imports = True [mypy-msgpack] ignore_missing_imports = True -[mypy-nacl.*] -ignore_missing_imports = True - +# Note: WIP stubs available at +# https://github.com/microsoft/python-type-stubs/tree/64934207f523ad6b611e6cfe039d85d7175d7d0d/netaddr [mypy-netaddr] ignore_missing_imports = True [mypy-parameterized.*] ignore_missing_imports = True -[mypy-phonenumbers.*] -ignore_missing_imports = True - -[mypy-prometheus_client.*] -ignore_missing_imports = True - [mypy-pymacaroons.*] ignore_missing_imports = True [mypy-pympler.*] ignore_missing_imports = True -[mypy-redbaron.*] -ignore_missing_imports = True - [mypy-rust_python_jaeger_reporter.*] ignore_missing_imports = True [mypy-saml2.*] ignore_missing_imports = True -[mypy-sentry_sdk] -ignore_missing_imports = True - [mypy-service_identity.*] ignore_missing_imports = True -[mypy-signedjson.*] -ignore_missing_imports = True - [mypy-srvlookup.*] ignore_missing_imports = True [mypy-treq.*] ignore_missing_imports = True -[mypy-twisted.*] -ignore_missing_imports = True - -[mypy-zope] -ignore_missing_imports = True - [mypy-incremental.*] ignore_missing_imports = True From ae7858f184f4fd1533d0a5dd70174e3c70f529ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 May 2022 11:47:21 +0100 Subject: [PATCH 094/263] Fix race when persisting an event and deleting a room (#12594) This works by taking a row level lock on the `rooms` table at the start of both transactions, ensuring that they don't run at the same time. In the event persistence transaction we also check that there is an entry still in the `rooms` table. I can't figure out how to do this in SQLite. I was just going to lock the table, but it seems that we don't support that in SQLite either, so I'm *really* confused as to how we maintain integrity in SQLite when using `lock_table`.... --- changelog.d/12594.bugfix | 1 + synapse/storage/databases/main/events.py | 15 +++++++++++++++ synapse/storage/databases/main/purge_events.py | 8 ++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12594.bugfix diff --git a/changelog.d/12594.bugfix b/changelog.d/12594.bugfix new file mode 100644 index 000000000000..7411d9c07934 --- /dev/null +++ b/changelog.d/12594.bugfix @@ -0,0 +1 @@ +Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 2a1e567ce08e..9a6c2fd47a55 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -47,6 +47,7 @@ ) from synapse.storage.databases.main.events_worker import EventCacheEntry from synapse.storage.databases.main.search import SearchEntry +from synapse.storage.engines.postgres import PostgresEngine from synapse.storage.util.id_generators import AbstractStreamIdGenerator from synapse.storage.util.sequence import SequenceGenerator from synapse.types import StateMap, get_domain_from_id @@ -364,6 +365,20 @@ def _persist_events_txn( min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering max_stream_order = events_and_contexts[-1][0].internal_metadata.stream_ordering + # We check that the room still exists for events we're trying to + # persist. This is to protect against races with deleting a room. + # + # Annoyingly SQLite doesn't support row level locking. + if isinstance(self.database_engine, PostgresEngine): + for room_id in {e.room_id for e, _ in events_and_contexts}: + txn.execute( + "SELECT room_version FROM rooms WHERE room_id = ? FOR SHARE", + (room_id,), + ) + row = txn.fetchone() + if row is None: + raise Exception(f"Room does not exist {room_id}") + # stream orderings should have been assigned by now assert min_stream_order assert max_stream_order diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index 2e3818e43244..bfc85b3add98 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -324,7 +324,12 @@ async def purge_room(self, room_id: str) -> List[int]: ) def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: - # First we fetch all the state groups that should be deleted, before + # We *immediately* delete the room from the rooms table. This ensures + # that we don't race when persisting events (as that transaction checks + # that the room exists). + txn.execute("DELETE FROM rooms WHERE room_id = ?", (room_id,)) + + # Next, we fetch all the state groups that should be deleted, before # we delete that information. txn.execute( """ @@ -403,7 +408,6 @@ def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: "room_stats_state", "room_stats_current", "room_stats_earliest_token", - "rooms", "stream_ordering_to_exterm", "users_in_public_rooms", "users_who_share_private_rooms", From bf2fea8f7dace1aa6597b9fc8fa782433409d718 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 3 May 2022 11:50:03 +0100 Subject: [PATCH 095/263] Add sanity checks to the release script (#12556) Check we're on the right branch before tagging, and on the right tag before uploading * Abort if we're on the wrong branch * Check we have the right tag checked out * Clarify that `publish` only releases to GitHub --- changelog.d/12556.misc | 1 + scripts-dev/release.py | 63 +++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 changelog.d/12556.misc diff --git a/changelog.d/12556.misc b/changelog.d/12556.misc new file mode 100644 index 000000000000..dc245397fbcd --- /dev/null +++ b/changelog.d/12556.misc @@ -0,0 +1 @@ +Release script: confirm the commit to be tagged before tagging. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 14f3f3a45d8a..f4269e09bb7f 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -89,13 +89,7 @@ def prepare() -> None: """ # Make sure we're in a git repo. - try: - repo = git.Repo() - except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") - - if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + repo = get_repo_and_check_clean_checkout() click.secho("Updating git repo...") repo.remote().fetch() @@ -171,9 +165,7 @@ def prepare() -> None: assert not parsed_new_version.is_devrelease assert not parsed_new_version.is_postrelease - release_branch_name = ( - f"release-v{parsed_new_version.major}.{parsed_new_version.minor}" - ) + release_branch_name = get_release_branch_name(parsed_new_version) release_branch = find_ref(repo, release_branch_name) if release_branch: if release_branch.is_remote(): @@ -274,13 +266,7 @@ def tag(gh_token: Optional[str]) -> None: """Tags the release and generates a draft GitHub release""" # Make sure we're in a git repo. - try: - repo = git.Repo() - except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") - - if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + repo = get_repo_and_check_clean_checkout() click.secho("Updating git repo...") repo.remote().fetch() @@ -293,6 +279,15 @@ def tag(gh_token: Optional[str]) -> None: if tag_name in repo.tags: raise click.ClickException(f"Tag {tag_name} already exists!\n") + # Check we're on the right release branch + release_branch = get_release_branch_name(current_version) + if repo.active_branch.name != release_branch: + click.echo( + f"Need to be on the release branch ({release_branch}) before tagging. " + f"Currently on ({repo.active_branch.name})." + ) + click.get_current_context().abort() + # Get the appropriate changelogs and tag. changes = get_changes_for_version(current_version) @@ -358,21 +353,15 @@ def tag(gh_token: Optional[str]) -> None: @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) def publish(gh_token: str) -> None: - """Publish release.""" + """Publish release on GitHub.""" # Make sure we're in a git repo. - try: - repo = git.Repo() - except git.InvalidGitRepositoryError: - raise click.ClickException("Not in Synapse repo.") - - if repo.is_dirty(): - raise click.ClickException("Uncommitted changes exist.") + get_repo_and_check_clean_checkout() current_version = get_package_version() tag_name = f"v{current_version}" - if not click.confirm(f"Publish {tag_name}?", default=True): + if not click.confirm(f"Publish release {tag_name} on GitHub?", default=True): return # Publish the draft release @@ -406,6 +395,13 @@ def upload() -> None: current_version = get_package_version() tag_name = f"v{current_version}" + # Check we have the right tag checked out. + repo = get_repo_and_check_clean_checkout() + tag = repo.tag(f"refs/tags/{tag_name}") + if repo.head.commit != tag.commit: + click.echo("Tag {tag_name} (tag.commit) is not currently checked out!") + click.get_current_context().abort() + pypi_asset_names = [ f"matrix_synapse-{current_version}-py3-none-any.whl", f"matrix-synapse-{current_version}.tar.gz", @@ -469,6 +465,21 @@ def get_package_version() -> version.Version: return version.Version(version_string) +def get_release_branch_name(version_number: version.Version) -> str: + return f"release-v{version_number.major}.{version_number.minor}" + + +def get_repo_and_check_clean_checkout() -> git.Repo: + """Get the project repo and check it's not got any uncommitted changes.""" + try: + repo = git.Repo() + except git.InvalidGitRepositoryError: + raise click.ClickException("Not in Synapse repo.") + if repo.is_dirty(): + raise click.ClickException("Uncommitted changes exist.") + return repo + + def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]: """Find the branch/ref, looking first locally then in the remote.""" if ref_name in repo.references: From db2edf5a65c5bcac565e052b2dbd74253755a717 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 3 May 2022 13:47:56 +0100 Subject: [PATCH 096/263] Exclude OOB memberships from the federation sender (#12570) As the comment says, there is no need to process such events, and indeed we need to avoid doing so. Fixes #12509. --- changelog.d/12570.bugfix | 1 + synapse/events/__init__.py | 15 ++++++++--- synapse/federation/sender/__init__.py | 39 +++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12570.bugfix diff --git a/changelog.d/12570.bugfix b/changelog.d/12570.bugfix new file mode 100644 index 000000000000..1038646f358d --- /dev/null +++ b/changelog.d/12570.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 9acb3c0cc454..3e4b4f0384ac 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -213,10 +213,17 @@ def is_outlier(self) -> bool: return self.outlier def is_out_of_band_membership(self) -> bool: - """Whether this is an out of band membership, like an invite or an invite - rejection. This is needed as those events are marked as outliers, but - they still need to be processed as if they're new events (e.g. updating - invite state in the database, relaying to clients, etc). + """Whether this event is an out-of-band membership. + + OOB memberships are a special case of outlier events: they are membership events + for federated rooms that we aren't full members. Examples include invites + received over federation, and rejections for such invites. + + The concept of an OOB membership is needed because these events need to be + processed as if they're new regular events (e.g. updating membership state in + the database, relaying to clients via /sync, etc) despite being outliers. + + See also https://matrix-org.github.io/synapse/develop/development/room-dag-concepts.html#out-of-band-membership-events. (Added in synapse 0.99.0, so may be unreliable for events received before that) """ diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 30e2421efc6d..d795b7b26bcc 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -355,6 +355,45 @@ async def handle_event(event: EventBase) -> None: if not is_mine and send_on_behalf_of is None: return + # We also want to not send out-of-band membership events. + # + # OOB memberships are used in three (and a half) situations: + # + # (1) invite events which we have received over federation. Those + # will have a `sender` on a different server, so will be + # skipped by the "is_mine" test above anyway. + # + # (2) rejections of invites to federated rooms - either remotely + # or locally generated. (Such rejections are normally + # created via federation, in which case the remote server is + # responsible for sending out the rejection. If that fails, + # we'll create a leave event locally, but that's only really + # for the benefit of the invited user - we don't have enough + # information to send it out over federation). + # + # (2a) rescinded knocks. These are identical to rejected invites. + # + # (3) knock events which we have sent over federation. As with + # invite rejections, the remote server should send them out to + # the federation. + # + # So, in all the above cases, we want to ignore such events. + # + # OOB memberships are always(?) outliers anyway, so if we *don't* + # ignore them, we'll get an exception further down when we try to + # fetch the membership list for the room. + # + # Arguably, we could equivalently ignore all outliers here, since + # in theory the only way for an outlier with a local `sender` to + # exist is by being an OOB membership (via one of (2), (2a) or (3) + # above). + # + if event.internal_metadata.is_out_of_band_membership(): + return + + # Finally, there are some other events that we should not send out + # until someone asks for them. They are explicitly flagged as such + # with `proactively_send: False`. if not event.internal_metadata.should_proactively_send(): return From 5938928c59bac95670c3e582025dd6d819992e4f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 3 May 2022 13:50:50 +0100 Subject: [PATCH 097/263] minor wording fix in docstring --- synapse/events/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 3e4b4f0384ac..56c6ccc22c18 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import abc -import os +import abcimport os from typing import ( TYPE_CHECKING, Any, @@ -216,7 +215,7 @@ def is_out_of_band_membership(self) -> bool: """Whether this event is an out-of-band membership. OOB memberships are a special case of outlier events: they are membership events - for federated rooms that we aren't full members. Examples include invites + for federated rooms that we aren't full members of. Examples include invites received over federation, and rejections for such invites. The concept of an OOB membership is needed because these events need to be From 77dee1b451aa5c5c3644e5e4e5afefda2a44aa15 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 3 May 2022 13:59:28 +0100 Subject: [PATCH 098/263] fix imports broken in 5938928 :-S --- synapse/events/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 56c6ccc22c18..c238376caf62 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import abcimport os +import abc +import os from typing import ( TYPE_CHECKING, Any, From c4514b97db551215334c4128cbc8ef12def560ba Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 3 May 2022 15:46:42 +0100 Subject: [PATCH 099/263] Add missing space before 'docker' link in release announcement script (#12612) --- changelog.d/12612.bugfix | 1 + scripts-dev/release.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12612.bugfix diff --git a/changelog.d/12612.bugfix b/changelog.d/12612.bugfix new file mode 100644 index 000000000000..c39e97f0cbaa --- /dev/null +++ b/changelog.d/12612.bugfix @@ -0,0 +1 @@ +Fix a typo in the announcement text generated by the Synapse release development script. \ No newline at end of file diff --git a/scripts-dev/release.py b/scripts-dev/release.py index f4269e09bb7f..0031ba3e4b2f 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -434,7 +434,7 @@ def announce() -> None: f""" Hi everyone. Synapse {current_version} has just been released. -[notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) |\ +[notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) | \ [docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \ [debs](https://packages.matrix.org/debian/) | \ [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)""" From d66d68f9177d3addc32f6ab02458e3620f5ffc1f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 3 May 2022 16:32:40 +0100 Subject: [PATCH 100/263] Add extra debug logging to federation sender (#12614) ... in order to debug some problems we've been having with certain events not being sent when expected. --- changelog.d/12614.misc | 1 + synapse/federation/sender/__init__.py | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12614.misc diff --git a/changelog.d/12614.misc b/changelog.d/12614.misc new file mode 100644 index 000000000000..79022df127d0 --- /dev/null +++ b/changelog.d/12614.misc @@ -0,0 +1 @@ +Add extra debug logging to federation sender. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index d795b7b26bcc..6d2f46318bea 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -343,9 +343,16 @@ async def _process_event_queue_loop(self) -> None: last_token, self._last_poked_id, limit=100 ) - logger.debug("Handling %s -> %s", last_token, next_token) + logger.debug( + "Handling %i -> %i: %i events to send (current id %i)", + last_token, + next_token, + len(events), + self._last_poked_id, + ) if not events and next_token >= self._last_poked_id: + logger.debug("All events processed") break async def handle_event(event: EventBase) -> None: @@ -353,6 +360,7 @@ async def handle_event(event: EventBase) -> None: send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of() is_mine = self.is_mine_id(event.sender) if not is_mine and send_on_behalf_of is None: + logger.debug("Not sending remote-origin event %s", event) return # We also want to not send out-of-band membership events. @@ -389,12 +397,16 @@ async def handle_event(event: EventBase) -> None: # above). # if event.internal_metadata.is_out_of_band_membership(): + logger.debug("Not sending OOB membership event %s", event) return # Finally, there are some other events that we should not send out # until someone asks for them. They are explicitly flagged as such # with `proactively_send: False`. if not event.internal_metadata.should_proactively_send(): + logger.debug( + "Not sending event with proactively_send=false: %s", event + ) return destinations: Optional[Set[str]] = None @@ -458,7 +470,10 @@ async def handle_event(event: EventBase) -> None: "federation_sender" ).observe((now - ts) / 1000) - async def handle_room_events(events: Iterable[EventBase]) -> None: + async def handle_room_events(events: List[EventBase]) -> None: + logger.debug( + "Handling %i events in room %s", len(events), events[0].room_id + ) with Measure(self.clock, "handle_room_events"): for event in events: await handle_event(event) @@ -477,6 +492,7 @@ async def handle_room_events(events: Iterable[EventBase]) -> None: ) ) + logger.debug("Successfully handled up to %i", next_token) await self.store.update_federation_out_pos("events", next_token) if events: From aa5f5ede3331bf916f459137e9d0df3ac0743093 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 3 May 2022 12:43:12 -0400 Subject: [PATCH 101/263] Remove unstable identifiers for MSC3069. (#12596) --- changelog.d/12596.removal | 1 + synapse/rest/client/account.py | 2 -- tests/rest/client/test_account.py | 6 ------ 3 files changed, 1 insertion(+), 8 deletions(-) create mode 100644 changelog.d/12596.removal diff --git a/changelog.d/12596.removal b/changelog.d/12596.removal new file mode 100644 index 000000000000..14fbfb39540a --- /dev/null +++ b/changelog.d/12596.removal @@ -0,0 +1 @@ +Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 5587cae98a61..bdc4a9c0683d 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -882,9 +882,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: response = { "user_id": requester.user.to_string(), - # MSC: https://github.com/matrix-org/matrix-doc/pull/3069 # Entered spec in Matrix 1.2 - "org.matrix.msc3069.is_guest": bool(requester.is_guest), "is_guest": bool(requester.is_guest), } diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index e00b5c171c78..e0a11da97b67 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -520,8 +520,6 @@ def test_GET_whoami(self) -> None: { "user_id": user_id, "device_id": device_id, - # MSC3069 entered spec in Matrix 1.2 but maintained compatibility - "org.matrix.msc3069.is_guest": False, "is_guest": False, }, ) @@ -540,8 +538,6 @@ def test_GET_whoami_guests(self) -> None: { "user_id": user_id, "device_id": device_id, - # MSC3069 entered spec in Matrix 1.2 but maintained compatibility - "org.matrix.msc3069.is_guest": True, "is_guest": True, }, ) @@ -564,8 +560,6 @@ def test_GET_whoami_appservices(self) -> None: whoami, { "user_id": user_id, - # MSC3069 entered spec in Matrix 1.2 but maintained compatibility - "org.matrix.msc3069.is_guest": False, "is_guest": False, }, ) From 9ce51a47f6e37abd0a1275281806399d874eb026 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 3 May 2022 19:22:06 +0100 Subject: [PATCH 102/263] Bump Synapse minimum Python version to 3.7.1 (#12613) --- changelog.d/12613.removal | 1 + poetry.lock | 4 ++-- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12613.removal diff --git a/changelog.d/12613.removal b/changelog.d/12613.removal new file mode 100644 index 000000000000..b1a9e207b00f --- /dev/null +++ b/changelog.d/12613.removal @@ -0,0 +1 @@ +Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. diff --git a/poetry.lock b/poetry.lock index e27a44989cd4..89e78576b7b6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1561,8 +1561,8 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" -python-versions = "^3.7" -content-hash = "3825cef058b8c9f520ef4b7acb92519be95db9a663a61c2e89a5fe431ed55655" +python-versions = "^3.7.1" +content-hash = "2bda1a7cfc8cc02832b4a7d16bf7e1615cb05e0639bdb30688aadf692d851942" [metadata.files] attrs = [ diff --git a/pyproject.toml b/pyproject.toml index 62e26fd95b40..cf3982d12467 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,7 +100,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main" update_synapse_database = "synapse._scripts.update_synapse_database:main" [tool.poetry.dependencies] -python = "^3.7" +python = "^3.7.1" # Mandatory Dependencies # ---------------------- From 96e0cdbc5af0563ee805ec4e588e1df14899af66 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 3 May 2022 21:27:52 +0100 Subject: [PATCH 103/263] Add a consistency check on events read from the database (#12620) I've seen a few errors which can only plausibly be explained by the calculated event id for an event being different from the ID of the event in the database. It should be cheap to check this, so let's do so and raise an exception. --- changelog.d/12620.misc | 1 + .../storage/databases/main/events_worker.py | 12 ++++ .../databases/main/test_events_worker.py | 59 ++++++++++++------- 3 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 changelog.d/12620.misc diff --git a/changelog.d/12620.misc b/changelog.d/12620.misc new file mode 100644 index 000000000000..63f8e540c37c --- /dev/null +++ b/changelog.d/12620.misc @@ -0,0 +1 @@ +Add a consistency check on events which we read from the database. diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index c31fc00eaace..0a48e5d29f18 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1094,6 +1094,18 @@ async def _get_events_from_db( original_ev.internal_metadata.stream_ordering = row.stream_ordering original_ev.internal_metadata.outlier = row.outlier + # Consistency check: if the content of the event has been modified in the + # database, then the calculated event ID will not match the event id in the + # database. + if original_ev.event_id != event_id: + # it's difficult to see what to do here. Pretty much all bets are off + # if Synapse cannot rely on the consistency of its database. + raise RuntimeError( + f"Database corruption: Event {event_id} in room {d['room_id']} " + f"from the database appears to have been modified (calculated " + f"event id {original_ev.event_id})" + ) + event_map[event_id] = original_ev # finally, we can decide whether each one needs redacting, and build diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index bf6374f93d52..c237a8c7e228 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -13,7 +13,7 @@ # limitations under the License. import json from contextlib import contextmanager -from typing import Generator, Tuple +from typing import Generator, List, Tuple from unittest import mock from twisted.enterprise.adbapi import ConnectionPool @@ -21,6 +21,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.room_versions import EventFormatVersions, RoomVersions +from synapse.events import make_event_from_dict from synapse.logging.context import LoggingContext from synapse.rest import admin from synapse.rest.client import login, room @@ -49,23 +50,28 @@ def prepare(self, reactor, clock, hs): ) ) - for idx, (rid, eid) in enumerate( + self.event_ids: List[str] = [] + for idx, rid in enumerate( ( - ("room1", "event10"), - ("room1", "event11"), - ("room1", "event12"), - ("room2", "event20"), + "room1", + "room1", + "room1", + "room2", ) ): + event_json = {"type": f"test {idx}", "room_id": rid} + event = make_event_from_dict(event_json, room_version=RoomVersions.V4) + event_id = event.event_id + self.get_success( self.store.db_pool.simple_insert( "events", { - "event_id": eid, + "event_id": event_id, "room_id": rid, "topological_ordering": idx, "stream_ordering": idx, - "type": "test", + "type": event.type, "processed": True, "outlier": False, }, @@ -75,21 +81,22 @@ def prepare(self, reactor, clock, hs): self.store.db_pool.simple_insert( "event_json", { - "event_id": eid, + "event_id": event_id, "room_id": rid, - "json": json.dumps({"type": "test", "room_id": rid}), + "json": json.dumps(event_json), "internal_metadata": "{}", "format_version": 3, }, ) ) + self.event_ids.append(event_id) def test_simple(self): with LoggingContext(name="test") as ctx: res = self.get_success( - self.store.have_seen_events("room1", ["event10", "event19"]) + self.store.have_seen_events("room1", [self.event_ids[0], "event19"]) ) - self.assertEqual(res, {"event10"}) + self.assertEqual(res, {self.event_ids[0]}) # that should result in a single db query self.assertEqual(ctx.get_resource_usage().db_txn_count, 1) @@ -97,19 +104,21 @@ def test_simple(self): # a second lookup of the same events should cause no queries with LoggingContext(name="test") as ctx: res = self.get_success( - self.store.have_seen_events("room1", ["event10", "event19"]) + self.store.have_seen_events("room1", [self.event_ids[0], "event19"]) ) - self.assertEqual(res, {"event10"}) + self.assertEqual(res, {self.event_ids[0]}) self.assertEqual(ctx.get_resource_usage().db_txn_count, 0) def test_query_via_event_cache(self): # fetch an event into the event cache - self.get_success(self.store.get_event("event10")) + self.get_success(self.store.get_event(self.event_ids[0])) # looking it up should now cause no db hits with LoggingContext(name="test") as ctx: - res = self.get_success(self.store.have_seen_events("room1", ["event10"])) - self.assertEqual(res, {"event10"}) + res = self.get_success( + self.store.have_seen_events("room1", [self.event_ids[0]]) + ) + self.assertEqual(res, {self.event_ids[0]}) self.assertEqual(ctx.get_resource_usage().db_txn_count, 0) @@ -167,7 +176,6 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer): self.store: EventsWorkerStore = hs.get_datastores().main self.room_id = f"!room:{hs.hostname}" - self.event_ids = [f"event{i}" for i in range(20)] self._populate_events() @@ -190,8 +198,14 @@ def _populate_events(self) -> None: ) ) - self.event_ids = [f"event{i}" for i in range(20)] - for idx, event_id in enumerate(self.event_ids): + self.event_ids: List[str] = [] + for idx in range(20): + event_json = { + "type": f"test {idx}", + "room_id": self.room_id, + } + event = make_event_from_dict(event_json, room_version=RoomVersions.V4) + event_id = event.event_id self.get_success( self.store.db_pool.simple_upsert( "events", @@ -201,7 +215,7 @@ def _populate_events(self) -> None: "room_id": self.room_id, "topological_ordering": idx, "stream_ordering": idx, - "type": "test", + "type": event.type, "processed": True, "outlier": False, }, @@ -213,12 +227,13 @@ def _populate_events(self) -> None: {"event_id": event_id}, { "room_id": self.room_id, - "json": json.dumps({"type": "test", "room_id": self.room_id}), + "json": json.dumps(event_json), "internal_metadata": "{}", "format_version": EventFormatVersions.V3, }, ) ) + self.event_ids.append(event_id) @contextmanager def _outage(self) -> Generator[None, None, None]: From 873d467976fef7a58e54b033ee114849d547c793 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 4 May 2022 11:02:19 +0100 Subject: [PATCH 104/263] Fixes to the formatting of README.rst (#12627) Fixes a couple of formatting errors which were introduced in #12475. --- README.rst | 14 +++++++------- changelog.d/12627.doc | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changelog.d/12627.doc diff --git a/README.rst b/README.rst index 80201d4eb155..219e32de8ea4 100644 --- a/README.rst +++ b/README.rst @@ -55,7 +55,7 @@ solutions. The hope is for Matrix to act as the building blocks for a new generation of fully open and interoperable messaging and VoIP apps for the internet. -Synapse is a Matrix "homeserver" implementation developed by the matrix.org core +Synapse is a Matrix "homeserver" implementation developed by the matrix.org core team, written in Python 3/Twisted. In Matrix, every user runs one or more Matrix clients, which connect through to @@ -294,13 +294,13 @@ directory of your choice:: cd synapse Synapse has a number of external dependencies. We maintain a fixed development -environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend +environment using `Poetry `_. First, install poetry. We recommend:: - | pip install --user pipx - | pipx install poetry + pip install --user pipx + pipx install poetry as described `here `_. -(See `poetry's installation docs ` +(See `poetry's installation docs `_ for other installation methods.) Then ask poetry to create a virtual environment from the project and install Synapse's dependencies:: @@ -309,11 +309,11 @@ from the project and install Synapse's dependencies:: This will run a process of downloading and installing all the needed dependencies into a virtual env. -We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082` +We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`:: poetry run ./demo/start.sh -(to stop, you can use `poetry run ./demo/stop.sh`) +(to stop, you can use ``poetry run ./demo/stop.sh``) See the `demo documentation `_ for more information. diff --git a/changelog.d/12627.doc b/changelog.d/12627.doc new file mode 100644 index 000000000000..3a787dfef275 --- /dev/null +++ b/changelog.d/12627.doc @@ -0,0 +1 @@ +Fixes to the formatting of README.rst. From 01e625513afbd9645ff642810d559f96fab13278 Mon Sep 17 00:00:00 2001 From: andrew do Date: Wed, 4 May 2022 04:26:11 -0700 Subject: [PATCH 105/263] remove constantly lib use and switch to enums. (#12624) --- changelog.d/12624.misc | 1 + mypy.ini | 3 -- synapse/handlers/events.py | 2 +- synapse/handlers/federation.py | 4 +- synapse/handlers/federation_event.py | 2 +- synapse/handlers/message.py | 4 +- synapse/state/__init__.py | 2 +- .../storage/databases/main/events_worker.py | 42 +++++++++---------- synapse/storage/databases/main/search.py | 8 ++-- synapse/storage/databases/main/signatures.py | 2 +- synapse/storage/persist_events.py | 4 +- 11 files changed, 36 insertions(+), 38 deletions(-) create mode 100644 changelog.d/12624.misc diff --git a/changelog.d/12624.misc b/changelog.d/12624.misc new file mode 100644 index 000000000000..8772d40fa70f --- /dev/null +++ b/changelog.d/12624.misc @@ -0,0 +1 @@ +Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh. diff --git a/mypy.ini b/mypy.ini index 78699e318704..ba0de419f5ea 100644 --- a/mypy.ini +++ b/mypy.ini @@ -244,9 +244,6 @@ ignore_missing_imports = True [mypy-canonicaljson] ignore_missing_imports = True -[mypy-constantly] -ignore_missing_imports = True - [mypy-ijson.*] ignore_missing_imports = True diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 5b94b00bc3e7..82a5aac3dda6 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -164,7 +164,7 @@ async def get_event( event. """ redact_behaviour = ( - EventRedactBehaviour.AS_IS if show_redacted else EventRedactBehaviour.REDACT + EventRedactBehaviour.as_is if show_redacted else EventRedactBehaviour.redact ) event = await self.store.get_event( event_id, check_room_id=room_id, redact_behaviour=redact_behaviour diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d2ba70a814d0..38dc5b1f6edf 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -316,7 +316,7 @@ async def _maybe_backfill_inner( events_to_check = await self.store.get_events_as_list( event_ids_to_check, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, ) @@ -1494,7 +1494,7 @@ async def _sync_partial_state_room( events = await self.store.get_events_as_list( batch, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, allow_rejected=True, ) for event in events: diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 693b544286ec..6cf927e4ff7b 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -860,7 +860,7 @@ async def _resolve_state_at_missing_prevs( evs = await self._store.get_events( list(state_map.values()), get_prev_content=False, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, ) event_map.update(evs) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1b092e900eb5..95a89ac01f4b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1407,7 +1407,7 @@ async def persist_and_notify_client_event( original_event = await self.store.get_event( event.redacts, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, allow_rejected=False, allow_none=True, @@ -1504,7 +1504,7 @@ async def persist_and_notify_client_event( original_event = await self.store.get_event( event.redacts, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, allow_rejected=False, allow_none=True, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index fbf7ba4600f8..cad3b4264007 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -800,7 +800,7 @@ def get_events( return self.store.get_events( event_ids, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, get_prev_content=False, allow_rejected=allow_rejected, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 0a48e5d29f18..a4a604a49915 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -14,6 +14,7 @@ import logging import threading +from enum import Enum, auto from typing import ( TYPE_CHECKING, Any, @@ -30,7 +31,6 @@ ) import attr -from constantly import NamedConstant, Names from prometheus_client import Gauge from typing_extensions import Literal @@ -150,14 +150,14 @@ class _EventRow: outlier: bool -class EventRedactBehaviour(Names): +class EventRedactBehaviour(Enum): """ What to do when retrieving a redacted event from the database. """ - AS_IS = NamedConstant() - REDACT = NamedConstant() - BLOCK = NamedConstant() + as_is = auto() + redact = auto() + block = auto() class EventsWorkerStore(SQLBaseStore): @@ -327,7 +327,7 @@ async def have_censored_event(self, event_id: str) -> bool: async def get_event( self, event_id: str, - redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.REDACT, + redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.redact, get_prev_content: bool = ..., allow_rejected: bool = ..., allow_none: Literal[False] = ..., @@ -339,7 +339,7 @@ async def get_event( async def get_event( self, event_id: str, - redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.REDACT, + redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.redact, get_prev_content: bool = ..., allow_rejected: bool = ..., allow_none: Literal[True] = ..., @@ -350,7 +350,7 @@ async def get_event( async def get_event( self, event_id: str, - redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.REDACT, + redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.redact, get_prev_content: bool = False, allow_rejected: bool = False, allow_none: bool = False, @@ -362,9 +362,9 @@ async def get_event( event_id: The event_id of the event to fetch redact_behaviour: Determine what to do with a redacted event. Possible values: - * AS_IS - Return the full event body with no redacted content - * REDACT - Return the event but with a redacted body - * DISALLOW - Do not return redacted events (behave as per allow_none + * as_is - Return the full event body with no redacted content + * redact - Return the event but with a redacted body + * block - Do not return redacted events (behave as per allow_none if the event is redacted) get_prev_content: If True and event is a state event, @@ -406,7 +406,7 @@ async def get_event( async def get_events( self, event_ids: Collection[str], - redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.REDACT, + redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.redact, get_prev_content: bool = False, allow_rejected: bool = False, ) -> Dict[str, EventBase]: @@ -417,9 +417,9 @@ async def get_events( redact_behaviour: Determine what to do with a redacted event. Possible values: - * AS_IS - Return the full event body with no redacted content - * REDACT - Return the event but with a redacted body - * DISALLOW - Do not return redacted events (omit them from the response) + * as_is - Return the full event body with no redacted content + * redact - Return the event but with a redacted body + * block - Do not return redacted events (omit them from the response) get_prev_content: If True and event is a state event, include the previous states content in the unsigned field. @@ -442,7 +442,7 @@ async def get_events( async def get_events_as_list( self, event_ids: Collection[str], - redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.REDACT, + redact_behaviour: EventRedactBehaviour = EventRedactBehaviour.redact, get_prev_content: bool = False, allow_rejected: bool = False, ) -> List[EventBase]: @@ -455,9 +455,9 @@ async def get_events_as_list( event_ids: The event_ids of the events to fetch redact_behaviour: Determine what to do with a redacted event. Possible values: - * AS_IS - Return the full event body with no redacted content - * REDACT - Return the event but with a redacted body - * DISALLOW - Do not return redacted events (omit them from the response) + * as_is - Return the full event body with no redacted content + * redact - Return the event but with a redacted body + * block - Do not return redacted events (omit them from the response) get_prev_content: If True and event is a state event, include the previous states content in the unsigned field. @@ -568,10 +568,10 @@ async def get_events_as_list( event = entry.event if entry.redacted_event: - if redact_behaviour == EventRedactBehaviour.BLOCK: + if redact_behaviour == EventRedactBehaviour.block: # Skip this event continue - elif redact_behaviour == EventRedactBehaviour.REDACT: + elif redact_behaviour == EventRedactBehaviour.redact: event = entry.redacted_event events.append(event) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index d4482c06db66..3c49e7ec98e2 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -494,11 +494,11 @@ async def search_msgs( results = list(filter(lambda row: row["room_id"] in room_ids, results)) - # We set redact_behaviour to BLOCK here to prevent redacted events being returned in + # We set redact_behaviour to block here to prevent redacted events being returned in # search results (which is a data leak) events = await self.get_events_as_list( # type: ignore[attr-defined] [r["event_id"] for r in results], - redact_behaviour=EventRedactBehaviour.BLOCK, + redact_behaviour=EventRedactBehaviour.block, ) event_map = {ev.event_id: ev for ev in events} @@ -652,11 +652,11 @@ async def search_rooms( results = list(filter(lambda row: row["room_id"] in room_ids, results)) - # We set redact_behaviour to BLOCK here to prevent redacted events being returned in + # We set redact_behaviour to block here to prevent redacted events being returned in # search results (which is a data leak) events = await self.get_events_as_list( # type: ignore[attr-defined] [r["event_id"] for r in results], - redact_behaviour=EventRedactBehaviour.BLOCK, + redact_behaviour=EventRedactBehaviour.block, ) event_map = {ev.event_id: ev for ev in events} diff --git a/synapse/storage/databases/main/signatures.py b/synapse/storage/databases/main/signatures.py index 95148fd2273a..05da15074a73 100644 --- a/synapse/storage/databases/main/signatures.py +++ b/synapse/storage/databases/main/signatures.py @@ -48,7 +48,7 @@ async def get_event_reference_hashes( """ events = await self.get_events( event_ids, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, allow_rejected=True, ) diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index e496ba7bed6e..97118045a1ad 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -943,7 +943,7 @@ async def _prune_extremities( dropped_events = await self.main_store.get_events( dropped_extrems, allow_rejected=True, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, ) new_senders = {get_domain_from_id(e.sender) for e, _ in events_context} @@ -974,7 +974,7 @@ async def _prune_extremities( prev_events = await self.main_store.get_events( new_events, allow_rejected=True, - redact_behaviour=EventRedactBehaviour.AS_IS, + redact_behaviour=EventRedactBehaviour.as_is, ) events_to_check = prev_events.values() From 75dff3dc980974960f55fa21fc8e672201f63045 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 4 May 2022 08:38:18 -0400 Subject: [PATCH 106/263] Include bundled aggregations for the latest event in a thread. (#12273) The `latest_event` field of the bundled aggregations for `m.thread` relations did not include bundled aggregations itself. This resulted in clients needing to immediately request the event from the server (and thus making it useless that the latest event itself was serialized instead of just including an event ID). --- changelog.d/12273.bugfix | 1 + synapse/events/utils.py | 55 ++++++---- synapse/handlers/relations.py | 74 ++++++++++---- synapse/storage/databases/main/relations.py | 11 +- tests/rest/client/test_relations.py | 108 +++++++++++++++++++- 5 files changed, 198 insertions(+), 51 deletions(-) create mode 100644 changelog.d/12273.bugfix diff --git a/changelog.d/12273.bugfix b/changelog.d/12273.bugfix new file mode 100644 index 000000000000..f8d7b6c88956 --- /dev/null +++ b/changelog.d/12273.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index a6c48308b310..026dcde8d83f 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -426,13 +426,12 @@ def serialize_event( # Check if there are any bundled aggregations to include with the event. if bundle_aggregations: - event_aggregations = bundle_aggregations.get(event.event_id) - if event_aggregations: + if event.event_id in bundle_aggregations: self._inject_bundled_aggregations( event, time_now, config, - event_aggregations, + bundle_aggregations, serialized_event, apply_edits=apply_edits, ) @@ -471,7 +470,7 @@ def _inject_bundled_aggregations( event: EventBase, time_now: int, config: SerializeEventConfig, - aggregations: "BundledAggregations", + bundled_aggregations: Dict[str, "BundledAggregations"], serialized_event: JsonDict, apply_edits: bool, ) -> None: @@ -481,22 +480,37 @@ def _inject_bundled_aggregations( event: The event being serialized. time_now: The current time in milliseconds config: Event serialization config - aggregations: The bundled aggregation to serialize. + bundled_aggregations: Bundled aggregations to be injected. + A map from event_id to aggregation data. Must contain at least an + entry for `event`. + + While serializing the bundled aggregations this map may be searched + again for additional events in a recursive manner. serialized_event: The serialized event which may be modified. apply_edits: Whether the content of the event should be modified to reflect any replacement in `aggregations.replace`. """ + + # We have already checked that aggregations exist for this event. + event_aggregations = bundled_aggregations[event.event_id] + + # The JSON dictionary to be added under the unsigned property of the event + # being serialized. serialized_aggregations = {} - if aggregations.annotations: - serialized_aggregations[RelationTypes.ANNOTATION] = aggregations.annotations + if event_aggregations.annotations: + serialized_aggregations[ + RelationTypes.ANNOTATION + ] = event_aggregations.annotations - if aggregations.references: - serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references + if event_aggregations.references: + serialized_aggregations[ + RelationTypes.REFERENCE + ] = event_aggregations.references - if aggregations.replace: + if event_aggregations.replace: # If there is an edit, optionally apply it to the event. - edit = aggregations.replace + edit = event_aggregations.replace if apply_edits: self._apply_edit(event, serialized_event, edit) @@ -507,19 +521,16 @@ def _inject_bundled_aggregations( "sender": edit.sender, } - # If this event is the start of a thread, include a summary of the replies. - if aggregations.thread: - thread = aggregations.thread + # Include any threaded replies to this event. + if event_aggregations.thread: + thread = event_aggregations.thread - # Don't bundle aggregations as this could recurse forever. - serialized_latest_event = serialize_event( - thread.latest_event, time_now, config=config + serialized_latest_event = self.serialize_event( + thread.latest_event, + time_now, + config=config, + bundle_aggregations=bundled_aggregations, ) - # Manually apply an edit, if one exists. - if thread.latest_edit: - self._apply_edit( - thread.latest_event, serialized_latest_event, thread.latest_edit - ) thread_summary = { "latest_event": serialized_latest_event, diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index b5dc9f74b372..cec5740fbd26 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -44,8 +44,6 @@ class _ThreadAggregation: # The latest event in the thread. latest_event: EventBase - # The latest edit to the latest event in the thread. - latest_edit: Optional[EventBase] # The total number of events in the thread. count: int # True if the current user has sent an event to the thread. @@ -295,7 +293,7 @@ async def get_threads_for_events( for event_id, summary in summaries.items(): if summary: - thread_count, latest_thread_event, edit = summary + thread_count, latest_thread_event = summary # Subtract off the count of any ignored users. for ignored_user in ignored_users: @@ -340,7 +338,6 @@ async def get_threads_for_events( results[event_id] = _ThreadAggregation( latest_event=latest_thread_event, - latest_edit=edit, count=thread_count, # If there's a thread summary it must also exist in the # participated dictionary. @@ -359,8 +356,13 @@ async def get_bundled_aggregations( user_id: The user requesting the bundled aggregations. Returns: - A map of event ID to the bundled aggregation for the event. Not all - events may have bundled aggregations in the results. + A map of event ID to the bundled aggregations for the event. + + Not all requested events may exist in the results (if they don't have + bundled aggregations). + + The results may include additional events which are related to the + requested events. """ # De-duplicate events by ID to handle the same event requested multiple times. # @@ -369,22 +371,59 @@ async def get_bundled_aggregations( event.event_id: event for event in events if not event.is_state() } + # A map of event ID to the relation in that event, if there is one. + relations_by_id: Dict[str, str] = {} + for event_id, event in events_by_id.items(): + relates_to = event.content.get("m.relates_to") + if isinstance(relates_to, collections.abc.Mapping): + relation_type = relates_to.get("rel_type") + if isinstance(relation_type, str): + relations_by_id[event_id] = relation_type + # event ID -> bundled aggregation in non-serialized form. results: Dict[str, BundledAggregations] = {} # Fetch any ignored users of the requesting user. ignored_users = await self._main_store.ignored_users(user_id) + # Threads are special as the latest event of a thread might cause additional + # events to be fetched. Thus, we check those first! + + # Fetch thread summaries (but only for the directly requested events). + threads = await self.get_threads_for_events( + # It is not valid to start a thread on an event which itself relates to another event. + [eid for eid in events_by_id.keys() if eid not in relations_by_id], + user_id, + ignored_users, + ) + for event_id, thread in threads.items(): + results.setdefault(event_id, BundledAggregations()).thread = thread + + # If the latest event in a thread is not already being fetched, + # add it. This ensures that the bundled aggregations for the + # latest thread event is correct. + latest_thread_event = thread.latest_event + if latest_thread_event and latest_thread_event.event_id not in events_by_id: + events_by_id[latest_thread_event.event_id] = latest_thread_event + # Keep relations_by_id in sync with events_by_id: + # + # We know that the latest event in a thread has a thread relation + # (as that is what makes it part of the thread). + relations_by_id[latest_thread_event.event_id] = RelationTypes.THREAD + # Fetch other relations per event. for event in events_by_id.values(): - # Do not bundle aggregations for an event which represents an edit or an - # annotation. It does not make sense for them to have related events. - relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, collections.abc.Mapping): - relation_type = relates_to.get("rel_type") - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): - continue - + # An event which is a replacement (ie edit) or annotation (ie, reaction) + # may not have any other event related to it. + # + # XXX This is buggy, see https://github.com/matrix-org/synapse/issues/12566 + if relations_by_id.get(event.event_id) in ( + RelationTypes.ANNOTATION, + RelationTypes.REPLACE, + ): + continue + + # Fetch any annotations (ie, reactions) to bundle with this event. annotations = await self.get_annotations_for_event( event.event_id, event.room_id, ignored_users=ignored_users ) @@ -393,6 +432,7 @@ async def get_bundled_aggregations( event.event_id, BundledAggregations() ).annotations = {"chunk": annotations} + # Fetch any references to bundle with this event. references, next_token = await self.get_relations_for_event( event.event_id, event, @@ -425,10 +465,4 @@ async def get_bundled_aggregations( for event_id, edit in edits.items(): results.setdefault(event_id, BundledAggregations()).replace = edit - threads = await self.get_threads_for_events( - events_by_id.keys(), user_id, ignored_users - ) - for event_id, thread in threads.items(): - results.setdefault(event_id, BundledAggregations()).thread = thread - return results diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index a5c31f6787d9..484976ca6b0b 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -445,8 +445,8 @@ def get_thread_summary(self, event_id: str) -> Optional[Tuple[int, EventBase]]: @cachedList(cached_method_name="get_thread_summary", list_name="event_ids") async def get_thread_summaries( self, event_ids: Collection[str] - ) -> Dict[str, Optional[Tuple[int, EventBase, Optional[EventBase]]]]: - """Get the number of threaded replies, the latest reply (if any), and the latest edit for that reply for the given event. + ) -> Dict[str, Optional[Tuple[int, EventBase]]]: + """Get the number of threaded replies and the latest reply (if any) for the given events. Args: event_ids: Summarize the thread related to this event ID. @@ -458,7 +458,6 @@ async def get_thread_summaries( Each summary is a tuple of: The number of events in the thread. The most recent event in the thread. - The most recent edit to the most recent event in the thread, if applicable. """ def _get_thread_summaries_txn( @@ -544,9 +543,6 @@ def _get_thread_summaries_txn( latest_events = await self.get_events(latest_event_ids.values()) # type: ignore[attr-defined] - # Check to see if any of those events are edited. - latest_edits = await self.get_applicable_edits(latest_event_ids.values()) - # Map to the event IDs to the thread summary. # # There might not be a summary due to there not being a thread or @@ -557,8 +553,7 @@ def _get_thread_summaries_txn( summary = None if latest_event: - latest_edit = latest_edits.get(latest_event_id) - summary = (counts[parent_event_id], latest_event, latest_edit) + summary = (counts[parent_event_id], latest_event) summaries[parent_event_id] = summary return summaries diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 39667e3225e2..33ce9471b3a8 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -1029,7 +1029,106 @@ def assert_thread(bundled_aggregations: JsonDict) -> None: bundled_aggregations.get("latest_event"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 10) + + def test_thread_with_bundled_aggregations_for_latest(self) -> None: + """ + Bundled aggregations should get applied to the latest thread event. + """ + self._send_relation(RelationTypes.THREAD, "m.room.test") + channel = self._send_relation(RelationTypes.THREAD, "m.room.test") + thread_2 = channel.json_body["event_id"] + + self._send_relation( + RelationTypes.ANNOTATION, "m.reaction", "a", parent_id=thread_2 + ) + + def assert_thread(bundled_aggregations: JsonDict) -> None: + self.assertEqual(2, bundled_aggregations.get("count")) + self.assertTrue(bundled_aggregations.get("current_user_participated")) + # The latest thread event has some fields that don't matter. + self.assert_dict( + { + "content": { + "m.relates_to": { + "event_id": self.parent_id, + "rel_type": RelationTypes.THREAD, + } + }, + "event_id": thread_2, + "sender": self.user_id, + "type": "m.room.test", + }, + bundled_aggregations.get("latest_event"), + ) + # Check the unsigned field on the latest event. + self.assert_dict( + { + "m.relations": { + RelationTypes.ANNOTATION: { + "chunk": [ + {"type": "m.reaction", "key": "a", "count": 1}, + ] + }, + } + }, + bundled_aggregations["latest_event"].get("unsigned"), + ) + + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 10) + + def test_nested_thread(self) -> None: + """ + Ensure that a nested thread gets ignored by bundled aggregations, as + those are forbidden. + """ + + # Start a thread. + channel = self._send_relation(RelationTypes.THREAD, "m.room.test") + reply_event_id = channel.json_body["event_id"] + + # Disable the validation to pretend this came over federation, since it is + # not an event the Client-Server API will allow.. + with patch( + "synapse.handlers.message.EventCreationHandler._validate_event_relation", + new=lambda self, event: make_awaitable(None), + ): + # Create a sub-thread off the thread, which is not allowed. + self._send_relation( + RelationTypes.THREAD, "m.room.test", parent_id=reply_event_id + ) + + # Fetch the thread root, to get the bundled aggregation for the thread. + relations_from_event = self._get_bundled_aggregations() + + # Ensure that requesting the room messages also does not return the sub-thread. + channel = self.make_request( + "GET", + f"/rooms/{self.room}/messages?dir=b", + access_token=self.user_token, + ) + self.assertEqual(200, channel.code, channel.json_body) + event = self._find_event_in_chunk(channel.json_body["chunk"]) + relations_from_messages = event["unsigned"]["m.relations"] + + # Check the bundled aggregations from each point. + for aggregations, desc in ( + (relations_from_event, "/event"), + (relations_from_messages, "/messages"), + ): + # The latest event should have bundled aggregations. + self.assertIn(RelationTypes.THREAD, aggregations, desc) + thread_summary = aggregations[RelationTypes.THREAD] + self.assertIn("latest_event", thread_summary, desc) + self.assertEqual( + thread_summary["latest_event"]["event_id"], reply_event_id, desc + ) + + # The latest event should not have any bundled aggregations (since the + # only relation to it is another thread, which is invalid). + self.assertNotIn( + "m.relations", thread_summary["latest_event"]["unsigned"], desc + ) def test_thread_edit_latest_event(self) -> None: """Test that editing the latest event in a thread works.""" @@ -1049,6 +1148,7 @@ def test_thread_edit_latest_event(self) -> None: content={"msgtype": "m.text", "body": "foo", "m.new_content": new_body}, parent_id=threaded_event_id, ) + edit_event_id = channel.json_body["event_id"] # Fetch the thread root, to get the bundled aggregation for the thread. relations_dict = self._get_bundled_aggregations() @@ -1061,6 +1161,12 @@ def test_thread_edit_latest_event(self) -> None: self.assertIn("latest_event", thread_summary) latest_event_in_thread = thread_summary["latest_event"] self.assertEqual(latest_event_in_thread["content"]["body"], "I've been edited!") + # The latest event in the thread should have the edit appear under the + # bundled aggregations. + self.assertDictContainsSubset( + {"event_id": edit_event_id, "sender": "@alice:test"}, + latest_event_in_thread["unsigned"]["m.relations"][RelationTypes.REPLACE], + ) def test_aggregation_get_event_for_annotation(self) -> None: """Test that annotations do not get bundled aggregations included From b2df0716bc0cf31b5f5f90a0599bc1d04a837e27 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 4 May 2022 13:38:55 +0100 Subject: [PATCH 107/263] Improve logging for cancelled requests (#12587) Don't log stack traces for cancelled requests and use a custom HTTP status code of 499. Signed-off-by: Sean Quah --- changelog.d/12587.misc | 1 + docs/usage/administration/request_log.md | 2 +- synapse/http/server.py | 30 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12587.misc diff --git a/changelog.d/12587.misc b/changelog.d/12587.misc new file mode 100644 index 000000000000..d26e332305ce --- /dev/null +++ b/changelog.d/12587.misc @@ -0,0 +1 @@ +Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. diff --git a/docs/usage/administration/request_log.md b/docs/usage/administration/request_log.md index 316304c7348a..adb5f4f5f353 100644 --- a/docs/usage/administration/request_log.md +++ b/docs/usage/administration/request_log.md @@ -28,7 +28,7 @@ See the following for how to decode the dense data available from the default lo | NNNN | Total time waiting for response to DB queries across all parallel DB work from this request | | OOOO | Count of DB transactions performed | | PPPP | Response body size | -| QQQQ | Response status code (prefixed with ! if the socket was closed before the response was generated) | +| QQQQ | Response status code
Suffixed with `!` if the socket was closed before the response was generated.
A `499!` status code indicates that Synapse also cancelled request processing after the socket was closed.
| | RRRR | Request | | SSSS | User-agent | | TTTT | Events fetched from DB to service this request (note that this does not include events fetched from the cache) | diff --git a/synapse/http/server.py b/synapse/http/server.py index 1cf49830e89b..657bffcddd88 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -43,6 +43,7 @@ from zope.interface import implementer from twisted.internet import defer, interfaces +from twisted.internet.defer import CancelledError from twisted.python import failure from twisted.web import resource from twisted.web.server import NOT_DONE_YET, Request @@ -82,6 +83,14 @@ """ +# A fictional HTTP status code for requests where the client has disconnected and we +# successfully cancelled the request. Used only for logging purposes. Clients will never +# observe this code unless cancellations leak across requests or we raise a +# `CancelledError` ourselves. +# Analogous to nginx's 499 status code: +# https://github.com/nginx/nginx/blob/release-1.21.6/src/http/ngx_http_request.h#L128-L134 +HTTP_STATUS_REQUEST_CANCELLED = 499 + def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: """Sends a JSON error response to clients.""" @@ -93,6 +102,17 @@ def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: error_dict = exc.error_dict() logger.info("%s SynapseError: %s - %s", request, error_code, exc.msg) + elif f.check(CancelledError): + error_code = HTTP_STATUS_REQUEST_CANCELLED + error_dict = {"error": "Request cancelled", "errcode": Codes.UNKNOWN} + + if not request._disconnected: + logger.error( + "Got cancellation before client disconnection from %r: %r", + request.request_metrics.name, + request, + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] + ) else: error_code = 500 error_dict = {"error": "Internal server error", "errcode": Codes.UNKNOWN} @@ -155,6 +175,16 @@ def return_html_error( request, exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] ) + elif f.check(CancelledError): + code = HTTP_STATUS_REQUEST_CANCELLED + msg = "Request cancelled" + + if not request._disconnected: + logger.error( + "Got cancellation before client disconnection when handling request %r", + request, + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore[arg-type] + ) else: code = HTTPStatus.INTERNAL_SERVER_ERROR msg = "Internal server error" From ba3fd54badb0e11d4dc79488e2ab3099fc03a1e8 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 4 May 2022 09:53:21 -0400 Subject: [PATCH 108/263] Remove unstable/unspecced login types. (#12597) * `m.login.jwt`, which was never specced and has been deprecated since Synapse 1.16.0. (`org.matrix.login.jwt` can be used instead.) * `uk.half-shot.msc2778.login.application_service`, which was stabilized as part of the Matrix spec v1.2 release. --- CHANGES.md | 6 ++++++ changelog.d/12597.removal | 2 ++ docs/jwt.md | 3 --- synapse/rest/client/login.py | 15 ++++----------- tests/handlers/test_password_providers.py | 4 +--- tests/rest/client/test_login.py | 4 +--- 6 files changed, 14 insertions(+), 20 deletions(-) create mode 100644 changelog.d/12597.removal diff --git a/CHANGES.md b/CHANGES.md index 31f156127424..b4d91b27937f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 1.59.0 +============== + +The non-standard `m.login.jwt` login type has been removed from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. + + Synapse 1.58.0 (2022-05-03) =========================== diff --git a/changelog.d/12597.removal b/changelog.d/12597.removal new file mode 100644 index 000000000000..7927f1d68d5f --- /dev/null +++ b/changelog.d/12597.removal @@ -0,0 +1,2 @@ +Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from +[MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). diff --git a/docs/jwt.md b/docs/jwt.md index 32f58cc0cbbf..346daf78ad1e 100644 --- a/docs/jwt.md +++ b/docs/jwt.md @@ -17,9 +17,6 @@ follows: } ``` -Note that the login type of `m.login.jwt` is supported, but is deprecated. This -will be removed in a future version of Synapse. - The `token` field should include the JSON web token with the following claims: * A claim that encodes the local part of the user ID is required. By default, diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 4a4dbe75de6b..71d8038448da 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -69,9 +69,7 @@ class LoginRestServlet(RestServlet): SSO_TYPE = "m.login.sso" TOKEN_TYPE = "m.login.token" JWT_TYPE = "org.matrix.login.jwt" - JWT_TYPE_DEPRECATED = "m.login.jwt" APPSERVICE_TYPE = "m.login.application_service" - APPSERVICE_TYPE_UNSTABLE = "uk.half-shot.msc2778.login.application_service" REFRESH_TOKEN_PARAM = "refresh_token" def __init__(self, hs: "HomeServer"): @@ -126,7 +124,6 @@ def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: flows: List[JsonDict] = [] if self.jwt_enabled: flows.append({"type": LoginRestServlet.JWT_TYPE}) - flows.append({"type": LoginRestServlet.JWT_TYPE_DEPRECATED}) if self.cas_enabled: # we advertise CAS for backwards compat, though MSC1721 renamed it @@ -156,7 +153,6 @@ def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: flows.extend({"type": t} for t in self.auth_handler.get_supported_login_types()) flows.append({"type": LoginRestServlet.APPSERVICE_TYPE}) - flows.append({"type": LoginRestServlet.APPSERVICE_TYPE_UNSTABLE}) return 200, {"flows": flows} @@ -175,10 +171,7 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, LoginResponse]: ) try: - if login_submission["type"] in ( - LoginRestServlet.APPSERVICE_TYPE, - LoginRestServlet.APPSERVICE_TYPE_UNSTABLE, - ): + if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE: appservice = self.auth.get_appservice_by_req(request) if appservice.is_rate_limited(): @@ -191,9 +184,9 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, LoginResponse]: appservice, should_issue_refresh_token=should_issue_refresh_token, ) - elif self.jwt_enabled and ( - login_submission["type"] == LoginRestServlet.JWT_TYPE - or login_submission["type"] == LoginRestServlet.JWT_TYPE_DEPRECATED + elif ( + self.jwt_enabled + and login_submission["type"] == LoginRestServlet.JWT_TYPE ): await self._address_ratelimiter.ratelimit(None, request.getClientIP()) result = await self._do_jwt_login( diff --git a/tests/handlers/test_password_providers.py b/tests/handlers/test_password_providers.py index addf14fa2ba0..82b3bb3b735d 100644 --- a/tests/handlers/test_password_providers.py +++ b/tests/handlers/test_password_providers.py @@ -30,11 +30,9 @@ from tests.test_utils import make_awaitable from tests.unittest import override_config -# (possibly experimental) login flows we expect to appear in the list after the normal -# ones +# Login flows we expect to appear in the list after the normal ones. ADDITIONAL_LOGIN_FLOWS = [ {"type": "m.login.application_service"}, - {"type": "uk.half-shot.msc2778.login.application_service"}, ] # a mock instance which the dummy auth providers delegate to, so we can see what's going diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index 0a3d017dc9b9..4920468f7ab8 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -81,11 +81,9 @@ # the query params in TEST_CLIENT_REDIRECT_URL EXPECTED_CLIENT_REDIRECT_URL_PARAMS = [("", ""), ('q" =+"', '"fö&=o"')] -# (possibly experimental) login flows we expect to appear in the list after the normal -# ones +# Login flows we expect to appear in the list after the normal ones. ADDITIONAL_LOGIN_FLOWS = [ {"type": "m.login.application_service"}, - {"type": "uk.half-shot.msc2778.login.application_service"}, ] From 332cce8dcf9c28314f568c290b57e98036a0e723 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 4 May 2022 16:41:40 +0100 Subject: [PATCH 109/263] Disable device name lookup over federation by default (#12616) --- changelog.d/12616.misc | 1 + docs/sample_config.yaml | 8 ++++---- docs/upgrade.md | 11 +++++++++++ docs/usage/configuration/config_documentation.md | 6 +++--- synapse/config/federation.py | 10 +++++----- 5 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 changelog.d/12616.misc diff --git a/changelog.d/12616.misc b/changelog.d/12616.misc new file mode 100644 index 000000000000..d17ce24cdf29 --- /dev/null +++ b/changelog.d/12616.misc @@ -0,0 +1 @@ +Prevent remote homeservers from requesting local user device names by default. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 67184c6b1ae1..5eba0fcf3d1d 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -709,11 +709,11 @@ retention: # #allow_profile_lookup_over_federation: false -# Uncomment to disable device display name lookup over federation. By default, the -# Federation API allows other homeservers to obtain device display names of any user -# on this homeserver. Defaults to 'true'. +# Uncomment to allow device display name lookup over federation. By default, the +# Federation API prevents other homeservers from obtaining the display names of +# user devices on this homeserver. Defaults to 'false'. # -#allow_device_name_lookup_over_federation: false +#allow_device_name_lookup_over_federation: true ## Caching ## diff --git a/docs/upgrade.md b/docs/upgrade.md index 3a8aeb039533..b40cac86f0c4 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -89,6 +89,17 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.59.0 + +## Device name lookup over federation has been disabled by default + +The names of user devices are no longer visible to users on other homeservers by default. +Device IDs are unaffected, as these are necessary to facilitate end-to-end encryption. + +To re-enable this functionality, set the +[`allow_device_name_lookup_over_federation`](https://matrix-org.github.io/synapse/v1.59/usage/configuration/config_documentation.html#federation) +homeserver config option to `true`. + # Upgrading to v1.58.0 ## Groups/communities feature has been disabled by default diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 968b0fbfaff4..36db649467ca 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1035,13 +1035,13 @@ allow_profile_lookup_over_federation: false --- Config option: `allow_device_name_lookup_over_federation` -Set this option to false to disable device display name lookup over federation. By default, the -Federation API allows other homeservers to obtain device display names of any user +Set this option to true to allow device display name lookup over federation. By default, the +Federation API prevents other homeservers from obtaining the display names of any user devices on this homeserver. Example configuration: ```yaml -allow_device_name_lookup_over_federation: false +allow_device_name_lookup_over_federation: true ``` --- ## Caching ## diff --git a/synapse/config/federation.py b/synapse/config/federation.py index 0e74f7078455..f83f93c0ef11 100644 --- a/synapse/config/federation.py +++ b/synapse/config/federation.py @@ -46,7 +46,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) self.allow_device_name_lookup_over_federation = config.get( - "allow_device_name_lookup_over_federation", True + "allow_device_name_lookup_over_federation", False ) def generate_config_section(self, **kwargs: Any) -> str: @@ -81,11 +81,11 @@ def generate_config_section(self, **kwargs: Any) -> str: # #allow_profile_lookup_over_federation: false - # Uncomment to disable device display name lookup over federation. By default, the - # Federation API allows other homeservers to obtain device display names of any user - # on this homeserver. Defaults to 'true'. + # Uncomment to allow device display name lookup over federation. By default, the + # Federation API prevents other homeservers from obtaining the display names of + # user devices on this homeserver. Defaults to 'false'. # - #allow_device_name_lookup_over_federation: false + #allow_device_name_lookup_over_federation: true """ From 116a4c8340b729ffde43be33df24d417384cb28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 4 May 2022 17:59:22 +0200 Subject: [PATCH 110/263] Implement changes to MSC2285 (hidden read receipts) (#12168) * Changes hidden read receipts to be a separate receipt type (instead of a field on `m.read`). * Updates the `/receipts` endpoint to accept `m.fully_read`. --- changelog.d/12168.feature | 1 + synapse/api/constants.py | 6 +- synapse/handlers/receipts.py | 65 ++--- synapse/handlers/sync.py | 2 +- synapse/push/push_tools.py | 4 +- synapse/rest/client/notifications.py | 2 +- synapse/rest/client/read_marker.py | 32 ++- synapse/rest/client/receipts.py | 51 ++-- synapse/storage/databases/main/receipts.py | 142 ++++++++--- tests/handlers/test_receipts.py | 129 ++++++---- .../slave/storage/test_receipts.py | 238 +++++++++++++++++- tests/rest/client/test_sync.py | 161 ++++++++++-- 12 files changed, 647 insertions(+), 186 deletions(-) create mode 100644 changelog.d/12168.feature diff --git a/changelog.d/12168.feature b/changelog.d/12168.feature new file mode 100644 index 000000000000..cd5c45029ee1 --- /dev/null +++ b/changelog.d/12168.feature @@ -0,0 +1 @@ +Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0172eb60b8dc..0ccd4c95581e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -255,7 +255,5 @@ class GuestAccess: class ReceiptTypes: READ: Final = "m.read" - - -class ReadReceiptEventFields: - MSC2285_HIDDEN: Final = "org.matrix.msc2285.hidden" + READ_PRIVATE: Final = "org.matrix.msc2285.read.private" + FULLY_READ: Final = "m.fully_read" diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index cfe860decc95..ae41fd674e13 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -14,7 +14,7 @@ import logging from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes +from synapse.api.constants import ReceiptTypes from synapse.appservice import ApplicationService from synapse.streams import EventSource from synapse.types import JsonDict, ReadReceipt, UserID, get_domain_from_id @@ -112,7 +112,7 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool: ) if not res: - # res will be None if this read receipt is 'old' + # res will be None if this receipt is 'old' continue stream_id, max_persisted_id = res @@ -138,7 +138,7 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool: return True async def received_client_receipt( - self, room_id: str, receipt_type: str, user_id: str, event_id: str, hidden: bool + self, room_id: str, receipt_type: str, user_id: str, event_id: str ) -> None: """Called when a client tells us a local user has read up to the given event_id in the room. @@ -148,16 +148,14 @@ async def received_client_receipt( receipt_type=receipt_type, user_id=user_id, event_ids=[event_id], - data={"ts": int(self.clock.time_msec()), "hidden": hidden}, + data={"ts": int(self.clock.time_msec())}, ) is_new = await self._handle_new_receipts([receipt]) if not is_new: return - if self.federation_sender and not ( - self.hs.config.experimental.msc2285_enabled and hidden - ): + if self.federation_sender and receipt_type != ReceiptTypes.READ_PRIVATE: await self.federation_sender.send_read_receipt(receipt) @@ -168,6 +166,13 @@ def __init__(self, hs: "HomeServer"): @staticmethod def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]: + """ + This method takes in what is returned by + get_linearized_receipts_for_rooms() and goes through read receipts + filtering out m.read.private receipts if they were not sent by the + current user. + """ + visible_events = [] # filter out hidden receipts the user shouldn't see @@ -176,37 +181,21 @@ def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]: new_event = event.copy() new_event["content"] = {} - for event_id in content.keys(): - event_content = content.get(event_id, {}) - m_read = event_content.get(ReceiptTypes.READ, {}) - - # If m_read is missing copy over the original event_content as there is nothing to process here - if not m_read: - new_event["content"][event_id] = event_content.copy() - continue - - new_users = {} - for rr_user_id, user_rr in m_read.items(): - try: - hidden = user_rr.get("hidden") - except AttributeError: - # Due to https://github.com/matrix-org/synapse/issues/10376 - # there are cases where user_rr is a string, in those cases - # we just ignore the read receipt - continue - - if hidden is not True or rr_user_id == user_id: - new_users[rr_user_id] = user_rr.copy() - # If hidden has a value replace hidden with the correct prefixed key - if hidden is not None: - new_users[rr_user_id].pop("hidden") - new_users[rr_user_id][ - ReadReceiptEventFields.MSC2285_HIDDEN - ] = hidden - - # Set new users unless empty - if len(new_users.keys()) > 0: - new_event["content"][event_id] = {ReceiptTypes.READ: new_users} + for event_id, event_content in content.items(): + receipt_event = {} + for receipt_type, receipt_content in event_content.items(): + if receipt_type == ReceiptTypes.READ_PRIVATE: + user_rr = receipt_content.get(user_id, None) + if user_rr: + receipt_event[ReceiptTypes.READ_PRIVATE] = { + user_id: user_rr.copy() + } + else: + receipt_event[receipt_type] = receipt_content.copy() + + # Only include the receipt event if it is non-empty. + if receipt_event: + new_event["content"][event_id] = receipt_event # Append new_event to visible_events unless empty if len(new_event["content"].keys()) > 0: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5125126a807c..2c555a66d066 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1045,7 +1045,7 @@ async def unread_notifs_for_room_id( last_unread_event_id = await self.store.get_last_receipt_event_id_for_user( user_id=sync_config.user.to_string(), room_id=room_id, - receipt_type=ReceiptTypes.READ, + receipt_types=(ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE), ) return await self.store.get_unread_event_push_actions_by_room_for_user( diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index 957c9b780b94..a1bf5b20dd42 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -24,7 +24,9 @@ async def get_badge_count(store: DataStore, user_id: str, group_by_room: bool) - invites = await store.get_invited_rooms_for_local_user(user_id) joins = await store.get_rooms_for_user(user_id) - my_receipts_by_room = await store.get_receipts_for_user(user_id, ReceiptTypes.READ) + my_receipts_by_room = await store.get_receipts_for_user( + user_id, (ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE) + ) badge = len(invites) diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index ff040de6b840..24bc7c90957f 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -58,7 +58,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: ) receipts_by_room = await self.store.get_receipts_for_user_with_orderings( - user_id, ReceiptTypes.READ + user_id, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] ) notif_event_ids = [pa.event_id for pa in push_actions] diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index f51be511d1f4..1583e903cd88 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -15,8 +15,8 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes -from synapse.api.errors import Codes, SynapseError +from synapse.api.constants import ReceiptTypes +from synapse.api.errors import SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -36,6 +36,7 @@ class ReadMarkerRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() + self.config = hs.config self.receipts_handler = hs.get_receipts_handler() self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() @@ -48,27 +49,38 @@ async def on_POST( await self.presence_handler.bump_presence_active_time(requester.user) body = parse_json_object_from_request(request) - read_event_id = body.get(ReceiptTypes.READ, None) - hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False) - if not isinstance(hidden, bool): + valid_receipt_types = {ReceiptTypes.READ, ReceiptTypes.FULLY_READ} + if self.config.experimental.msc2285_enabled: + valid_receipt_types.add(ReceiptTypes.READ_PRIVATE) + + if set(body.keys()) > valid_receipt_types: raise SynapseError( 400, - "Param %s must be a boolean, if given" - % ReadReceiptEventFields.MSC2285_HIDDEN, - Codes.BAD_JSON, + "Receipt type must be 'm.read', 'org.matrix.msc2285.read.private' or 'm.fully_read'" + if self.config.experimental.msc2285_enabled + else "Receipt type must be 'm.read' or 'm.fully_read'", ) + read_event_id = body.get(ReceiptTypes.READ, None) if read_event_id: await self.receipts_handler.received_client_receipt( room_id, ReceiptTypes.READ, user_id=requester.user.to_string(), event_id=read_event_id, - hidden=hidden, ) - read_marker_event_id = body.get("m.fully_read", None) + read_private_event_id = body.get(ReceiptTypes.READ_PRIVATE, None) + if read_private_event_id and self.config.experimental.msc2285_enabled: + await self.receipts_handler.received_client_receipt( + room_id, + ReceiptTypes.READ_PRIVATE, + user_id=requester.user.to_string(), + event_id=read_private_event_id, + ) + + read_marker_event_id = body.get(ReceiptTypes.FULLY_READ, None) if read_marker_event_id: await self.read_marker_handler.received_client_read_marker( room_id, diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index b24ad2d1be13..f9caab663523 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -16,8 +16,8 @@ import re from typing import TYPE_CHECKING, Tuple -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes -from synapse.api.errors import Codes, SynapseError +from synapse.api.constants import ReceiptTypes +from synapse.api.errors import SynapseError from synapse.http import get_request_user_agent from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request @@ -46,6 +46,7 @@ def __init__(self, hs: "HomeServer"): self.hs = hs self.auth = hs.get_auth() self.receipts_handler = hs.get_receipts_handler() + self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() async def on_POST( @@ -53,7 +54,19 @@ async def on_POST( ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - if receipt_type != ReceiptTypes.READ: + if self.hs.config.experimental.msc2285_enabled and receipt_type not in [ + ReceiptTypes.READ, + ReceiptTypes.READ_PRIVATE, + ReceiptTypes.FULLY_READ, + ]: + raise SynapseError( + 400, + "Receipt type must be 'm.read', 'org.matrix.msc2285.read.private' or 'm.fully_read'", + ) + elif ( + not self.hs.config.experimental.msc2285_enabled + and receipt_type != ReceiptTypes.READ + ): raise SynapseError(400, "Receipt type must be 'm.read'") # Do not allow older SchildiChat and Element Android clients (prior to Element/1.[012].x) to send an empty body. @@ -62,26 +75,24 @@ async def on_POST( if "Android" in user_agent: if pattern.match(user_agent) or "Riot" in user_agent: allow_empty_body = True - body = parse_json_object_from_request(request, allow_empty_body) - hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False) - - if not isinstance(hidden, bool): - raise SynapseError( - 400, - "Param %s must be a boolean, if given" - % ReadReceiptEventFields.MSC2285_HIDDEN, - Codes.BAD_JSON, - ) + # This call makes sure possible empty body is handled correctly + parse_json_object_from_request(request, allow_empty_body) await self.presence_handler.bump_presence_active_time(requester.user) - await self.receipts_handler.received_client_receipt( - room_id, - receipt_type, - user_id=requester.user.to_string(), - event_id=event_id, - hidden=hidden, - ) + if receipt_type == ReceiptTypes.FULLY_READ: + await self.read_marker_handler.received_client_read_marker( + room_id, + user_id=requester.user.to_string(), + event_id=event_id, + ) + else: + await self.receipts_handler.received_client_receipt( + room_id, + receipt_type, + user_id=requester.user.to_string(), + event_id=event_id, + ) return 200, {} diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 7d96f4fedabf..9e3d838eab9a 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -144,43 +144,77 @@ async def get_receipts_for_room( desc="get_receipts_for_room", ) - @cached() async def get_last_receipt_event_id_for_user( - self, user_id: str, room_id: str, receipt_type: str + self, user_id: str, room_id: str, receipt_types: Iterable[str] ) -> Optional[str]: """ - Fetch the event ID for the latest receipt in a room with the given receipt type. + Fetch the event ID for the latest receipt in a room with one of the given receipt types. Args: user_id: The user to fetch receipts for. room_id: The room ID to fetch the receipt for. - receipt_type: The receipt type to fetch. + receipt_type: The receipt types to fetch. Earlier receipt types + are given priority if multiple receipts point to the same event. Returns: - The event ID of the latest receipt, if one exists; otherwise `None`. + The latest receipt, if one exists. """ - return await self.db_pool.simple_select_one_onecol( - table="receipts_linearized", - keyvalues={ - "room_id": room_id, - "receipt_type": receipt_type, - "user_id": user_id, - }, - retcol="event_id", - desc="get_own_receipt_for_user", - allow_none=True, - ) + latest_event_id: Optional[str] = None + latest_stream_ordering = 0 + for receipt_type in receipt_types: + result = await self._get_last_receipt_event_id_for_user( + user_id, room_id, receipt_type + ) + if result is None: + continue + event_id, stream_ordering = result + + if latest_event_id is None or latest_stream_ordering < stream_ordering: + latest_event_id = event_id + latest_stream_ordering = stream_ordering + + return latest_event_id @cached() + async def _get_last_receipt_event_id_for_user( + self, user_id: str, room_id: str, receipt_type: str + ) -> Optional[Tuple[str, int]]: + """ + Fetch the event ID and stream ordering for the latest receipt. + + Args: + user_id: The user to fetch receipts for. + room_id: The room ID to fetch the receipt for. + receipt_type: The receipt type to fetch. + + Returns: + The event ID and stream ordering of the latest receipt, if one exists; + otherwise `None`. + """ + sql = """ + SELECT event_id, stream_ordering + FROM receipts_linearized + INNER JOIN events USING (room_id, event_id) + WHERE user_id = ? + AND room_id = ? + AND receipt_type = ? + """ + + def f(txn: LoggingTransaction) -> Optional[Tuple[str, int]]: + txn.execute(sql, (user_id, room_id, receipt_type)) + return cast(Optional[Tuple[str, int]], txn.fetchone()) + + return await self.db_pool.runInteraction("get_own_receipt_for_user", f) + async def get_receipts_for_user( - self, user_id: str, receipt_type: str + self, user_id: str, receipt_types: Iterable[str] ) -> Dict[str, str]: """ Fetch the event IDs for the latest receipts sent by the given user. Args: user_id: The user to fetch receipts for. - receipt_type: The receipt type to fetch. + receipt_types: The receipt types to check. Returns: A map of room ID to the event ID of the latest receipt for that room. @@ -188,16 +222,48 @@ async def get_receipts_for_user( If the user has not sent a receipt to a room then it will not appear in the returned dictionary. """ - rows = await self.db_pool.simple_select_list( - table="receipts_linearized", - keyvalues={"user_id": user_id, "receipt_type": receipt_type}, - retcols=("room_id", "event_id"), - desc="get_receipts_for_user", + results = await self.get_receipts_for_user_with_orderings( + user_id, receipt_types ) - return {row["room_id"]: row["event_id"] for row in rows} + # Reduce the result to room ID -> event ID. + return { + room_id: room_result["event_id"] for room_id, room_result in results.items() + } async def get_receipts_for_user_with_orderings( + self, user_id: str, receipt_types: Iterable[str] + ) -> JsonDict: + """ + Fetch receipts for all rooms that the given user is joined to. + + Args: + user_id: The user to fetch receipts for. + receipt_types: The receipt types to fetch. Earlier receipt types + are given priority if multiple receipts point to the same event. + + Returns: + A map of room ID to the latest receipt (for the given types). + """ + results: JsonDict = {} + for receipt_type in receipt_types: + partial_result = await self._get_receipts_for_user_with_orderings( + user_id, receipt_type + ) + for room_id, room_result in partial_result.items(): + # If the room has not yet been seen, or the receipt is newer, + # use it. + if ( + room_id not in results + or results[room_id]["stream_ordering"] + < room_result["stream_ordering"] + ): + results[room_id] = room_result + + return results + + @cached() + async def _get_receipts_for_user_with_orderings( self, user_id: str, receipt_type: str ) -> JsonDict: """ @@ -220,8 +286,9 @@ def f(txn: LoggingTransaction) -> List[Tuple[str, str, int, int]]: " WHERE rl.room_id = e.room_id" " AND rl.event_id = e.event_id" " AND user_id = ?" + " AND receipt_type = ?" ) - txn.execute(sql, (user_id,)) + txn.execute(sql, (user_id, receipt_type)) return cast(List[Tuple[str, str, int, int]], txn.fetchall()) rows = await self.db_pool.runInteraction( @@ -552,9 +619,9 @@ def _invalidate_get_users_with_receipts_in_room( def invalidate_caches_for_receipt( self, room_id: str, receipt_type: str, user_id: str ) -> None: - self.get_receipts_for_user.invalidate((user_id, receipt_type)) + self._get_receipts_for_user_with_orderings.invalidate((user_id, receipt_type)) self._get_linearized_receipts_for_room.invalidate((room_id,)) - self.get_last_receipt_event_id_for_user.invalidate( + self._get_last_receipt_event_id_for_user.invalidate( (user_id, room_id, receipt_type) ) self._invalidate_get_users_with_receipts_in_room(room_id, receipt_type, user_id) @@ -590,8 +657,8 @@ def insert_linearized_receipt_txn( """Inserts a receipt into the database if it's newer than the current one. Returns: - None if the RR is older than the current RR - otherwise, the rx timestamp of the event that the RR corresponds to + None if the receipt is older than the current receipt + otherwise, the rx timestamp of the event that the receipt corresponds to (or 0 if the event is unknown) """ assert self._can_write_to_receipts @@ -612,7 +679,7 @@ def insert_linearized_receipt_txn( if stream_ordering is not None: sql = ( "SELECT stream_ordering, event_id FROM events" - " INNER JOIN receipts_linearized as r USING (event_id, room_id)" + " INNER JOIN receipts_linearized AS r USING (event_id, room_id)" " WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?" ) txn.execute(sql, (room_id, receipt_type, user_id)) @@ -653,7 +720,10 @@ def insert_linearized_receipt_txn( lock=False, ) - if receipt_type == ReceiptTypes.READ and stream_ordering is not None: + if ( + receipt_type in (ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE) + and stream_ordering is not None + ): self._remove_old_push_actions_before_txn( # type: ignore[attr-defined] txn, room_id=room_id, user_id=user_id, stream_ordering=stream_ordering ) @@ -672,6 +742,10 @@ async def insert_receipt( Automatically does conversion between linearized and graph representations. + + Returns: + The new receipts stream ID and token, if the receipt is newer than + what was previously persisted. None, otherwise. """ assert self._can_write_to_receipts @@ -719,6 +793,7 @@ def graph_to_linear(txn: LoggingTransaction) -> str: stream_id=stream_id, ) + # If the receipt was older than the currently persisted one, nothing to do. if event_ts is None: return None @@ -774,7 +849,10 @@ def insert_graph_receipt_txn( receipt_type, user_id, ) - txn.call_after(self.get_receipts_for_user.invalidate, (user_id, receipt_type)) + txn.call_after( + self._get_receipts_for_user_with_orderings.invalidate, + (user_id, receipt_type), + ) # FIXME: This shouldn't invalidate the whole cache txn.call_after(self._get_linearized_receipts_for_room.invalidate, (room_id,)) diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index 65ab7db0c811..c12a9120f029 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -15,7 +15,7 @@ from typing import List -from synapse.api.constants import ReadReceiptEventFields, ReceiptTypes +from synapse.api.constants import ReceiptTypes from synapse.types import JsonDict from tests import unittest @@ -25,20 +25,15 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor, clock, hs): self.event_source = hs.get_event_sources().sources.receipt - # In the first param of _test_filters_hidden we use "hidden" instead of - # ReadReceiptEventFields.MSC2285_HIDDEN. We do this because we're mocking - # the data from the database which doesn't use the prefix - def test_filters_out_hidden_receipt(self): self._test_filters_hidden( [ { "content": { "$1435641916114394fHBLK:matrix.org": { - ReceiptTypes.READ: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, - "hidden": True, } } } @@ -50,58 +45,23 @@ def test_filters_out_hidden_receipt(self): [], ) - def test_does_not_filter_out_our_hidden_receipt(self): - self._test_filters_hidden( - [ - { - "content": { - "$1435641916hfgh4394fHBLK:matrix.org": { - ReceiptTypes.READ: { - "@me:server.org": { - "ts": 1436451550453, - "hidden": True, - }, - } - } - }, - "room_id": "!jEsUZKDJdhlrceRyVU:example.org", - "type": "m.receipt", - } - ], - [ - { - "content": { - "$1435641916hfgh4394fHBLK:matrix.org": { - ReceiptTypes.READ: { - "@me:server.org": { - "ts": 1436451550453, - ReadReceiptEventFields.MSC2285_HIDDEN: True, - }, - } - } - }, - "room_id": "!jEsUZKDJdhlrceRyVU:example.org", - "type": "m.receipt", - } - ], - ) - def test_filters_out_hidden_receipt_and_ignores_rest(self): self._test_filters_hidden( [ { "content": { "$1dgdgrd5641916114394fHBLK:matrix.org": { - ReceiptTypes.READ: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, - "hidden": True, }, + }, + ReceiptTypes.READ: { "@user:jki.re": { "ts": 1436451550453, }, - } - } + }, + }, }, "room_id": "!jEsUZKDJdhlrceRyVU:example.org", "type": "m.receipt", @@ -130,10 +90,9 @@ def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self): { "content": { "$14356419edgd14394fHBLK:matrix.org": { - ReceiptTypes.READ: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, - "hidden": True, }, } }, @@ -223,7 +182,6 @@ def test_handles_empty_event(self): [ { "content": { - "$143564gdfg6114394fHBLK:matrix.org": {}, "$1435641916114394fHBLK:matrix.org": { ReceiptTypes.READ: { "@user:jki.re": { @@ -244,10 +202,9 @@ def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(sel { "content": { "$14356419edgd14394fHBLK:matrix.org": { - ReceiptTypes.READ: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, - "hidden": True, }, } }, @@ -306,7 +263,73 @@ def test_handles_string_data(self): "type": "m.receipt", }, ], - [], + [ + { + "content": { + "$14356419edgd14394fHBLK:matrix.org": { + ReceiptTypes.READ: { + "@rikj:jki.re": "string", + } + }, + }, + "room_id": "!jEsUZKDJdhlrceRyVU:example.org", + "type": "m.receipt", + }, + ], + ) + + def test_leaves_our_hidden_and_their_public(self): + self._test_filters_hidden( + [ + { + "content": { + "$1dgdgrd5641916114394fHBLK:matrix.org": { + ReceiptTypes.READ_PRIVATE: { + "@me:server.org": { + "ts": 1436451550453, + }, + }, + ReceiptTypes.READ: { + "@rikj:jki.re": { + "ts": 1436451550453, + }, + }, + "a.receipt.type": { + "@rikj:jki.re": { + "ts": 1436451550453, + }, + }, + }, + }, + "room_id": "!jEsUZKDJdhlrceRyVU:example.org", + "type": "m.receipt", + } + ], + [ + { + "content": { + "$1dgdgrd5641916114394fHBLK:matrix.org": { + ReceiptTypes.READ_PRIVATE: { + "@me:server.org": { + "ts": 1436451550453, + }, + }, + ReceiptTypes.READ: { + "@rikj:jki.re": { + "ts": 1436451550453, + }, + }, + "a.receipt.type": { + "@rikj:jki.re": { + "ts": 1436451550453, + }, + }, + } + }, + "room_id": "!jEsUZKDJdhlrceRyVU:example.org", + "type": "m.receipt", + } + ], ) def _test_filters_hidden( diff --git a/tests/replication/slave/storage/test_receipts.py b/tests/replication/slave/storage/test_receipts.py index de19e75b9dec..5bbbd5fbcbab 100644 --- a/tests/replication/slave/storage/test_receipts.py +++ b/tests/replication/slave/storage/test_receipts.py @@ -14,26 +14,246 @@ from synapse.api.constants import ReceiptTypes from synapse.replication.slave.storage.receipts import SlavedReceiptsStore +from synapse.types import UserID, create_requester + +from tests.test_utils.event_injection import create_event from ._base import BaseSlavedStoreTestCase -USER_ID = "@feeling:blue" -ROOM_ID = "!room:blue" -EVENT_ID = "$event:blue" +OTHER_USER_ID = "@other:test" +OUR_USER_ID = "@our:test" class SlavedReceiptTestCase(BaseSlavedStoreTestCase): STORE_TYPE = SlavedReceiptsStore - def test_receipt(self): - self.check("get_receipts_for_user", [USER_ID, ReceiptTypes.READ], {}) + def prepare(self, reactor, clock, homeserver): + super().prepare(reactor, clock, homeserver) + self.room_creator = homeserver.get_room_creation_handler() + self.persist_event_storage = self.hs.get_storage().persistence + + # Create a test user + self.ourUser = UserID.from_string(OUR_USER_ID) + self.ourRequester = create_requester(self.ourUser) + + # Create a second test user + self.otherUser = UserID.from_string(OTHER_USER_ID) + self.otherRequester = create_requester(self.otherUser) + + # Create a test room + info, _ = self.get_success(self.room_creator.create_room(self.ourRequester, {})) + self.room_id1 = info["room_id"] + + # Create a second test room + info, _ = self.get_success(self.room_creator.create_room(self.ourRequester, {})) + self.room_id2 = info["room_id"] + + # Join the second user to the first room + memberEvent, memberEventContext = self.get_success( + create_event( + self.hs, + room_id=self.room_id1, + type="m.room.member", + sender=self.otherRequester.user.to_string(), + state_key=self.otherRequester.user.to_string(), + content={"membership": "join"}, + ) + ) + self.get_success( + self.persist_event_storage.persist_event(memberEvent, memberEventContext) + ) + + # Join the second user to the second room + memberEvent, memberEventContext = self.get_success( + create_event( + self.hs, + room_id=self.room_id2, + type="m.room.member", + sender=self.otherRequester.user.to_string(), + state_key=self.otherRequester.user.to_string(), + content={"membership": "join"}, + ) + ) + self.get_success( + self.persist_event_storage.persist_event(memberEvent, memberEventContext) + ) + + def test_return_empty_with_no_data(self): + res = self.get_success( + self.master_store.get_receipts_for_user( + OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] + ) + ) + self.assertEqual(res, {}) + + res = self.get_success( + self.master_store.get_receipts_for_user_with_orderings( + OUR_USER_ID, + [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], + ) + ) + self.assertEqual(res, {}) + + res = self.get_success( + self.master_store.get_last_receipt_event_id_for_user( + OUR_USER_ID, + self.room_id1, + [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], + ) + ) + self.assertEqual(res, None) + + def test_get_receipts_for_user(self): + # Send some events into the first room + event1_1_id = self.create_and_send_event( + self.room_id1, UserID.from_string(OTHER_USER_ID) + ) + event1_2_id = self.create_and_send_event( + self.room_id1, UserID.from_string(OTHER_USER_ID) + ) + + # Send public read receipt for the first event + self.get_success( + self.master_store.insert_receipt( + self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_1_id], {} + ) + ) + # Send private read receipt for the second event self.get_success( self.master_store.insert_receipt( - ROOM_ID, ReceiptTypes.READ, USER_ID, [EVENT_ID], {} + self.room_id1, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event1_2_id], {} + ) + ) + + # Test we get the latest event when we want both private and public receipts + res = self.get_success( + self.master_store.get_receipts_for_user( + OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] ) ) - self.replicate() - self.check( - "get_receipts_for_user", [USER_ID, ReceiptTypes.READ], {ROOM_ID: EVENT_ID} + self.assertEqual(res, {self.room_id1: event1_2_id}) + + # Test we get the older event when we want only public receipt + res = self.get_success( + self.master_store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ]) + ) + self.assertEqual(res, {self.room_id1: event1_1_id}) + + # Test we get the latest event when we want only the public receipt + res = self.get_success( + self.master_store.get_receipts_for_user( + OUR_USER_ID, [ReceiptTypes.READ_PRIVATE] + ) + ) + self.assertEqual(res, {self.room_id1: event1_2_id}) + + # Test receipt updating + self.get_success( + self.master_store.insert_receipt( + self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_2_id], {} + ) + ) + res = self.get_success( + self.master_store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ]) + ) + self.assertEqual(res, {self.room_id1: event1_2_id}) + + # Send some events into the second room + event2_1_id = self.create_and_send_event( + self.room_id2, UserID.from_string(OTHER_USER_ID) + ) + + # Test new room is reflected in what the method returns + self.get_success( + self.master_store.insert_receipt( + self.room_id2, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event2_1_id], {} + ) + ) + res = self.get_success( + self.master_store.get_receipts_for_user( + OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] + ) + ) + self.assertEqual(res, {self.room_id1: event1_2_id, self.room_id2: event2_1_id}) + + def test_get_last_receipt_event_id_for_user(self): + # Send some events into the first room + event1_1_id = self.create_and_send_event( + self.room_id1, UserID.from_string(OTHER_USER_ID) + ) + event1_2_id = self.create_and_send_event( + self.room_id1, UserID.from_string(OTHER_USER_ID) + ) + + # Send public read receipt for the first event + self.get_success( + self.master_store.insert_receipt( + self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_1_id], {} + ) + ) + # Send private read receipt for the second event + self.get_success( + self.master_store.insert_receipt( + self.room_id1, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event1_2_id], {} + ) + ) + + # Test we get the latest event when we want both private and public receipts + res = self.get_success( + self.master_store.get_last_receipt_event_id_for_user( + OUR_USER_ID, + self.room_id1, + [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], + ) + ) + self.assertEqual(res, event1_2_id) + + # Test we get the older event when we want only public receipt + res = self.get_success( + self.master_store.get_last_receipt_event_id_for_user( + OUR_USER_ID, self.room_id1, [ReceiptTypes.READ] + ) + ) + self.assertEqual(res, event1_1_id) + + # Test we get the latest event when we want only the private receipt + res = self.get_success( + self.master_store.get_last_receipt_event_id_for_user( + OUR_USER_ID, self.room_id1, [ReceiptTypes.READ_PRIVATE] + ) + ) + self.assertEqual(res, event1_2_id) + + # Test receipt updating + self.get_success( + self.master_store.insert_receipt( + self.room_id1, ReceiptTypes.READ, OUR_USER_ID, [event1_2_id], {} + ) + ) + res = self.get_success( + self.master_store.get_last_receipt_event_id_for_user( + OUR_USER_ID, self.room_id1, [ReceiptTypes.READ] + ) + ) + self.assertEqual(res, event1_2_id) + + # Send some events into the second room + event2_1_id = self.create_and_send_event( + self.room_id2, UserID.from_string(OTHER_USER_ID) + ) + + # Test new room is reflected in what the method returns + self.get_success( + self.master_store.insert_receipt( + self.room_id2, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event2_1_id], {} + ) + ) + res = self.get_success( + self.master_store.get_last_receipt_event_id_for_user( + OUR_USER_ID, + self.room_id2, + [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], + ) ) + self.assertEqual(res, event2_1_id) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index cb765455c16d..67c94dd18fbd 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -23,7 +23,6 @@ from synapse.api.constants import ( EventContentFields, EventTypes, - ReadReceiptEventFields, ReceiptTypes, RelationTypes, ) @@ -347,7 +346,7 @@ def test_knock_room_state(self) -> None: # Knock on a room channel = self.make_request( "POST", - "/_matrix/client/r0/knock/%s" % (self.room_id,), + f"/_matrix/client/r0/knock/{self.room_id}", b"{}", self.knocker_tok, ) @@ -412,18 +411,79 @@ def test_hidden_read_receipts(self) -> None: # Send a message as the first user res = self.helper.send(self.room_id, body="hello", tok=self.tok) - # Send a read receipt to tell the server the first user's message was read - body = json.dumps({ReadReceiptEventFields.MSC2285_HIDDEN: True}).encode("utf8") + # Send a private read receipt to tell the server the first user's message was read channel = self.make_request( "POST", - "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]), - body, + f"/rooms/{self.room_id}/receipt/org.matrix.msc2285.read.private/{res['event_id']}", + {}, access_token=self.tok2, ) self.assertEqual(channel.code, 200) - # Test that the first user can't see the other user's hidden read receipt - self.assertEqual(self._get_read_receipt(), None) + # Test that the first user can't see the other user's private read receipt + self.assertIsNone(self._get_read_receipt()) + + @override_config({"experimental_features": {"msc2285_enabled": True}}) + def test_public_receipt_can_override_private(self) -> None: + """ + Sending a public read receipt to the same event which has a private read + receipt should cause that receipt to become public. + """ + # Send a message as the first user + res = self.helper.send(self.room_id, body="hello", tok=self.tok) + + # Send a private read receipt + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res['event_id']}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200) + self.assertIsNone(self._get_read_receipt()) + + # Send a public read receipt + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ}/{res['event_id']}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200) + + # Test that we did override the private read receipt + self.assertNotEqual(self._get_read_receipt(), None) + + @override_config({"experimental_features": {"msc2285_enabled": True}}) + def test_private_receipt_cannot_override_public(self) -> None: + """ + Sending a private read receipt to the same event which has a public read + receipt should cause no change. + """ + # Send a message as the first user + res = self.helper.send(self.room_id, body="hello", tok=self.tok) + + # Send a public read receipt + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ}/{res['event_id']}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200) + self.assertNotEqual(self._get_read_receipt(), None) + + # Send a private read receipt + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res['event_id']}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200) + + # Test that we didn't override the public read receipt + self.assertIsNone(self._get_read_receipt()) @parameterized.expand( [ @@ -455,7 +515,7 @@ def test_read_receipt_with_empty_body( # Send a read receipt for this message with an empty body channel = self.make_request( "POST", - "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]), + f"/rooms/{self.room_id}/receipt/m.read/{res['event_id']}", access_token=self.tok2, custom_headers=[("User-Agent", user_agent)], ) @@ -479,6 +539,9 @@ def is_read_receipt(event: JsonDict) -> bool: # Store the next batch for the next request. self.next_batch = channel.json_body["next_batch"] + if channel.json_body.get("rooms", None) is None: + return None + # Return the read receipt ephemeral_events = channel.json_body["rooms"]["join"][self.room_id][ "ephemeral" @@ -499,7 +562,10 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() - config["experimental_features"] = {"msc2654_enabled": True} + config["experimental_features"] = { + "msc2654_enabled": True, + "msc2285_enabled": True, + } return config def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: @@ -564,7 +630,7 @@ def test_unread_counts(self) -> None: body = json.dumps({ReceiptTypes.READ: res["event_id"]}).encode("utf8") channel = self.make_request( "POST", - "/rooms/%s/read_markers" % self.room_id, + f"/rooms/{self.room_id}/read_markers", body, access_token=self.tok, ) @@ -578,11 +644,10 @@ def test_unread_counts(self) -> None: self._check_unread_count(1) # Send a read receipt to tell the server we've read the latest event. - body = json.dumps({ReadReceiptEventFields.MSC2285_HIDDEN: True}).encode("utf8") channel = self.make_request( "POST", - "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]), - body, + f"/rooms/{self.room_id}/receipt/org.matrix.msc2285.read.private/{res['event_id']}", + {}, access_token=self.tok, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -644,13 +709,73 @@ def test_unread_counts(self) -> None: self._check_unread_count(4) # Check that tombstone events changes increase the unread counter. - self.helper.send_state( + res1 = self.helper.send_state( self.room_id, EventTypes.Tombstone, {"replacement_room": "!someroom:test"}, tok=self.tok2, ) self._check_unread_count(5) + res2 = self.helper.send(self.room_id, "hello", tok=self.tok2) + + # Make sure both m.read and org.matrix.msc2285.read.private advance + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/m.read/{res1['event_id']}", + {}, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self._check_unread_count(1) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/org.matrix.msc2285.read.private/{res2['event_id']}", + {}, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self._check_unread_count(0) + + # We test for both receipt types that influence notification counts + @parameterized.expand([ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE]) + def test_read_receipts_only_go_down(self, receipt_type: ReceiptTypes) -> None: + # Join the new user + self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + + # Send messages + res1 = self.helper.send(self.room_id, "hello", tok=self.tok2) + res2 = self.helper.send(self.room_id, "hello", tok=self.tok2) + + # Read last event + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{receipt_type}/{res2['event_id']}", + {}, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self._check_unread_count(0) + + # Make sure neither m.read nor org.matrix.msc2285.read.private make the + # read receipt go up to an older event + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/org.matrix.msc2285.read.private/{res1['event_id']}", + {}, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self._check_unread_count(0) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/m.read/{res1['event_id']}", + {}, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + self._check_unread_count(0) def _check_unread_count(self, expected_count: int) -> None: """Syncs and compares the unread count with the expected value.""" @@ -663,9 +788,11 @@ def _check_unread_count(self, expected_count: int) -> None: self.assertEqual(channel.code, 200, channel.json_body) - room_entry = channel.json_body["rooms"]["join"][self.room_id] + room_entry = ( + channel.json_body.get("rooms", {}).get("join", {}).get(self.room_id, {}) + ) self.assertEqual( - room_entry["org.matrix.msc2654.unread_count"], + room_entry.get("org.matrix.msc2654.unread_count", 0), expected_count, room_entry, ) From 7fbf42499d92ec3c9a05d9f36ec5fecd1ab1f18c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 4 May 2022 14:11:21 -0400 Subject: [PATCH 111/263] Use `getClientAddress` instead of `getClientIP`. (#12599) getClientIP was deprecated in Twisted 18.4.0, which also added getClientAddress. The Synapse minimum version for Twisted is currently 18.9.0, so all supported versions have the new API. --- changelog.d/12599.misc | 1 + synapse/api/auth.py | 4 ++-- synapse/handlers/auth.py | 2 +- synapse/handlers/identity.py | 2 +- synapse/handlers/sso.py | 4 ++-- synapse/http/site.py | 6 +++--- synapse/logging/opentracing.py | 2 +- synapse/rest/client/auth.py | 8 +++++--- synapse/rest/client/login.py | 14 ++++++++++---- synapse/rest/client/register.py | 6 +++--- tests/api/test_auth.py | 18 +++++++++--------- tests/handlers/test_cas.py | 2 +- tests/handlers/test_oidc.py | 4 ++-- tests/handlers/test_saml.py | 2 +- tests/replication/_base.py | 20 ++++++++++++-------- tests/server.py | 13 ++++++++----- 16 files changed, 62 insertions(+), 46 deletions(-) create mode 100644 changelog.d/12599.misc diff --git a/changelog.d/12599.misc b/changelog.d/12599.misc new file mode 100644 index 000000000000..d01278bbce51 --- /dev/null +++ b/changelog.d/12599.misc @@ -0,0 +1 @@ +Use `getClientAddress` instead of the deprecated `getClientIP`. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 01c32417d862..f6202ef7a5ab 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -187,7 +187,7 @@ async def _wrapped_get_user_by_req( Once get_user_by_req has set up the opentracing span, this does the actual work. """ try: - ip_addr = request.getClientIP() + ip_addr = request.getClientAddress().host user_agent = get_request_user_agent(request) access_token = self.get_access_token_from_request(request) @@ -356,7 +356,7 @@ async def _get_appservice_user_id_and_device_id( return None, None, None if app_service.ip_range_whitelist: - ip_address = IPAddress(request.getClientIP()) + ip_address = IPAddress(request.getClientAddress().host) if ip_address not in app_service.ip_range_whitelist: return None, None, None diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 22678d486d73..ad41337b2892 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -551,7 +551,7 @@ async def check_ui_auth( await self.store.set_ui_auth_clientdict(sid, clientdict) user_agent = get_request_user_agent(request) - clientip = request.getClientIP() + clientip = request.getClientAddress().host await self.store.add_user_agent_ip_to_ui_auth_session( session.session_id, user_agent, clientip diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index c183e9c46523..9bca2bc4b24e 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -92,7 +92,7 @@ async def ratelimit_request_token_requests( """ await self._3pid_validation_ratelimiter_ip.ratelimit( - None, (medium, request.getClientIP()) + None, (medium, request.getClientAddress().host) ) await self._3pid_validation_ratelimiter_address.ratelimit( None, (medium, address) diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index e4fe94e557ad..1e171f3f7115 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -468,7 +468,7 @@ async def complete_sso_login_request( auth_provider_id, remote_user_id, get_request_user_agent(request), - request.getClientIP(), + request.getClientAddress().host, ) new_user = True elif self._sso_update_profile_information: @@ -928,7 +928,7 @@ async def register_sso_user(self, request: Request, session_id: str) -> None: session.auth_provider_id, session.remote_user_id, get_request_user_agent(request), - request.getClientIP(), + request.getClientAddress().host, ) logger.info( diff --git a/synapse/http/site.py b/synapse/http/site.py index 40f6c0489451..0b85a57d7787 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -238,7 +238,7 @@ def render(self, resrc: Resource) -> None: request_id, request=ContextRequest( request_id=request_id, - ip_address=self.getClientIP(), + ip_address=self.getClientAddress().host, site_tag=self.synapse_site.site_tag, # The requester is going to be unknown at this point. requester=None, @@ -381,7 +381,7 @@ def _started_processing(self, servlet_name: str) -> None: self.synapse_site.access_logger.debug( "%s - %s - Received request: %s %s", - self.getClientIP(), + self.getClientAddress().host, self.synapse_site.site_tag, self.get_method(), self.get_redacted_uri(), @@ -429,7 +429,7 @@ def _finished_processing(self) -> None: "%s - %s - {%s}" " Processed request: %.3fsec/%.3fsec (%.3fsec, %.3fsec) (%.3fsec/%.3fsec/%d)" ' %sB %s "%s %s %s" "%s" [%d dbevts]', - self.getClientIP(), + self.getClientAddress().host, self.synapse_site.site_tag, requester, processing_time, diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index f86ee9aac7a5..a02b5bf6bd28 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -884,7 +884,7 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False): tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER, tags.HTTP_METHOD: request.get_method(), tags.HTTP_URL: request.get_redacted_uri(), - tags.PEER_HOST_IPV6: request.getClientIP(), + tags.PEER_HOST_IPV6: request.getClientAddress().host, } request_name = request.request_metrics.name diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index e0b2b80e5b9b..eb77337044da 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -112,7 +112,7 @@ async def on_POST(self, request: Request, stagetype: str) -> None: try: await self.auth_handler.add_oob_auth( - LoginType.RECAPTCHA, authdict, request.getClientIP() + LoginType.RECAPTCHA, authdict, request.getClientAddress().host ) except LoginError as e: # Authentication failed, let user try again @@ -132,7 +132,7 @@ async def on_POST(self, request: Request, stagetype: str) -> None: try: await self.auth_handler.add_oob_auth( - LoginType.TERMS, authdict, request.getClientIP() + LoginType.TERMS, authdict, request.getClientAddress().host ) except LoginError as e: # Authentication failed, let user try again @@ -161,7 +161,9 @@ async def on_POST(self, request: Request, stagetype: str) -> None: try: await self.auth_handler.add_oob_auth( - LoginType.REGISTRATION_TOKEN, authdict, request.getClientIP() + LoginType.REGISTRATION_TOKEN, + authdict, + request.getClientAddress().host, ) except LoginError as e: html = self.registration_token_template.render( diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 71d8038448da..cf4196ac0a2b 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -176,7 +176,7 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, LoginResponse]: if appservice.is_rate_limited(): await self._address_ratelimiter.ratelimit( - None, request.getClientIP() + None, request.getClientAddress().host ) result = await self._do_appservice_login( @@ -188,19 +188,25 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, LoginResponse]: self.jwt_enabled and login_submission["type"] == LoginRestServlet.JWT_TYPE ): - await self._address_ratelimiter.ratelimit(None, request.getClientIP()) + await self._address_ratelimiter.ratelimit( + None, request.getClientAddress().host + ) result = await self._do_jwt_login( login_submission, should_issue_refresh_token=should_issue_refresh_token, ) elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE: - await self._address_ratelimiter.ratelimit(None, request.getClientIP()) + await self._address_ratelimiter.ratelimit( + None, request.getClientAddress().host + ) result = await self._do_token_login( login_submission, should_issue_refresh_token=should_issue_refresh_token, ) else: - await self._address_ratelimiter.ratelimit(None, request.getClientIP()) + await self._address_ratelimiter.ratelimit( + None, request.getClientAddress().host + ) result = await self._do_other_login( login_submission, should_issue_refresh_token=should_issue_refresh_token, diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 13ef6b35a066..47b6db1ebfba 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -352,7 +352,7 @@ async def on_GET(self, request: Request) -> Tuple[int, JsonDict]: if self.inhibit_user_in_use_error: return 200, {"available": True} - ip = request.getClientIP() + ip = request.getClientAddress().host with self.ratelimiter.ratelimit(ip) as wait_deferred: await wait_deferred @@ -394,7 +394,7 @@ def __init__(self, hs: "HomeServer"): ) async def on_GET(self, request: Request) -> Tuple[int, JsonDict]: - await self.ratelimiter.ratelimit(None, (request.getClientIP(),)) + await self.ratelimiter.ratelimit(None, (request.getClientAddress().host,)) if not self.hs.config.registration.enable_registration: raise SynapseError( @@ -441,7 +441,7 @@ def __init__(self, hs: "HomeServer"): async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: body = parse_json_object_from_request(request) - client_addr = request.getClientIP() + client_addr = request.getClientAddress().host await self.ratelimiter.ratelimit(None, client_addr, update=False) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 3e057899236d..d547df8a648d 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -105,7 +105,7 @@ def test_get_user_by_req_appservice_valid_token(self): self.store.get_user_by_access_token = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) @@ -124,7 +124,7 @@ def test_get_user_by_req_appservice_valid_token_good_ip(self): self.store.get_user_by_access_token = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "192.168.10.10" + request.getClientAddress.return_value.host = "192.168.10.10" request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) @@ -143,7 +143,7 @@ def test_get_user_by_req_appservice_valid_token_bad_ip(self): self.store.get_user_by_access_token = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "131.111.8.42" + request.getClientAddress.return_value.host = "131.111.8.42" request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() f = self.get_failure( @@ -190,7 +190,7 @@ def test_get_user_by_req_appservice_valid_token_valid_user_id(self): self.store.get_user_by_access_token = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.args[b"user_id"] = [masquerading_user_id] request.requestHeaders.getRawHeaders = mock_getRawHeaders() @@ -209,7 +209,7 @@ def test_get_user_by_req_appservice_valid_token_bad_user_id(self): self.store.get_user_by_access_token = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.args[b"user_id"] = [masquerading_user_id] request.requestHeaders.getRawHeaders = mock_getRawHeaders() @@ -236,7 +236,7 @@ def test_get_user_by_req_appservice_valid_token_valid_device_id(self): self.store.get_device = simple_async_mock({"hidden": False}) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.args[b"user_id"] = [masquerading_user_id] request.args[b"org.matrix.msc3202.device_id"] = [masquerading_device_id] @@ -268,7 +268,7 @@ def test_get_user_by_req_appservice_valid_token_invalid_device_id(self): self.store.get_device = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.args[b"user_id"] = [masquerading_user_id] request.args[b"org.matrix.msc3202.device_id"] = [masquerading_device_id] @@ -288,7 +288,7 @@ def test_get_user_by_req__puppeted_token__not_tracking_puppeted_mau(self): ) self.store.insert_client_ip = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_success(self.auth.get_user_by_req(request)) @@ -305,7 +305,7 @@ def test_get_user_by_req__puppeted_token__tracking_puppeted_mau(self): ) self.store.insert_client_ip = simple_async_mock(None) request = Mock(args={}) - request.getClientIP.return_value = "127.0.0.1" + request.getClientAddress.return_value.host = "127.0.0.1" request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() self.get_success(self.auth.get_user_by_req(request)) diff --git a/tests/handlers/test_cas.py b/tests/handlers/test_cas.py index 751025c5dae0..2b21547d0f55 100644 --- a/tests/handlers/test_cas.py +++ b/tests/handlers/test_cas.py @@ -204,7 +204,7 @@ def _mock_request(): mock = Mock( spec=[ "finish", - "getClientIP", + "getClientAddress", "getHeader", "setHeader", "setResponseCode", diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 9684120c70f3..1231aed94458 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -1300,7 +1300,7 @@ def _build_callback_request( "getCookie", "cookies", "requestHeaders", - "getClientIP", + "getClientAddress", "getHeader", ] ) @@ -1310,5 +1310,5 @@ def _build_callback_request( request.args = {} request.args[b"code"] = [code.encode("utf-8")] request.args[b"state"] = [state.encode("utf-8")] - request.getClientIP.return_value = ip_address + request.getClientAddress.return_value.host = ip_address return request diff --git a/tests/handlers/test_saml.py b/tests/handlers/test_saml.py index e2f0f90ef114..a0f84e29403d 100644 --- a/tests/handlers/test_saml.py +++ b/tests/handlers/test_saml.py @@ -352,7 +352,7 @@ def _mock_request(): mock = Mock( spec=[ "finish", - "getClientIP", + "getClientAddress", "getHeader", "setHeader", "setResponseCode", diff --git a/tests/replication/_base.py b/tests/replication/_base.py index a0589b6d6a4b..a7602b4c96ae 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -154,10 +154,12 @@ def handle_http_replication_attempt(self) -> SynapseRequest: self.assertEqual(port, 8765) # Set up client side protocol - client_protocol = client_factory.buildProtocol(None) + client_address = IPv4Address("TCP", "127.0.0.1", 1234) + client_protocol = client_factory.buildProtocol(("127.0.0.1", 1234)) # Set up the server side protocol - channel = self.site.buildProtocol(None) + server_address = IPv4Address("TCP", host, port) + channel = self.site.buildProtocol((host, port)) # hook into the channel's request factory so that we can keep a record # of the requests @@ -173,12 +175,12 @@ def request_factory(*args, **kwargs): # Connect client to server and vice versa. client_to_server_transport = FakeTransport( - channel, self.reactor, client_protocol + channel, self.reactor, client_protocol, server_address, client_address ) client_protocol.makeConnection(client_to_server_transport) server_to_client_transport = FakeTransport( - client_protocol, self.reactor, channel + client_protocol, self.reactor, channel, client_address, server_address ) channel.makeConnection(server_to_client_transport) @@ -406,19 +408,21 @@ def _handle_http_replication_attempt(self, hs, repl_port): self.assertEqual(port, repl_port) # Set up client side protocol - client_protocol = client_factory.buildProtocol(None) + client_address = IPv4Address("TCP", "127.0.0.1", 1234) + client_protocol = client_factory.buildProtocol(("127.0.0.1", 1234)) # Set up the server side protocol - channel = self._hs_to_site[hs].buildProtocol(None) + server_address = IPv4Address("TCP", host, port) + channel = self._hs_to_site[hs].buildProtocol((host, port)) # Connect client to server and vice versa. client_to_server_transport = FakeTransport( - channel, self.reactor, client_protocol + channel, self.reactor, client_protocol, server_address, client_address ) client_protocol.makeConnection(client_to_server_transport) server_to_client_transport = FakeTransport( - client_protocol, self.reactor, channel + client_protocol, self.reactor, channel, client_address, server_address ) channel.makeConnection(server_to_client_transport) diff --git a/tests/server.py b/tests/server.py index 16559d2588c2..8f30e250c83c 100644 --- a/tests/server.py +++ b/tests/server.py @@ -181,7 +181,7 @@ def requestDone(self, _self): self.resource_usage = _self.logcontext.get_resource_usage() def getPeer(self): - # We give an address so that getClientIP returns a non null entry, + # We give an address so that getClientAddress/getClientIP returns a non null entry, # causing us to record the MAU return address.IPv4Address("TCP", self._ip, 3423) @@ -562,7 +562,10 @@ class FakeTransport: """ _peer_address: Optional[IAddress] = attr.ib(default=None) - """The value to be returend by getPeer""" + """The value to be returned by getPeer""" + + _host_address: Optional[IAddress] = attr.ib(default=None) + """The value to be returned by getHost""" disconnecting = False disconnected = False @@ -571,11 +574,11 @@ class FakeTransport: producer = attr.ib(default=None) autoflush = attr.ib(default=True) - def getPeer(self): + def getPeer(self) -> Optional[IAddress]: return self._peer_address - def getHost(self): - return None + def getHost(self) -> Optional[IAddress]: + return self._host_address def loseConnection(self, reason=None): if not self.disconnecting: From 2d74a8c17876ed235249b1423d63c920c614a530 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 4 May 2022 19:33:26 +0100 Subject: [PATCH 112/263] Add `mau_appservice_trial_days` config (#12619) * Add mau_appservice_trial_days * Add a test * Tweaks * changelog * Ensure we sync after the delay * Fix types * Add config statement * Fix test * Reinstate logging that got removed * Fix feature name --- changelog.d/12619.feature | 1 + docs/sample_config.yaml | 7 ++ .../configuration/config_documentation.md | 14 ++++ synapse/config/server.py | 8 ++ .../storage/databases/main/registration.py | 8 +- tests/test_mau.py | 74 +++++++++++++++++++ 6 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12619.feature diff --git a/changelog.d/12619.feature b/changelog.d/12619.feature new file mode 100644 index 000000000000..b0fc0f5fedf8 --- /dev/null +++ b/changelog.d/12619.feature @@ -0,0 +1 @@ +Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 5eba0fcf3d1d..a803b8261dcd 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -407,6 +407,11 @@ manhole_settings: # sign up in a short space of time never to return after their initial # session. # +# The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but +# applies a different trial number if the user was registered by an appservice. +# A value of 0 means no trial days are applied. Appservices not listed in this +# dictionary use the value of `mau_trial_days` instead. +# # 'mau_limit_alerting' is a means of limiting client side alerting # should the mau limit be reached. This is useful for small instances # where the admin has 5 mau seats (say) for 5 specific people and no @@ -417,6 +422,8 @@ manhole_settings: #max_mau_value: 50 #mau_trial_days: 2 #mau_limit_alerting: false +#mau_appservice_trial_days: +# "appservice-id": 1 # If enabled, the metrics for the number of monthly active users will # be populated, however no one will be limited. If limit_usage_by_mau diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 36db649467ca..21dad0ac41e2 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -627,6 +627,20 @@ Example configuration: mau_trial_days: 5 ``` --- +Config option: `mau_appservice_trial_days` + +The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but applies a different +trial number if the user was registered by an appservice. A value +of 0 means no trial days are applied. Appservices not listed in this dictionary +use the value of `mau_trial_days` instead. + +Example configuration: +```yaml +mau_appservice_trial_days: + my_appservice_id: 3 + another_appservice_id: 6 +``` +--- Config option: `mau_limit_alerting` The option `mau_limit_alerting` is a means of limiting client-side alerting diff --git a/synapse/config/server.py b/synapse/config/server.py index b6cd32641688..1e709c7cf519 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -413,6 +413,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) self.mau_trial_days = config.get("mau_trial_days", 0) + self.mau_appservice_trial_days = config.get("mau_appservice_trial_days", {}) self.mau_limit_alerting = config.get("mau_limit_alerting", True) # How long to keep redacted events in the database in unredacted form @@ -1105,6 +1106,11 @@ def generate_config_section( # sign up in a short space of time never to return after their initial # session. # + # The option `mau_appservice_trial_days` is similar to `mau_trial_days`, but + # applies a different trial number if the user was registered by an appservice. + # A value of 0 means no trial days are applied. Appservices not listed in this + # dictionary use the value of `mau_trial_days` instead. + # # 'mau_limit_alerting' is a means of limiting client side alerting # should the mau limit be reached. This is useful for small instances # where the admin has 5 mau seats (say) for 5 specific people and no @@ -1115,6 +1121,8 @@ def generate_config_section( #max_mau_value: 50 #mau_trial_days: 2 #mau_limit_alerting: false + #mau_appservice_trial_days: + # "appservice-id": 1 # If enabled, the metrics for the number of monthly active users will # be populated, however no one will be limited. If limit_usage_by_mau diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index d43163c27cae..4991360b70c9 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -215,7 +215,8 @@ async def get_userinfo_by_id(self, user_id: str) -> Optional[UserInfo]: async def is_trial_user(self, user_id: str) -> bool: """Checks if user is in the "trial" period, i.e. within the first - N days of registration defined by `mau_trial_days` config + N days of registration defined by `mau_trial_days` config or the + `mau_appservice_trial_days` config. Args: user_id: The user to check for trial status. @@ -226,7 +227,10 @@ async def is_trial_user(self, user_id: str) -> bool: return False now = self._clock.time_msec() - trial_duration_ms = self.config.server.mau_trial_days * 24 * 60 * 60 * 1000 + days = self.config.server.mau_appservice_trial_days.get( + info["appservice_id"], self.config.server.mau_trial_days + ) + trial_duration_ms = days * 24 * 60 * 60 * 1000 is_trial = (now - info["creation_ts"] * 1000) < trial_duration_ms return is_trial diff --git a/tests/test_mau.py b/tests/test_mau.py index 46bd3075de7b..5bbc361aa240 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -14,6 +14,8 @@ """Tests REST events for /rooms paths.""" +from typing import List + from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService @@ -229,6 +231,78 @@ def test_tracked_but_not_limited(self): self.reactor.advance(100) self.assertEqual(2, self.successResultOf(count)) + @override_config( + { + "mau_trial_days": 3, + "mau_appservice_trial_days": {"SomeASID": 1, "AnotherASID": 2}, + } + ) + def test_as_trial_days(self): + user_tokens: List[str] = [] + + def advance_time_and_sync(): + self.reactor.advance(24 * 60 * 61) + for token in user_tokens: + self.do_sync_for_user(token) + + # Cheekily add an application service that we use to register a new user + # with. + as_token_1 = "foobartoken1" + self.store.services_cache.append( + ApplicationService( + token=as_token_1, + hostname=self.hs.hostname, + id="SomeASID", + sender="@as_sender_1:test", + namespaces={"users": [{"regex": "@as_1.*", "exclusive": True}]}, + ) + ) + + as_token_2 = "foobartoken2" + self.store.services_cache.append( + ApplicationService( + token=as_token_2, + hostname=self.hs.hostname, + id="AnotherASID", + sender="@as_sender_2:test", + namespaces={"users": [{"regex": "@as_2.*", "exclusive": True}]}, + ) + ) + + user_tokens.append(self.create_user("kermit1")) + user_tokens.append(self.create_user("kermit2")) + user_tokens.append( + self.create_user("as_1kermit3", token=as_token_1, appservice=True) + ) + user_tokens.append( + self.create_user("as_2kermit4", token=as_token_2, appservice=True) + ) + + # Advance time by 1 day to include the first appservice + advance_time_and_sync() + self.assertEqual( + self.get_success(self.store.get_monthly_active_count_by_service()), + {"SomeASID": 1}, + ) + + # Advance time by 1 day to include the next appservice + advance_time_and_sync() + self.assertEqual( + self.get_success(self.store.get_monthly_active_count_by_service()), + {"SomeASID": 1, "AnotherASID": 1}, + ) + + # Advance time by 1 day to include the native users + advance_time_and_sync() + self.assertEqual( + self.get_success(self.store.get_monthly_active_count_by_service()), + { + "SomeASID": 1, + "AnotherASID": 1, + "native": 2, + }, + ) + def create_user(self, localpart, token=None, appservice=False): request_data = { "username": localpart, From c0379d6e5b3ec277788018670e69f9dc848bfb34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 May 2022 10:20:23 +0100 Subject: [PATCH 113/263] Reduce log spam when running multiple event persisters (#12610) --- changelog.d/12610.misc | 1 + synapse/replication/tcp/handler.py | 9 +++++++-- synapse/replication/tcp/resource.py | 9 +++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12610.misc diff --git a/changelog.d/12610.misc b/changelog.d/12610.misc new file mode 100644 index 000000000000..02efe0c72f51 --- /dev/null +++ b/changelog.d/12610.misc @@ -0,0 +1 @@ +Reduce log spam when running multiple event persisters. diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 615f1828dd73..9aba1cd45111 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -537,7 +537,7 @@ def on_POSITION(self, conn: IReplicationConnection, cmd: PositionCommand) -> Non # Ignore POSITION that are just our own echoes return - logger.info("Handling '%s %s'", cmd.NAME, cmd.to_line()) + logger.debug("Handling '%s %s'", cmd.NAME, cmd.to_line()) self._add_command_to_stream_queue(conn, cmd) @@ -567,6 +567,11 @@ async def _process_position( # between then and now. missing_updates = cmd.prev_token != current_token while missing_updates: + # Note: There may very well not be any new updates, but we check to + # make sure. This can particularly happen for the event stream where + # event persisters continuously send `POSITION`. See `resource.py` + # for why this can happen. + logger.info( "Fetching replication rows for '%s' between %i and %i", stream_name, @@ -590,7 +595,7 @@ async def _process_position( [stream.parse_row(row) for row in rows], ) - logger.info("Caught up with stream '%s' to %i", stream_name, cmd.new_token) + logger.info("Caught up with stream '%s' to %i", stream_name, cmd.new_token) # We've now caught up to position sent to us, notify handler. await self._replication_data_handler.on_position( diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index c6870df8f954..99f09669f00b 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -204,6 +204,15 @@ async def _run_notifier_loop(self) -> None: # turns out that e.g. account data streams share # their "current token" with each other, meaning # that it is *not* safe to send a POSITION. + + # Note: `last_token` may not *actually* be the + # last token we sent out in a RDATA or POSITION. + # This can happen if we sent out an RDATA for + # position X when our current token was say X+1. + # Other workers will see RDATA for X and then a + # POSITION with last token of X+1, which will + # cause them to check if there were any missing + # updates between X and X+1. logger.info( "Sending position: %s -> %s", stream.NAME, From cc7656099d882735507c41ea6efcba961f53b7ec Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 5 May 2022 12:11:52 +0100 Subject: [PATCH 114/263] Fix typo in some instances of enable_registration_token_3pid_bypass. (#12639) --- changelog.d/12639.bugfix | 1 + synapse/config/registration.py | 4 ++-- synapse/handlers/ui_auth/checkers.py | 2 +- synapse/rest/client/register.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12639.bugfix diff --git a/changelog.d/12639.bugfix b/changelog.d/12639.bugfix new file mode 100644 index 000000000000..c01596282c9f --- /dev/null +++ b/changelog.d/12639.bugfix @@ -0,0 +1 @@ +Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. \ No newline at end of file diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 70eb7e6a9778..d2d0425e62cb 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -43,8 +43,8 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.registration_requires_token = config.get( "registration_requires_token", False ) - self.enable_registration_token_3pid_bypasss = config.get( - "enable_registration_token_3pid_bypasss", False + self.enable_registration_token_3pid_bypass = config.get( + "enable_registration_token_3pid_bypass", False ) self.registration_shared_secret = config.get("registration_shared_secret") diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py index e2a441066d1c..05cebb5d4d89 100644 --- a/synapse/handlers/ui_auth/checkers.py +++ b/synapse/handlers/ui_auth/checkers.py @@ -258,7 +258,7 @@ def __init__(self, hs: "HomeServer"): self.hs = hs self._enabled = bool( hs.config.registration.registration_requires_token - ) or bool(hs.config.registration.enable_registration_token_3pid_bypasss) + ) or bool(hs.config.registration.enable_registration_token_3pid_bypass) self.store = hs.get_datastores().main def is_enabled(self) -> bool: diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 47b6db1ebfba..e8e51a9c66ad 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -930,7 +930,7 @@ def _calculate_registration_flows( flows.append([LoginType.MSISDN, LoginType.EMAIL_IDENTITY]) # Add a flow that doesn't require any 3pids, if the config requests it. - if config.registration.enable_registration_token_3pid_bypasss: + if config.registration.enable_registration_token_3pid_bypass: flows.append([LoginType.REGISTRATION_TOKEN]) # Prepend m.login.terms to all flows if we're requiring consent From ddc8bba00f06321e0758afaf88a65e190ca4c714 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 5 May 2022 07:51:19 -0400 Subject: [PATCH 115/263] Remove unused receipt datastore methods. (#12632) The last usage was removed in 5a1dd297c3ce105a7f516d9d9fe87b94b9d356c8 (#8059). --- changelog.d/12632.misc | 1 + synapse/storage/databases/main/receipts.py | 54 ---------------------- 2 files changed, 1 insertion(+), 54 deletions(-) create mode 100644 changelog.d/12632.misc diff --git a/changelog.d/12632.misc b/changelog.d/12632.misc new file mode 100644 index 000000000000..9e4ba79c794f --- /dev/null +++ b/changelog.d/12632.misc @@ -0,0 +1 @@ +Remove unused code related to receipts. diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 9e3d838eab9a..d035969a3178 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -22,7 +22,6 @@ Iterable, List, Optional, - Set, Tuple, cast, ) @@ -117,33 +116,6 @@ def get_max_receipt_stream_id(self) -> int: """Get the current max stream ID for receipts stream""" return self._receipts_id_gen.get_current_token() - @cached() - async def get_users_with_read_receipts_in_room(self, room_id: str) -> Set[str]: - receipts = await self.get_receipts_for_room(room_id, ReceiptTypes.READ) - return {r["user_id"] for r in receipts} - - @cached() - async def get_receipts_for_room( - self, room_id: str, receipt_type: str - ) -> List[Dict[str, Any]]: - """ - Fetch the event IDs for the latest receipt for all users in a room with the given receipt type. - - Args: - room_id: The room ID to fetch the receipt for. - receipt_type: The receipt type to fetch. - - Returns: - A list of dictionaries, one for each user ID. Each dictionary - contains a user ID and the event ID of that user's latest receipt. - """ - return await self.db_pool.simple_select_list( - table="receipts_linearized", - keyvalues={"room_id": room_id, "receipt_type": receipt_type}, - retcols=("user_id", "event_id"), - desc="get_receipts_for_room", - ) - async def get_last_receipt_event_id_for_user( self, user_id: str, room_id: str, receipt_types: Iterable[str] ) -> Optional[str]: @@ -599,23 +571,6 @@ def get_all_updated_receipts_txn( "get_all_updated_receipts", get_all_updated_receipts_txn ) - def _invalidate_get_users_with_receipts_in_room( - self, room_id: str, receipt_type: str, user_id: str - ) -> None: - if receipt_type != ReceiptTypes.READ: - return - - res = self.get_users_with_read_receipts_in_room.cache.get_immediate( - room_id, None, update_metrics=False - ) - - if res and user_id in res: - # We'd only be adding to the set, so no point invalidating if the - # user is already there - return - - self.get_users_with_read_receipts_in_room.invalidate((room_id,)) - def invalidate_caches_for_receipt( self, room_id: str, receipt_type: str, user_id: str ) -> None: @@ -624,8 +579,6 @@ def invalidate_caches_for_receipt( self._get_last_receipt_event_id_for_user.invalidate( (user_id, room_id, receipt_type) ) - self._invalidate_get_users_with_receipts_in_room(room_id, receipt_type, user_id) - self.get_receipts_for_room.invalidate((room_id, receipt_type)) def process_replication_rows( self, @@ -842,13 +795,6 @@ def insert_graph_receipt_txn( ) -> None: assert self._can_write_to_receipts - txn.call_after(self.get_receipts_for_room.invalidate, (room_id, receipt_type)) - txn.call_after( - self._invalidate_get_users_with_receipts_in_room, - room_id, - receipt_type, - user_id, - ) txn.call_after( self._get_receipts_for_user_with_orderings.invalidate, (user_id, receipt_type), From f90d381c7b93b98a5b6d7353196e2ae0912ea822 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 5 May 2022 08:15:12 -0400 Subject: [PATCH 116/263] Edits/annotations should not have any bundled aggregations calculated. (#12633) Fixes a regression from 8b309adb436c162510ed1402f33b8741d71fc058 (#11660) and b65acead428653b988351ae8d7b22127a22039cd (#11752) where events which themselves were an edit or an annotation could have bundled aggregations calculated, which is not allowed. --- changelog.d/12633.bugfix | 1 + synapse/handlers/relations.py | 38 ++++++++++++++--------------- tests/rest/client/test_relations.py | 31 +++++++++++++++++++++++ 3 files changed, 50 insertions(+), 20 deletions(-) create mode 100644 changelog.d/12633.bugfix diff --git a/changelog.d/12633.bugfix b/changelog.d/12633.bugfix new file mode 100644 index 000000000000..32332acd9a36 --- /dev/null +++ b/changelog.d/12633.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index cec5740fbd26..c2754ec918de 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -364,21 +364,29 @@ async def get_bundled_aggregations( The results may include additional events which are related to the requested events. """ - # De-duplicate events by ID to handle the same event requested multiple times. - # - # State events do not get bundled aggregations. - events_by_id = { - event.event_id: event for event in events if not event.is_state() - } - + # De-duplicated events by ID to handle the same event requested multiple times. + events_by_id = {} # A map of event ID to the relation in that event, if there is one. relations_by_id: Dict[str, str] = {} - for event_id, event in events_by_id.items(): + for event in events: + # State events do not get bundled aggregations. + if event.is_state(): + continue + relates_to = event.content.get("m.relates_to") + relation_type = None if isinstance(relates_to, collections.abc.Mapping): relation_type = relates_to.get("rel_type") - if isinstance(relation_type, str): - relations_by_id[event_id] = relation_type + # An event which is a replacement (ie edit) or annotation (ie, + # reaction) may not have any other event related to it. + if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): + continue + + # The event should get bundled aggregations. + events_by_id[event.event_id] = event + # Track the event's relation information for later. + if isinstance(relation_type, str): + relations_by_id[event.event_id] = relation_type # event ID -> bundled aggregation in non-serialized form. results: Dict[str, BundledAggregations] = {} @@ -413,16 +421,6 @@ async def get_bundled_aggregations( # Fetch other relations per event. for event in events_by_id.values(): - # An event which is a replacement (ie edit) or annotation (ie, reaction) - # may not have any other event related to it. - # - # XXX This is buggy, see https://github.com/matrix-org/synapse/issues/12566 - if relations_by_id.get(event.event_id) in ( - RelationTypes.ANNOTATION, - RelationTypes.REPLACE, - ): - continue - # Fetch any annotations (ie, reactions) to bundle with this event. annotations = await self.get_annotations_for_event( event.event_id, event.room_id, ignored_users=ignored_users diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 33ce9471b3a8..27dee8f6975d 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -620,6 +620,19 @@ def test_edit_edit(self) -> None: {"event_id": edit_event_id, "sender": self.user_id}, m_replace_dict ) + # Directly requesting the edit should not have the edit to the edit applied. + channel = self.make_request( + "GET", + f"/rooms/{self.room}/event/{edit_event_id}", + access_token=self.user_token, + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assertEqual("Wibble", channel.json_body["content"]["body"]) + self.assertIn("m.new_content", channel.json_body["content"]) + + # The relations information should not include the edit to the edit. + self.assertNotIn("m.relations", channel.json_body["unsigned"]) + def test_unknown_relations(self) -> None: """Unknown relations should be accepted.""" channel = self._send_relation("m.relation.test", "m.room.test") @@ -984,6 +997,24 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 7) + def test_annotation_to_annotation(self) -> None: + """Any relation to an annotation should be ignored.""" + channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction", "a") + event_id = channel.json_body["event_id"] + self._send_relation( + RelationTypes.ANNOTATION, "m.reaction", "b", parent_id=event_id + ) + + # Fetch the initial annotation event to see if it has bundled aggregations. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{self.room}/event/{event_id}", + access_token=self.user_token, + ) + self.assertEquals(200, channel.code, channel.json_body) + # The first annotationt should not have any bundled aggregations. + self.assertNotIn("m.relations", channel.json_body["unsigned"]) + def test_reference(self) -> None: """ Test that references get correctly bundled. From 9ae0253f4e9f7aafb64abe3b5f04ad067883a121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 5 May 2022 14:31:25 +0200 Subject: [PATCH 117/263] Use `private` instead of `hidden` in MSC2285 related code. (#12635) --- changelog.d/12635.feature | 1 + synapse/config/experimental.py | 2 +- synapse/handlers/initial_sync.py | 4 ++-- synapse/handlers/receipts.py | 6 +++--- synapse/rest/client/versions.py | 2 +- tests/handlers/test_receipts.py | 32 ++++++++++++++++---------------- tests/rest/client/test_sync.py | 4 ++-- 7 files changed, 26 insertions(+), 25 deletions(-) create mode 100644 changelog.d/12635.feature diff --git a/changelog.d/12635.feature b/changelog.d/12635.feature new file mode 100644 index 000000000000..cd5c45029ee1 --- /dev/null +++ b/changelog.d/12635.feature @@ -0,0 +1 @@ +Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 421ed7481baf..abed5e7edb3a 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -32,7 +32,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC2716 (importing historical messages) self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False) - # MSC2285 (hidden read receipts) + # MSC2285 (private read receipts) self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False) # MSC3244 (room version capabilities) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index a7db8feb57eb..7b94770f9722 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -143,7 +143,7 @@ async def _snapshot_all_rooms( to_key=int(now_token.receipt_key), ) if self.hs.config.experimental.msc2285_enabled: - receipt = ReceiptEventSource.filter_out_hidden(receipt, user_id) + receipt = ReceiptEventSource.filter_out_private(receipt, user_id) tags_by_room = await self.store.get_tags_for_user(user_id) @@ -449,7 +449,7 @@ async def get_receipts() -> List[JsonDict]: if not receipts: return [] if self.hs.config.experimental.msc2285_enabled: - receipts = ReceiptEventSource.filter_out_hidden(receipts, user_id) + receipts = ReceiptEventSource.filter_out_private(receipts, user_id) return receipts presence, receipts, (messages, token) = await make_deferred_yieldable( diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index ae41fd674e13..43d615357b3c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -165,7 +165,7 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config @staticmethod - def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]: + def filter_out_private(events: List[JsonDict], user_id: str) -> List[JsonDict]: """ This method takes in what is returned by get_linearized_receipts_for_rooms() and goes through read receipts @@ -175,7 +175,7 @@ def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]: visible_events = [] - # filter out hidden receipts the user shouldn't see + # filter out private receipts the user shouldn't see for event in events: content = event.get("content", {}) new_event = event.copy() @@ -223,7 +223,7 @@ async def get_new_events( ) if self.config.experimental.msc2285_enabled: - events = ReceiptEventSource.filter_out_hidden(events, user.to_string()) + events = ReceiptEventSource.filter_out_private(events, user.to_string()) return events, to_key diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index bfc1d4ee0829..c1bd775fece4 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -93,7 +93,7 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, # Supports the busy presence state described in MSC3026. "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, - # Supports receiving hidden read receipts as per MSC2285 + # Supports receiving private read receipts as per MSC2285 "org.matrix.msc2285": self.config.experimental.msc2285_enabled, # Adds support for importing historical messages as per MSC2716 "org.matrix.msc2716": self.config.experimental.msc2716_enabled, diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index c12a9120f029..0482a1ea34fb 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -25,8 +25,8 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor, clock, hs): self.event_source = hs.get_event_sources().sources.receipt - def test_filters_out_hidden_receipt(self): - self._test_filters_hidden( + def test_filters_out_private_receipt(self): + self._test_filters_private( [ { "content": { @@ -45,8 +45,8 @@ def test_filters_out_hidden_receipt(self): [], ) - def test_filters_out_hidden_receipt_and_ignores_rest(self): - self._test_filters_hidden( + def test_filters_out_private_receipt_and_ignores_rest(self): + self._test_filters_private( [ { "content": { @@ -84,8 +84,8 @@ def test_filters_out_hidden_receipt_and_ignores_rest(self): ], ) - def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self): - self._test_filters_hidden( + def test_filters_out_event_with_only_private_receipts_and_ignores_the_rest(self): + self._test_filters_private( [ { "content": { @@ -126,7 +126,7 @@ def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self): ) def test_handles_missing_content_of_m_read(self): - self._test_filters_hidden( + self._test_filters_private( [ { "content": { @@ -162,7 +162,7 @@ def test_handles_missing_content_of_m_read(self): ) def test_handles_empty_event(self): - self._test_filters_hidden( + self._test_filters_private( [ { "content": { @@ -196,8 +196,8 @@ def test_handles_empty_event(self): ], ) - def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(self): - self._test_filters_hidden( + def test_filters_out_receipt_event_with_only_private_receipt_and_ignores_rest(self): + self._test_filters_private( [ { "content": { @@ -249,7 +249,7 @@ def test_handles_string_data(self): Context: https://github.com/matrix-org/synapse/issues/10603 """ - self._test_filters_hidden( + self._test_filters_private( [ { "content": { @@ -278,8 +278,8 @@ def test_handles_string_data(self): ], ) - def test_leaves_our_hidden_and_their_public(self): - self._test_filters_hidden( + def test_leaves_our_private_and_their_public(self): + self._test_filters_private( [ { "content": { @@ -332,9 +332,9 @@ def test_leaves_our_hidden_and_their_public(self): ], ) - def _test_filters_hidden( + def _test_filters_private( self, events: List[JsonDict], expected_output: List[JsonDict] ): - """Tests that the _filter_out_hidden returns the expected output""" - filtered_events = self.event_source.filter_out_hidden(events, "@me:server.org") + """Tests that the _filter_out_private returns the expected output""" + filtered_events = self.event_source.filter_out_private(events, "@me:server.org") self.assertEqual(filtered_events, expected_output) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 67c94dd18fbd..010833764957 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -407,7 +407,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) @override_config({"experimental_features": {"msc2285_enabled": True}}) - def test_hidden_read_receipts(self) -> None: + def test_private_read_receipts(self) -> None: # Send a message as the first user res = self.helper.send(self.room_id, body="hello", tok=self.tok) @@ -639,7 +639,7 @@ def test_unread_counts(self) -> None: # Check that the unread counter is back to 0. self._check_unread_count(0) - # Check that hidden read receipts don't break unread counts + # Check that private read receipts don't break unread counts res = self.helper.send(self.room_id, "hello", tok=self.tok2) self._check_unread_count(1) From b8fa24b022df366495147b5ddce73c06ec9cbc3a Mon Sep 17 00:00:00 2001 From: Henry <97804910+henryclw@users.noreply.github.com> Date: Thu, 5 May 2022 05:36:42 -0700 Subject: [PATCH 118/263] Use `docker/metadata-action` to generate docker image tags (#12573) Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Signed-off-by: Henry <97804910+henryclw@users.noreply.github.com> --- .github/workflows/docker.yml | 30 +++++++++++------------------- changelog.d/12573.docker | 1 + 2 files changed, 12 insertions(+), 19 deletions(-) create mode 100644 changelog.d/12573.docker diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 124b17458f45..d20d30c0353c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,32 +34,24 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # TODO: consider using https://github.com/docker/metadata-action instead of this - # custom magic - name: Calculate docker image tag id: set-tag - run: | - case "${GITHUB_REF}" in - refs/heads/develop) - tag=develop - ;; - refs/heads/master|refs/heads/main) - tag=latest - ;; - refs/tags/*) - tag=${GITHUB_REF#refs/tags/} - ;; - *) - tag=${GITHUB_SHA} - ;; - esac - echo "::set-output name=tag::$tag" + uses: docker/metadata-action@master + with: + images: matrixdotorg/synapse + flavor: | + latest=false + tags: | + type=raw,value=develop,enable=${{ github.ref == 'refs/heads/develop' }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=pep440,pattern={{raw}} - name: Build and push all platforms uses: docker/build-push-action@v2 with: push: true labels: "gitsha1=${{ github.sha }}" - tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}" + tags: "${{ steps.set-tag.outputs.tags }}" file: "docker/Dockerfile" platforms: linux/amd64,linux/arm64 diff --git a/changelog.d/12573.docker b/changelog.d/12573.docker new file mode 100644 index 000000000000..5cc8de50acb3 --- /dev/null +++ b/changelog.d/12573.docker @@ -0,0 +1 @@ +Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by henryclw. \ No newline at end of file From 07fa53ec40106b97ba2c12a2dcc5446325e5fb61 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 5 May 2022 13:39:59 +0100 Subject: [PATCH 119/263] Improve comments and error messages around access tokens. (#12577) --- changelog.d/12577.misc | 1 + synapse/api/auth.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 changelog.d/12577.misc diff --git a/changelog.d/12577.misc b/changelog.d/12577.misc new file mode 100644 index 000000000000..8c4c47ad5292 --- /dev/null +++ b/changelog.d/12577.misc @@ -0,0 +1 @@ +Improve comments and error messages around access tokens. \ No newline at end of file diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f6202ef7a5ab..931750668ea2 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -417,7 +417,8 @@ async def get_user_by_access_token( """ if rights == "access": - # first look in the database + # First look in the database to see if the access token is present + # as an opaque token. r = await self.store.get_user_by_access_token(token) if r: valid_until_ms = r.valid_until_ms @@ -434,7 +435,8 @@ async def get_user_by_access_token( return r - # otherwise it needs to be a valid macaroon + # If the token isn't found in the database, then it could still be a + # macaroon, so we check that here. try: user_id, guest = self._parse_and_validate_macaroon(token, rights) @@ -482,8 +484,12 @@ async def get_user_by_access_token( TypeError, ValueError, ) as e: - logger.warning("Invalid macaroon in auth: %s %s", type(e), e) - raise InvalidClientTokenError("Invalid macaroon passed.") + logger.warning( + "Invalid access token in auth: %s %s.", + type(e), + e, + ) + raise InvalidClientTokenError("Invalid access token passed.") def _parse_and_validate_macaroon( self, token: str, rights: str = "access" @@ -504,10 +510,7 @@ def _parse_and_validate_macaroon( try: macaroon = pymacaroons.Macaroon.deserialize(token) except Exception: # deserialize can throw more-or-less anything - # doesn't look like a macaroon: treat it as an opaque token which - # must be in the database. - # TODO: it would be nice to get rid of this, but apparently some - # people use access tokens which aren't macaroons + # The access token doesn't look like a macaroon. raise _InvalidMacaroonException() try: From ef86cf3d281d09d8ff4ee64af7a8c48d2516e2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 5 May 2022 15:25:51 +0200 Subject: [PATCH 120/263] Update `_on_new_receipts()` to work with MSC2285 changes. (#12636) --- changelog.d/12636.feature | 1 + synapse/replication/tcp/client.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12636.feature diff --git a/changelog.d/12636.feature b/changelog.d/12636.feature new file mode 100644 index 000000000000..cd5c45029ee1 --- /dev/null +++ b/changelog.d/12636.feature @@ -0,0 +1 @@ +Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 122892c7bca2..350762f49447 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -21,7 +21,7 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.python.failure import Failure -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, ReceiptTypes from synapse.federation import send_queue from synapse.federation.sender import FederationSender from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -401,10 +401,8 @@ async def _on_new_receipts( # we only want to send on receipts for our own users if not self._is_mine_id(receipt.user_id): continue - if ( - receipt.data.get("hidden", False) - and self._hs.config.experimental.msc2285_enabled - ): + # Private read receipts never get sent over federation. + if receipt.receipt_type == ReceiptTypes.READ_PRIVATE: continue receipt_info = ReadReceipt( receipt.room_id, From e923fc20bda436a894301350ac9ab2b4bb942e18 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 5 May 2022 14:51:15 +0100 Subject: [PATCH 121/263] Include extra dependency groups 'systemd' and 'cache_memory' in debian packages (#12640) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- debian/build_virtualenv | 7 ++++++- debian/changelog | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index b068792592d6..d2955f762845 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -37,7 +37,12 @@ python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip pip install poetry==1.2.0b1 -poetry export --extras all --extras test -o exported_requirements.txt +poetry export \ + --extras all \ + --extras test \ + --extras systemd \ + --extras cache_memory \ + -o exported_requirements.txt deactivate rm -rf "$TEMP_VENV" diff --git a/debian/changelog b/debian/changelog index 53b2387776f5..c828514752e2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium + + * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which + were incorrectly omitted from the 1.58.0 package. + + -- Synapse Packaging team Thu, 05 May 2022 12:26:36 +0100 + matrix-synapse-py3 (1.58.0) stable; urgency=medium * New Synapse release 1.58.0. From 6a17a291a6692bc2a0d48ea052478f4014a19281 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:05:58 +0100 Subject: [PATCH 122/263] 1.58.1 --- CHANGES.md | 11 +++++++++++ debian/changelog | 5 +++-- pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 31f156127424..d221a37fafd6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +Synapse 1.58.1 (2022-05-05) +=========================== + +This patch release includes a fix to the Debian packages, installing the +`systemd` and `cache_memory` extra package groups, which were incorrectly +omitted in v1.58.0. This primarily prevented Synapse from starting +when the `systemd.journal.JournalHandler` log handler was configured. + +No significant changes since 1.58.0. + + Synapse 1.58.0 (2022-05-03) =========================== diff --git a/debian/changelog b/debian/changelog index c828514752e2..5440f91bc0ca 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,10 @@ -matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.58.1) stable; urgency=medium * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which were incorrectly omitted from the 1.58.0 package. + * New Synapse release 1.58.1. - -- Synapse Packaging team Thu, 05 May 2022 12:26:36 +0100 + -- Synapse Packaging team Thu, 05 May 2022 14:58:23 +0100 matrix-synapse-py3 (1.58.0) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index f0f029f0169f..4167abb23363 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.0" +version = "1.58.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From d2784b6567ccfdb08d017d3f83a6d5c1b42cc8f3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:06:39 +0100 Subject: [PATCH 123/263] Minor wording change to v1.58.1 release notes --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d221a37fafd6..c046e96eb1eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ This patch release includes a fix to the Debian packages, installing the omitted in v1.58.0. This primarily prevented Synapse from starting when the `systemd.journal.JournalHandler` log handler was configured. -No significant changes since 1.58.0. +Otherwise, no significant changes since 1.58.0. Synapse 1.58.0 (2022-05-03) From bc149a18f6b5e12792e14c295f294adda39cffc8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:10:24 +0100 Subject: [PATCH 124/263] link to relevant bug report in v1.58.1 changelog --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c046e96eb1eb..07541c803c91 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,8 @@ Synapse 1.58.1 (2022-05-05) This patch release includes a fix to the Debian packages, installing the `systemd` and `cache_memory` extra package groups, which were incorrectly omitted in v1.58.0. This primarily prevented Synapse from starting -when the `systemd.journal.JournalHandler` log handler was configured. +when the `systemd.journal.JournalHandler` log handler was configured +(closes [#12631](https://github.com/matrix-org/synapse/issues/12631)). Otherwise, no significant changes since 1.58.0. From 3a8ee229112697b02b876299869d7511474ecf92 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 May 2022 15:15:32 +0100 Subject: [PATCH 125/263] Update v1.58.1 changelog entry with more familiar language --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 07541c803c91..982f09b1db84 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,8 @@ Synapse 1.58.1 (2022-05-05) This patch release includes a fix to the Debian packages, installing the `systemd` and `cache_memory` extra package groups, which were incorrectly omitted in v1.58.0. This primarily prevented Synapse from starting -when the `systemd.journal.JournalHandler` log handler was configured -(closes [#12631](https://github.com/matrix-org/synapse/issues/12631)). +when the `systemd.journal.JournalHandler` log handler was configured. +See [#12631](https://github.com/matrix-org/synapse/issues/12631) for further information. Otherwise, no significant changes since 1.58.0. From a377a43386a14c8a11fe9e99c821d0471f7982a2 Mon Sep 17 00:00:00 2001 From: "DeepBlueV7.X" Date: Thu, 5 May 2022 14:25:00 +0000 Subject: [PATCH 126/263] Support MSC3266 room summaries over federation (#11507) Signed-off-by: Nicolas Werner --- changelog.d/11507.feature | 1 + synapse/federation/federation_client.py | 2 + synapse/handlers/room_summary.py | 54 ++++++++++++++++++++++--- tests/handlers/test_room_summary.py | 26 ++++++++++++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 changelog.d/11507.feature diff --git a/changelog.d/11507.feature b/changelog.d/11507.feature new file mode 100644 index 000000000000..72c5690cca4a --- /dev/null +++ b/changelog.d/11507.feature @@ -0,0 +1 @@ +Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6a59cb4b713e..b5e0b84cbc69 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1426,6 +1426,8 @@ async def send_request( room = res.get("room") if not isinstance(room, dict): raise InvalidResponseError("'room' must be a dict") + if room.get("room_id") != room_id: + raise InvalidResponseError("wrong room returned in hierarchy response") # Validate children_state of the room. children_state = room.pop("children_state", []) diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 486145f48aca..ff24ec806357 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -105,6 +105,7 @@ def __init__(self, hs: "HomeServer"): hs.get_clock(), "get_room_hierarchy", ) + self._msc3266_enabled = hs.config.experimental.msc3266_enabled async def get_room_hierarchy( self, @@ -630,7 +631,7 @@ async def _is_local_room_accessible( return False async def _is_remote_room_accessible( - self, requester: str, room_id: str, room: JsonDict + self, requester: Optional[str], room_id: str, room: JsonDict ) -> bool: """ Calculate whether the room received over federation should be shown to the requester. @@ -645,7 +646,8 @@ async def _is_remote_room_accessible( due to an invite, etc. Args: - requester: The user requesting the summary. + requester: The user requesting the summary. If not passed only world + readability is checked. room_id: The room ID returned over federation. room: The summary of the room returned over federation. @@ -659,6 +661,8 @@ async def _is_remote_room_accessible( or room.get("world_readable") is True ): return True + elif not requester: + return False # Check if the user is a member of any of the allowed rooms from the response. allowed_rooms = room.get("allowed_room_ids") @@ -715,6 +719,10 @@ async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDic "room_type": create_event.content.get(EventContentFields.ROOM_TYPE), } + if self._msc3266_enabled: + entry["im.nheko.summary.version"] = stats["version"] + entry["im.nheko.summary.encryption"] = stats["encryption"] + # Federation requests need to provide additional information so the # requested server is able to filter the response appropriately. if for_federation: @@ -812,9 +820,45 @@ async def get_room_summary( room_summary["membership"] = membership or "leave" else: - # TODO federation API, descoped from initial unstable implementation - # as MSC needs more maturing on that side. - raise SynapseError(400, "Federation is not currently supported.") + # Reuse the hierarchy query over federation + if remote_room_hosts is None: + raise SynapseError(400, "Missing via to query remote room") + + ( + room_entry, + children_room_entries, + inaccessible_children, + ) = await self._summarize_remote_room_hierarchy( + _RoomQueueEntry(room_id, remote_room_hosts), + suggested_only=True, + ) + + # The results over federation might include rooms that we, as the + # requesting server, are allowed to see, but the requesting user is + # not permitted to see. + # + # Filter the returned results to only what is accessible to the user. + if not room_entry or not await self._is_remote_room_accessible( + requester, room_entry.room_id, room_entry.room + ): + raise NotFoundError("Room not found or is not accessible") + + room = dict(room_entry.room) + room.pop("allowed_room_ids", None) + + # If there was a requester, add their membership. + # We keep the membership in the local membership table unless the + # room is purged even for remote rooms. + if requester: + ( + membership, + _, + ) = await self._store.get_local_current_membership_for_user_in_room( + requester, room_id + ) + room["membership"] = membership or "leave" + + return room return room_summary diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index d37292ce138e..e74eb71774df 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -1092,3 +1092,29 @@ def test_visibility(self): ) result = self.get_success(self.handler.get_room_summary(user2, self.room)) self.assertEqual(result.get("room_id"), self.room) + + def test_fed(self): + """ + Return data over federation and ensure that it is handled properly. + """ + fed_hostname = self.hs.hostname + "2" + fed_room = "#fed_room:" + fed_hostname + + requested_room_entry = _RoomEntry( + fed_room, + {"room_id": fed_room, "world_readable": True}, + ) + + async def summarize_remote_room_hierarchy(_self, room, suggested_only): + return requested_room_entry, {}, set() + + with mock.patch( + "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy", + new=summarize_remote_room_hierarchy, + ): + result = self.get_success( + self.handler.get_room_summary( + self.user, fed_room, remote_room_hosts=[fed_hostname] + ) + ) + self.assertEqual(result.get("room_id"), fed_room) From c2d50e9f6c5f7b01cbd8bf1dca36cb8c0e7b007f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Fri, 6 May 2022 11:43:53 +0100 Subject: [PATCH 127/263] Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. (#12452) --- changelog.d/12452.feature | 1 + docker/configure_workers_and_start.py | 4 +- docs/upgrade.md | 27 +++ docs/workers.md | 20 ++ synapse/app/generic_worker.py | 16 -- synapse/config/appservice.py | 1 - synapse/config/workers.py | 109 +++++++++- synapse/handlers/appservice.py | 2 +- tests/config/test_workers.py | 288 ++++++++++++++++++++++++++ 9 files changed, 447 insertions(+), 21 deletions(-) create mode 100644 changelog.d/12452.feature create mode 100644 tests/config/test_workers.py diff --git a/changelog.d/12452.feature b/changelog.d/12452.feature new file mode 100644 index 000000000000..22f054d34493 --- /dev/null +++ b/changelog.d/12452.feature @@ -0,0 +1 @@ +Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. \ No newline at end of file diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 33fc20d2182e..b2b7938ae801 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -69,10 +69,10 @@ "worker_extra_conf": "enable_media_repo: true", }, "appservice": { - "app": "synapse.app.appservice", + "app": "synapse.app.generic_worker", "listener_resources": [], "endpoint_patterns": [], - "shared_extra_conf": {"notify_appservices": False}, + "shared_extra_conf": {"notify_appservices_from_worker": "appservice"}, "worker_extra_conf": "", }, "federation_sender": { diff --git a/docs/upgrade.md b/docs/upgrade.md index b40cac86f0c4..18c33a4198ab 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -100,6 +100,32 @@ To re-enable this functionality, set the [`allow_device_name_lookup_over_federation`](https://matrix-org.github.io/synapse/v1.59/usage/configuration/config_documentation.html#federation) homeserver config option to `true`. + +## Deprecation of the `synapse.app.appservice` worker application type + +The `synapse.app.appservice` worker application type allowed you to configure a +single worker to use to notify application services of new events, as long +as this functionality was disabled on the main process with `notify_appservices: False`. + +To unify Synapse's worker types, the `synapse.app.appservice` worker application +type and the `notify_appservices` configuration option have been deprecated. + +To get the same functionality, it's now recommended that the `synapse.app.generic_worker` +worker application type is used and that the `notify_appservices_from_worker` option +is set to the name of a worker. + +For the time being, `notify_appservices_from_worker` can be used alongside +`synapse.app.appservice` and `notify_appservices` to make it easier to transition +between the two configurations, however please note that: + +- the options must not contradict each other (otherwise Synapse won't start); and +- the `notify_appservices` option will be removed in a future release of Synapse. + +Please see [the relevant section of the worker documentation][v1_59_notify_ases_from] for more information. + +[v1_59_notify_ases_from]: workers.md#notifying-application-services + + # Upgrading to v1.58.0 ## Groups/communities feature has been disabled by default @@ -107,6 +133,7 @@ homeserver config option to `true`. The non-standard groups/communities feature in Synapse has been disabled by default and will be removed in Synapse v1.61.0. + # Upgrading to v1.57.0 ## Changes to database schema for application services diff --git a/docs/workers.md b/docs/workers.md index afdcd785e408..1d049b6c4f28 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -435,6 +435,23 @@ An example for a dedicated background worker instance: {{#include systemd-with-workers/workers/background_worker.yaml}} ``` +#### Notifying Application Services + +You can designate one worker to send output traffic to Application Services. + +Specify its name in the shared configuration as follows: + +```yaml +notify_appservices_from_worker: worker_name +``` + +This work cannot be load-balanced; please ensure the main process is restarted +after setting this option in the shared configuration! + +This style of configuration supersedes the legacy `synapse.app.appservice` +worker application type. + + ### `synapse.app.pusher` Handles sending push notifications to sygnal and email. Doesn't handle any @@ -453,6 +470,9 @@ pusher_instances: ### `synapse.app.appservice` +**Deprecated as of Synapse v1.58.** [Use `synapse.app.generic_worker` with the +`notify_appservices_from_worker` option instead.](#notifying-application-services) + Handles sending output traffic to Application Services. Doesn't handle any REST endpoints itself, but you should set `notify_appservices: False` in the shared configuration file to stop the main synapse sending appservice notifications. diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 1865c671f41c..07dddc0b1326 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -441,22 +441,6 @@ def start(config_options: List[str]) -> None: "synapse.app.user_dir", ) - if config.worker.worker_app == "synapse.app.appservice": - if config.appservice.notify_appservices: - sys.stderr.write( - "\nThe appservices must be disabled in the main synapse process" - "\nbefore they can be run in a separate worker." - "\nPlease add ``notify_appservices: false`` to the main config" - "\n" - ) - sys.exit(1) - - # Force the appservice to start since they will be disabled in the main config - config.appservice.notify_appservices = True - else: - # For other worker types we force this to off. - config.appservice.notify_appservices = False - if config.worker.worker_app == "synapse.app.user_dir": if config.server.update_user_directory: sys.stderr.write( diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 720b90a28386..b13cc6bb6e4f 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -33,7 +33,6 @@ class AppServiceConfig(Config): def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.app_service_config_files = config.get("app_service_config_files", []) - self.notify_appservices = config.get("notify_appservices", True) self.track_appservice_user_ips = config.get("track_appservice_user_ips", False) def generate_config_section(cls, **kwargs: Any) -> str: diff --git a/synapse/config/workers.py b/synapse/config/workers.py index a5479dfca98b..a9dbcc6d3dc7 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -14,7 +14,8 @@ # limitations under the License. import argparse -from typing import Any, List, Union +import logging +from typing import Any, Dict, List, Union import attr @@ -42,6 +43,13 @@ Please add ``start_pushers: false`` to the main config """ +_DEPRECATED_WORKER_DUTY_OPTION_USED = """ +The '%s' configuration option is deprecated and will be removed in a future +Synapse version. Please use ``%s: name_of_worker`` instead. +""" + +logger = logging.getLogger(__name__) + def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]: """Helper for allowing parsing a string or list of strings to a config @@ -296,6 +304,105 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.worker_name is None and background_tasks_instance == "master" ) or self.worker_name == background_tasks_instance + self.should_notify_appservices = self._should_this_worker_perform_duty( + config, + legacy_master_option_name="notify_appservices", + legacy_worker_app_name="synapse.app.appservice", + new_option_name="notify_appservices_from_worker", + ) + + def _should_this_worker_perform_duty( + self, + config: Dict[str, Any], + legacy_master_option_name: str, + legacy_worker_app_name: str, + new_option_name: str, + ) -> bool: + """ + Figures out whether this worker should perform a certain duty. + + This function is temporary and is only to deal with the complexity + of allowing old, transitional and new configurations all at once. + + Contradictions between the legacy and new part of a transitional configuration + will lead to a ConfigError. + + Parameters: + config: The config dictionary + legacy_master_option_name: The name of a legacy option, whose value is boolean, + specifying whether it's the master that should handle a certain duty. + e.g. "notify_appservices" + legacy_worker_app_name: The name of a legacy Synapse worker application + that would traditionally perform this duty. + e.g. "synapse.app.appservice" + new_option_name: The name of the new option, whose value is the name of a + designated worker to perform the duty. + e.g. "notify_appservices_from_worker" + """ + + # None means 'unspecified'; True means 'run here' and False means + # 'don't run here'. + new_option_should_run_here = None + if new_option_name in config: + designated_worker = config[new_option_name] or "master" + new_option_should_run_here = ( + designated_worker == "master" and self.worker_name is None + ) or designated_worker == self.worker_name + + legacy_option_should_run_here = None + if legacy_master_option_name in config: + run_on_master = bool(config[legacy_master_option_name]) + + legacy_option_should_run_here = ( + self.worker_name is None and run_on_master + ) or (self.worker_app == legacy_worker_app_name and not run_on_master) + + # Suggest using the new option instead. + logger.warning( + _DEPRECATED_WORKER_DUTY_OPTION_USED, + legacy_master_option_name, + new_option_name, + ) + + if self.worker_app == legacy_worker_app_name and config.get( + legacy_master_option_name, True + ): + # As an extra bit of complication, we need to check that the + # specialised worker is only used if the legacy config says the + # master isn't performing the duties. + raise ConfigError( + f"Cannot use deprecated worker app type '{legacy_worker_app_name}' whilst deprecated option '{legacy_master_option_name}' is not set to false.\n" + f"Consider setting `worker_app: synapse.app.generic_worker` and using the '{new_option_name}' option instead.\n" + f"The '{new_option_name}' option replaces '{legacy_master_option_name}'." + ) + + if new_option_should_run_here is None and legacy_option_should_run_here is None: + # Neither option specified; the fallback behaviour is to run on the main process + return self.worker_name is None + + if ( + new_option_should_run_here is not None + and legacy_option_should_run_here is not None + ): + # Both options specified; ensure they match! + if new_option_should_run_here != legacy_option_should_run_here: + update_worker_type = ( + " and set worker_app: synapse.app.generic_worker" + if self.worker_app == legacy_worker_app_name + else "" + ) + # If the values conflict, we suggest the admin removes the legacy option + # for simplicity. + raise ConfigError( + f"Conflicting configuration options: {legacy_master_option_name} (legacy), {new_option_name} (new).\n" + f"Suggestion: remove {legacy_master_option_name}{update_worker_type}.\n" + ) + + # We've already validated that these aren't conflicting; now just see if + # either is True. + # (By this point, these are either the same value or only one is not None.) + return bool(new_option_should_run_here or legacy_option_should_run_here) + def generate_config_section(self, **kwargs: Any) -> str: return """\ ## Workers ## diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index b3894666ccf5..85bd5e47682b 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -59,7 +59,7 @@ def __init__(self, hs: "HomeServer"): self.scheduler = hs.get_application_service_scheduler() self.started_scheduler = False self.clock = hs.get_clock() - self.notify_appservices = hs.config.appservice.notify_appservices + self.notify_appservices = hs.config.worker.should_notify_appservices self.event_sources = hs.get_event_sources() self._msc2409_to_device_messages_enabled = ( hs.config.experimental.msc2409_to_device_messages_enabled diff --git a/tests/config/test_workers.py b/tests/config/test_workers.py new file mode 100644 index 000000000000..da81bb9655fc --- /dev/null +++ b/tests/config/test_workers.py @@ -0,0 +1,288 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Mapping, Optional +from unittest.mock import Mock + +from frozendict import frozendict + +from synapse.config import ConfigError +from synapse.config.workers import WorkerConfig + +from tests.unittest import TestCase + +_EMPTY_FROZENDICT: Mapping[str, Any] = frozendict() + + +class WorkerDutyConfigTestCase(TestCase): + def _make_worker_config( + self, + worker_app: str, + worker_name: Optional[str], + extras: Mapping[str, Any] = _EMPTY_FROZENDICT, + ) -> WorkerConfig: + root_config = Mock() + root_config.worker_app = worker_app + root_config.worker_name = worker_name + worker_config = WorkerConfig(root_config) + worker_config_dict = { + "worker_name": worker_name, + "worker_app": worker_app, + **extras, + } + worker_config.read_config(worker_config_dict) + return worker_config + + def test_old_configs_master(self) -> None: + """ + Tests old (legacy) config options. This is for the master's config. + """ + main_process_config = self._make_worker_config( + worker_app="synapse.app.homeserver", worker_name=None + ) + + self.assertTrue( + main_process_config._should_this_worker_perform_duty( + {}, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + self.assertTrue( + main_process_config._should_this_worker_perform_duty( + { + "notify_appservices": True, + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + self.assertFalse( + main_process_config._should_this_worker_perform_duty( + { + "notify_appservices": False, + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + def test_old_configs_appservice_worker(self) -> None: + """ + Tests old (legacy) config options. This is for the worker's config. + """ + appservice_worker_config = self._make_worker_config( + worker_app="synapse.app.appservice", + worker_name="worker1", + extras={ + # Set notify_appservices to false for the initialiser's config, + # so that it doesn't raise an exception here. + # (This is not read by `_should_this_worker_perform_duty`.) + "notify_appservices": False, + }, + ) + + with self.assertRaises(ConfigError): + # This raises because you need to set notify_appservices: False + # before using the synapse.app.appservice worker type + self.assertFalse( + appservice_worker_config._should_this_worker_perform_duty( + {}, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + with self.assertRaises(ConfigError): + # This also raises because you need to set notify_appservices: False + # before using the synapse.app.appservice worker type + appservice_worker_config._should_this_worker_perform_duty( + { + "notify_appservices": True, + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + + self.assertTrue( + appservice_worker_config._should_this_worker_perform_duty( + { + "notify_appservices": False, + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + def test_transitional_configs_master(self) -> None: + """ + Tests transitional (legacy + new) config options. This is for the master's config. + """ + + main_process_config = self._make_worker_config( + worker_app="synapse.app.homeserver", worker_name=None + ) + + self.assertTrue( + main_process_config._should_this_worker_perform_duty( + { + "notify_appservices": True, + "notify_appservices_from_worker": "master", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + self.assertFalse( + main_process_config._should_this_worker_perform_duty( + { + "notify_appservices": False, + "notify_appservices_from_worker": "worker1", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + with self.assertRaises(ConfigError): + # Contradictory because we say the master should notify appservices, + # then we say worker1 is the designated worker to do that! + main_process_config._should_this_worker_perform_duty( + { + "notify_appservices": True, + "notify_appservices_from_worker": "worker1", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + + with self.assertRaises(ConfigError): + # Contradictory because we say the master shouldn't notify appservices, + # then we say master is the designated worker to do that! + main_process_config._should_this_worker_perform_duty( + { + "notify_appservices": False, + "notify_appservices_from_worker": "master", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + + def test_transitional_configs_appservice_worker(self) -> None: + """ + Tests transitional (legacy + new) config options. This is for the worker's config. + """ + appservice_worker_config = self._make_worker_config( + worker_app="synapse.app.appservice", + worker_name="worker1", + extras={ + # Set notify_appservices to false for the initialiser's config, + # so that it doesn't raise an exception here. + # (This is not read by `_should_this_worker_perform_duty`.) + "notify_appservices": False, + }, + ) + + self.assertTrue( + appservice_worker_config._should_this_worker_perform_duty( + { + "notify_appservices": False, + "notify_appservices_from_worker": "worker1", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + with self.assertRaises(ConfigError): + # This raises because this worker is the appservice app type, yet + # another worker is the designated worker! + appservice_worker_config._should_this_worker_perform_duty( + { + "notify_appservices": False, + "notify_appservices_from_worker": "worker2", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + + def test_new_configs_master(self) -> None: + """ + Tests new config options. This is for the master's config. + """ + main_process_config = self._make_worker_config( + worker_app="synapse.app.homeserver", worker_name=None + ) + + self.assertTrue( + main_process_config._should_this_worker_perform_duty( + {"notify_appservices_from_worker": None}, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + self.assertFalse( + main_process_config._should_this_worker_perform_duty( + {"notify_appservices_from_worker": "worker1"}, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + def test_new_configs_appservice_worker(self) -> None: + """ + Tests new config options. This is for the worker's config. + """ + appservice_worker_config = self._make_worker_config( + worker_app="synapse.app.generic_worker", worker_name="worker1" + ) + + self.assertTrue( + appservice_worker_config._should_this_worker_perform_duty( + { + "notify_appservices_from_worker": "worker1", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) + + self.assertFalse( + appservice_worker_config._should_this_worker_perform_duty( + { + "notify_appservices_from_worker": "worker2", + }, + "notify_appservices", + "synapse.app.appservice", + "notify_appservices_from_worker", + ) + ) From 2607b3e1816341b3b8534077bd5d3a4daf3a3d15 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 13:35:20 +0100 Subject: [PATCH 128/263] Update mypy to 0.950 and fix complaints (#12650) --- changelog.d/12650.misc | 1 + poetry.lock | 64 ++++++++++--------- pyproject.toml | 4 +- stubs/sortedcontainers/sorteddict.pyi | 19 ++++-- synapse/appservice/api.py | 3 +- synapse/config/appservice.py | 3 +- synapse/events/presence_router.py | 4 +- synapse/handlers/message.py | 3 +- synapse/metrics/background_process_metrics.py | 44 ++++++++++--- tests/storage/test_monthly_active_users.py | 10 +-- 10 files changed, 98 insertions(+), 57 deletions(-) create mode 100644 changelog.d/12650.misc diff --git a/changelog.d/12650.misc b/changelog.d/12650.misc new file mode 100644 index 000000000000..07bb4ce5a91a --- /dev/null +++ b/changelog.d/12650.misc @@ -0,0 +1 @@ +Update to mypy 0.950. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 89e78576b7b6..564ba7ec02c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -572,7 +572,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.931" +version = "0.950" description = "Optional static typing for Python" category = "dev" optional = false @@ -580,13 +580,14 @@ python-versions = ">=3.6" [package.dependencies] mypy-extensions = ">=0.4.3" -tomli = ">=1.1.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} typing-extensions = ">=3.10" [package.extras] dmypy = ["psutil (>=4.0)"] python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] [[package]] name = "mypy-extensions" @@ -598,14 +599,14 @@ python-versions = "*" [[package]] name = "mypy-zope" -version = "0.3.5" +version = "0.3.7" description = "Plugin for mypy to support zope interfaces" category = "dev" optional = false python-versions = "*" [package.dependencies] -mypy = "0.931" +mypy = "0.950" "zope.interface" = "*" "zope.schema" = "*" @@ -1020,7 +1021,7 @@ jeepney = ">=0.6" [[package]] name = "sentry-sdk" -version = "1.5.7" +version = "1.5.11" description = "Python client for Sentry (https://sentry.io)" category = "main" optional = true @@ -1562,7 +1563,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "2bda1a7cfc8cc02832b4a7d16bf7e1615cb05e0639bdb30688aadf692d851942" +content-hash = "f24699464828ac1a63f1034b4a18c841ef585737b9a802fd8311836444f1d702" [metadata.files] attrs = [ @@ -2089,34 +2090,37 @@ msgpack = [ {file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"}, ] mypy = [ - {file = "mypy-0.931-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a"}, - {file = "mypy-0.931-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00"}, - {file = "mypy-0.931-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714"}, - {file = "mypy-0.931-cp310-cp310-win_amd64.whl", hash = "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc"}, - {file = "mypy-0.931-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d"}, - {file = "mypy-0.931-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d"}, - {file = "mypy-0.931-cp36-cp36m-win_amd64.whl", hash = "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c"}, - {file = "mypy-0.931-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0"}, - {file = "mypy-0.931-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05"}, - {file = "mypy-0.931-cp37-cp37m-win_amd64.whl", hash = "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7"}, - {file = "mypy-0.931-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0"}, - {file = "mypy-0.931-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069"}, - {file = "mypy-0.931-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799"}, - {file = "mypy-0.931-cp38-cp38-win_amd64.whl", hash = "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a"}, - {file = "mypy-0.931-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166"}, - {file = "mypy-0.931-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266"}, - {file = "mypy-0.931-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd"}, - {file = "mypy-0.931-cp39-cp39-win_amd64.whl", hash = "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697"}, - {file = "mypy-0.931-py3-none-any.whl", hash = "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d"}, - {file = "mypy-0.931.tar.gz", hash = "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, + {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, + {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, + {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, + {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, + {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, + {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, + {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, + {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, + {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, + {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, + {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, + {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, + {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, + {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, + {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, + {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, + {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] mypy-zope = [ - {file = "mypy-zope-0.3.5.tar.gz", hash = "sha256:489e7da1c2af887f2cfe3496995fc247f296512b495b57817edddda9d22308f3"}, - {file = "mypy_zope-0.3.5-py3-none-any.whl", hash = "sha256:3bd0cc9a3e5933b02931af4b214ba32a4f4ff98adb30c979ce733857db91a18b"}, + {file = "mypy-zope-0.3.7.tar.gz", hash = "sha256:9da171e78e8ef7ac8922c86af1a62f1b7f3244f121020bd94a2246bc3f33c605"}, + {file = "mypy_zope-0.3.7-py3-none-any.whl", hash = "sha256:9c7637d066e4d1bafa0651abc091c752009769098043b236446e6725be2bc9c2"}, ] netaddr = [ {file = "netaddr-0.8.0-py2.py3-none-any.whl", hash = "sha256:9666d0232c32d2656e5e5f8d735f58fd6c7457ce52fc21c98d45f2af78f990ac"}, @@ -2386,8 +2390,8 @@ secretstorage = [ {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, ] sentry-sdk = [ - {file = "sentry-sdk-1.5.7.tar.gz", hash = "sha256:aa52da941c56b5a76fd838f8e9e92a850bf893a9eb1e33ffce6c21431d07ee30"}, - {file = "sentry_sdk-1.5.7-py2.py3-none-any.whl", hash = "sha256:411a8495bd18cf13038e5749e4710beb4efa53da6351f67b4c2f307c2d9b6d49"}, + {file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, + {file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, ] service-identity = [ {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"}, diff --git a/pyproject.toml b/pyproject.toml index 446895711b01..877f19708d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,8 +248,8 @@ flake8-bugbear = "==21.3.2" flake8 = "*" # Typechecking -mypy = "==0.931" -mypy-zope = "==0.3.5" +mypy = "*" +mypy-zope = "*" types-bleach = ">=4.1.0" types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 3a4f9c307685..7c399ab38d5e 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -85,12 +85,19 @@ class SortedDict(Dict[_KT, _VT]): def popitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def peekitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def setdefault(self, key: _KT, default: Optional[_VT] = ...) -> _VT: ... - @overload - def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... - @overload - def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... - @overload - def update(self, **kwargs: _VT) -> None: ... + # Mypy now reports the first overload as an error, because typeshed widened the type + # of `__map` to its internal `_typeshed.SupportsKeysAndGetItem` type in + # https://github.com/python/typeshed/pull/6653 + # Since sorteddicts don't change the signature of `update` from that of `dict`, we + # let the stubs for `update` inherit from the stubs for `dict`. (I suspect we could + # do the same for many othe methods.) We leave the stubs commented to better track + # how this file has evolved from the original stubs. + # @overload + # def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... + # @overload + # def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... + # @overload + # def update(self, **kwargs: _VT) -> None: ... def __reduce__( self, ) -> Tuple[ diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index adc6b074dae6..d19f8dd996b2 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple from prometheus_client import Counter +from typing_extensions import TypeGuard from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException @@ -66,7 +67,7 @@ def _is_valid_3pe_metadata(info: JsonDict) -> bool: return True -def _is_valid_3pe_result(r: JsonDict, field: str) -> bool: +def _is_valid_3pe_result(r: object, field: str) -> TypeGuard[JsonDict]: if not isinstance(r, dict): return False diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index b13cc6bb6e4f..24498e794433 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -55,7 +55,8 @@ def load_appservices( ) -> List[ApplicationService]: """Returns a list of Application Services from the config files.""" if not isinstance(config_files, list): - logger.warning("Expected %s to be a list of AS config files.", config_files) + # type-ignore: this function gets arbitrary json value; we do use this path. + logger.warning("Expected %s to be a list of AS config files.", config_files) # type: ignore[unreachable] return [] # Dicts of value -> filename diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index a58f313e8b1c..98555c8c0c65 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -147,7 +147,9 @@ async def get_users_for_states( # run all the callbacks for get_users_for_states and combine the results for callback in self._get_users_for_states_callbacks: try: - result = await callback(state_updates) + # Note: result is an object here, because we don't trust modules to + # return the types they're supposed to. + result: object = await callback(state_updates) except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 95a89ac01f4b..c28b792e6fe2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1427,7 +1427,7 @@ async def persist_and_notify_client_event( # Validate a newly added alias or newly added alt_aliases. original_alias = None - original_alt_aliases: List[str] = [] + original_alt_aliases: object = [] original_event_id = event.unsigned.get("replaces_state") if original_event_id: @@ -1455,6 +1455,7 @@ async def persist_and_notify_client_event( # If the old version of alt_aliases is of an unknown form, # completely replace it. if not isinstance(original_alt_aliases, (list, tuple)): + # TODO: check that the original_alt_aliases' entries are all strings original_alt_aliases = [] # Check that each alias is currently valid. diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index f61396bb79a9..298809742a94 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -28,11 +28,11 @@ Type, TypeVar, Union, - cast, ) from prometheus_client import Metric from prometheus_client.core import REGISTRY, Counter, Gauge +from typing_extensions import ParamSpec from twisted.internet import defer @@ -256,24 +256,48 @@ async def run() -> Optional[R]: return defer.ensureDeferred(run()) -F = TypeVar("F", bound=Callable[..., Awaitable[Optional[Any]]]) +P = ParamSpec("P") -def wrap_as_background_process(desc: str) -> Callable[[F], F]: - """Decorator that wraps a function that gets called as a background - process. +def wrap_as_background_process( + desc: str, +) -> Callable[ + [Callable[P, Awaitable[Optional[R]]]], + Callable[P, "defer.Deferred[Optional[R]]"], +]: + """Decorator that wraps an asynchronous function `func`, returning a synchronous + decorated function. Calling the decorated version runs `func` as a background + process, forwarding all arguments verbatim. + + That is, + + @wrap_as_background_process + def func(*args): ... + func(1, 2, third=3) + + is equivalent to: + + def func(*args): ... + run_as_background_process(func, 1, 2, third=3) - Equivalent to calling the function with `run_as_background_process` + The former can be convenient if `func` needs to be run as a background process in + multiple places. """ - def wrap_as_background_process_inner(func: F) -> F: + def wrap_as_background_process_inner( + func: Callable[P, Awaitable[Optional[R]]] + ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) def wrap_as_background_process_inner_2( - *args: Any, **kwargs: Any + *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - return run_as_background_process(desc, func, *args, **kwargs) + # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. + # Argument 4 to "run_as_background_process" has incompatible type + # "**P.kwargs"; expected "bool" + # See https://github.com/python/mypy/issues/8862 + return run_as_background_process(desc, func, *args, **kwargs) # type: ignore[arg-type] - return cast(F, wrap_as_background_process_inner_2) + return wrap_as_background_process_inner_2 return wrap_as_background_process_inner diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 0fbf46567091..4c29ad79b643 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -222,10 +222,11 @@ def test_reap_monthly_active_users_reserved_users(self): self.store.user_add_threepid(user, "email", email, now, now) ) - d = self.store.db_pool.runInteraction( - "initialise", self.store._initialise_reserved_users, threepids + self.get_success( + self.store.db_pool.runInteraction( + "initialise", self.store._initialise_reserved_users, threepids + ) ) - self.get_success(d) count = self.get_success(self.store.get_monthly_active_count()) self.assertEqual(count, initial_users) @@ -233,8 +234,7 @@ def test_reap_monthly_active_users_reserved_users(self): users = self.get_success(self.store.get_registered_reserved_users()) self.assertEqual(len(users), reserved_user_number) - d = self.store.reap_monthly_active_users() - self.get_success(d) + self.get_success(self.store.reap_monthly_active_users()) count = self.get_success(self.store.get_monthly_active_count()) self.assertEqual(count, self.hs.config.server.max_mau_value) From 4337d33a73988763e27eb9450307084f8eab6f16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2022 17:41:57 +0100 Subject: [PATCH 129/263] Prevent memory leak from reoccurring when presence is disabled. (#12656) --- changelog.d/12656.misc | 1 + synapse/handlers/presence.py | 42 +++++++++++++++++++++++------------- synapse/util/wheel_timer.py | 39 ++++++++++++++++++++++----------- 3 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 changelog.d/12656.misc diff --git a/changelog.d/12656.misc b/changelog.d/12656.misc new file mode 100644 index 000000000000..8a8743e614f3 --- /dev/null +++ b/changelog.d/12656.misc @@ -0,0 +1 @@ +Prevent memory leak from reoccurring when presence is disabled. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index d078162c2938..268481ec1963 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -659,27 +659,28 @@ def __init__(self, hs: "HomeServer"): ) now = self.clock.time_msec() - for state in self.user_to_current_state.values(): - self.wheel_timer.insert( - now=now, obj=state.user_id, then=state.last_active_ts + IDLE_TIMER - ) - self.wheel_timer.insert( - now=now, - obj=state.user_id, - then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, - ) - if self.is_mine_id(state.user_id): + if self._presence_enabled: + for state in self.user_to_current_state.values(): self.wheel_timer.insert( - now=now, - obj=state.user_id, - then=state.last_federation_update_ts + FEDERATION_PING_INTERVAL, + now=now, obj=state.user_id, then=state.last_active_ts + IDLE_TIMER ) - else: self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_federation_update_ts + FEDERATION_TIMEOUT, + then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, ) + if self.is_mine_id(state.user_id): + self.wheel_timer.insert( + now=now, + obj=state.user_id, + then=state.last_federation_update_ts + FEDERATION_PING_INTERVAL, + ) + else: + self.wheel_timer.insert( + now=now, + obj=state.user_id, + then=state.last_federation_update_ts + FEDERATION_TIMEOUT, + ) # Set of users who have presence in the `user_to_current_state` that # have not yet been persisted @@ -804,6 +805,13 @@ async def _update_states( This is currently used to bump the max presence stream ID without changing any user's presence (see PresenceHandler.add_users_to_send_full_presence_to). """ + if not self._presence_enabled: + # We shouldn't get here if presence is disabled, but we check anyway + # to ensure that we don't a) send out presence federation and b) + # don't add things to the wheel timer that will never be handled. + logger.warning("Tried to update presence states when presence is disabled") + return + now = self.clock.time_msec() with Measure(self.clock, "presence_update_states"): @@ -1229,6 +1237,10 @@ async def set_state( ): raise SynapseError(400, "Invalid presence state") + # If presence is disabled, no-op + if not self.hs.config.server.use_presence: + return + user_id = target_user.to_string() prev_state = await self.current_state_for_user(user_id) diff --git a/synapse/util/wheel_timer.py b/synapse/util/wheel_timer.py index e108adc4604f..177e198e7e75 100644 --- a/synapse/util/wheel_timer.py +++ b/synapse/util/wheel_timer.py @@ -11,17 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Generic, List, TypeVar +import logging +from typing import Generic, Hashable, List, Set, TypeVar -T = TypeVar("T") +import attr +logger = logging.getLogger(__name__) + +T = TypeVar("T", bound=Hashable) -class _Entry(Generic[T]): - __slots__ = ["end_key", "queue"] - def __init__(self, end_key: int) -> None: - self.end_key: int = end_key - self.queue: List[T] = [] +@attr.s(slots=True, frozen=True, auto_attribs=True) +class _Entry(Generic[T]): + end_key: int + elements: Set[T] = attr.Factory(set) class WheelTimer(Generic[T]): @@ -48,17 +51,27 @@ def insert(self, now: int, obj: T, then: int) -> None: then: When to return the object strictly after. """ then_key = int(then / self.bucket_size) + 1 + now_key = int(now / self.bucket_size) if self.entries: min_key = self.entries[0].end_key max_key = self.entries[-1].end_key + if min_key < now_key - 10: + # If we have ten buckets that are due and still nothing has + # called `fetch()` then we likely have a bug that is causing a + # memory leak. + logger.warning( + "Inserting into a wheel timer that hasn't been read from recently. Item: %s", + obj, + ) + if then_key <= max_key: # The max here is to protect against inserts for times in the past - self.entries[max(min_key, then_key) - min_key].queue.append(obj) + self.entries[max(min_key, then_key) - min_key].elements.add(obj) return - next_key = int(now / self.bucket_size) + 1 + next_key = now_key + 1 if self.entries: last_key = self.entries[-1].end_key else: @@ -71,7 +84,7 @@ def insert(self, now: int, obj: T, then: int) -> None: # to insert. This ensures there are no gaps. self.entries.extend(_Entry(key) for key in range(last_key, then_key + 1)) - self.entries[-1].queue.append(obj) + self.entries[-1].elements.add(obj) def fetch(self, now: int) -> List[T]: """Fetch any objects that have timed out @@ -84,11 +97,11 @@ def fetch(self, now: int) -> List[T]: """ now_key = int(now / self.bucket_size) - ret = [] + ret: List[T] = [] while self.entries and self.entries[0].end_key <= now_key: - ret.extend(self.entries.pop(0).queue) + ret.extend(self.entries.pop(0).elements) return ret def __len__(self) -> int: - return sum(len(entry.queue) for entry in self.entries) + return sum(len(entry.elements) for entry in self.entries) From 051a1c3f220938a0ea1a5b328c268bdb3d1ad592 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Sat, 7 May 2022 13:37:29 +0100 Subject: [PATCH 130/263] Convert stringy power levels to integers on room upgrade (#12657) --- changelog.d/12657.bugfix | 1 + synapse/events/utils.py | 61 +++++++++++++++++++------- synapse/handlers/room.py | 14 +++--- tests/events/test_utils.py | 41 ++++++++++++++++- tests/rest/client/test_upgrade_room.py | 44 +++++++++++++++++++ 5 files changed, 137 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12657.bugfix diff --git a/changelog.d/12657.bugfix b/changelog.d/12657.bugfix new file mode 100644 index 000000000000..7547ca40a724 --- /dev/null +++ b/changelog.d/12657.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 026dcde8d83f..ac91c5eb57d0 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -22,6 +22,7 @@ Iterable, List, Mapping, + MutableMapping, Optional, Union, ) @@ -580,10 +581,20 @@ def serialize_events( ] -def copy_power_levels_contents( - old_power_levels: Mapping[str, Union[int, Mapping[str, int]]] +_PowerLevel = Union[str, int] + + +def copy_and_fixup_power_levels_contents( + old_power_levels: Mapping[str, Union[_PowerLevel, Mapping[str, _PowerLevel]]] ) -> Dict[str, Union[int, Dict[str, int]]]: - """Copy the content of a power_levels event, unfreezing frozendicts along the way + """Copy the content of a power_levels event, unfreezing frozendicts along the way. + + We accept as input power level values which are strings, provided they represent an + integer, e.g. `"`100"` instead of 100. Such strings are converted to integers + in the returned dictionary (hence "fixup" in the function name). + + Note that future room versions will outlaw such stringy power levels (see + https://github.com/matrix-org/matrix-spec/issues/853). Raises: TypeError if the input does not look like a valid power levels event content @@ -592,29 +603,47 @@ def copy_power_levels_contents( raise TypeError("Not a valid power-levels content: %r" % (old_power_levels,)) power_levels: Dict[str, Union[int, Dict[str, int]]] = {} - for k, v in old_power_levels.items(): - - if isinstance(v, int): - power_levels[k] = v - continue + for k, v in old_power_levels.items(): if isinstance(v, collections.abc.Mapping): h: Dict[str, int] = {} power_levels[k] = h for k1, v1 in v.items(): - # we should only have one level of nesting - if not isinstance(v1, int): - raise TypeError( - "Invalid power_levels value for %s.%s: %r" % (k, k1, v1) - ) - h[k1] = v1 - continue + _copy_power_level_value_as_integer(v1, h, k1) - raise TypeError("Invalid power_levels value for %s: %r" % (k, v)) + else: + _copy_power_level_value_as_integer(v, power_levels, k) return power_levels +def _copy_power_level_value_as_integer( + old_value: object, + power_levels: MutableMapping[str, Any], + key: str, +) -> None: + """Set `power_levels[key]` to the integer represented by `old_value`. + + :raises TypeError: if `old_value` is not an integer, nor a base-10 string + representation of an integer. + """ + if isinstance(old_value, int): + power_levels[key] = old_value + return + + if isinstance(old_value, str): + try: + parsed_value = int(old_value, base=10) + except ValueError: + # Fall through to the final TypeError. + pass + else: + power_levels[key] = parsed_value + return + + raise TypeError(f"Invalid power_levels value for {key}: {old_value}") + + def validate_canonicaljson(value: Any) -> None: """ Ensure that the JSON object is valid according to the rules of canonical JSON. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b31f00b517a9..604eb6ec154a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -57,7 +57,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.event_auth import validate_event_for_room_version from synapse.events import EventBase -from synapse.events.utils import copy_power_levels_contents +from synapse.events.utils import copy_and_fixup_power_levels_contents from synapse.federation.federation_client import InvalidResponseError from synapse.handlers.federation import get_domains_from_state from synapse.handlers.relations import BundledAggregations @@ -337,13 +337,13 @@ async def _update_upgraded_room_pls( # 50, but if the default PL in a room is 50 or more, then we set the # required PL above that. - pl_content = dict(old_room_pl_state.content) - users_default = int(pl_content.get("users_default", 0)) + pl_content = copy_and_fixup_power_levels_contents(old_room_pl_state.content) + users_default: int = pl_content.get("users_default", 0) # type: ignore[assignment] restricted_level = max(users_default + 1, 50) updated = False for v in ("invite", "events_default"): - current = int(pl_content.get(v, 0)) + current: int = pl_content.get(v, 0) # type: ignore[assignment] if current < restricted_level: logger.debug( "Setting level for %s in %s to %i (was %i)", @@ -380,7 +380,9 @@ async def _update_upgraded_room_pls( "state_key": "", "room_id": new_room_id, "sender": requester.user.to_string(), - "content": old_room_pl_state.content, + "content": copy_and_fixup_power_levels_contents( + old_room_pl_state.content + ), }, ratelimit=False, ) @@ -471,7 +473,7 @@ async def clone_existing_room( # dict so we can't just copy.deepcopy it. initial_state[ (EventTypes.PowerLevels, "") - ] = power_levels = copy_power_levels_contents( + ] = power_levels = copy_and_fixup_power_levels_contents( initial_state[(EventTypes.PowerLevels, "")] ) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 00ad19e446db..b1c47efac7c0 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -17,7 +17,7 @@ from synapse.events import make_event_from_dict from synapse.events.utils import ( SerializeEventConfig, - copy_power_levels_contents, + copy_and_fixup_power_levels_contents, prune_event, serialize_event, ) @@ -529,7 +529,7 @@ def setUp(self) -> None: } def _test(self, input): - a = copy_power_levels_contents(input) + a = copy_and_fixup_power_levels_contents(input) self.assertEqual(a["ban"], 50) self.assertEqual(a["events"]["m.room.name"], 100) @@ -547,3 +547,40 @@ def test_unfrozen(self): def test_frozen(self): input = freeze(self.test_content) self._test(input) + + def test_stringy_integers(self): + """String representations of decimal integers are converted to integers.""" + input = { + "a": "100", + "b": { + "foo": 99, + "bar": "-98", + }, + "d": "0999", + } + output = copy_and_fixup_power_levels_contents(input) + expected_output = { + "a": 100, + "b": { + "foo": 99, + "bar": -98, + }, + "d": 999, + } + + self.assertEqual(output, expected_output) + + def test_strings_that_dont_represent_decimal_integers(self) -> None: + """Should raise when given inputs `s` for which `int(s, base=10)` raises.""" + for invalid_string in ["0x123", "123.0", "123.45", "hello", "0b1", "0o777"]: + with self.assertRaises(TypeError): + copy_and_fixup_power_levels_contents({"a": invalid_string}) + + def test_invalid_types_raise_type_error(self) -> None: + with self.assertRaises(TypeError): + copy_and_fixup_power_levels_contents({"a": ["hello", "grandma"]}) # type: ignore[arg-type] + copy_and_fixup_power_levels_contents({"a": None}) # type: ignore[arg-type] + + def test_invalid_nesting_raises_type_error(self) -> None: + with self.assertRaises(TypeError): + copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index b7d0f42daf5f..c86fc5df0d08 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import Optional +from unittest.mock import patch from twisted.test.proto_helpers import MemoryReactor @@ -167,6 +168,49 @@ def test_power_levels_tombstone(self) -> None: ) self.assertNotIn(self.other, power_levels["users"]) + def test_stringy_power_levels(self) -> None: + """The room upgrade converts stringy power levels to proper integers.""" + # Retrieve the room's current power levels. + power_levels = self.helper.get_state( + self.room_id, + "m.room.power_levels", + tok=self.creator_token, + ) + + # Set creator's power level to the string "100" instead of the integer `100`. + power_levels["users"][self.creator] = "100" + + # Synapse refuses to accept new stringy power level events. Bypass this by + # neutering the validation. + with patch("synapse.events.validator.jsonschema.validate"): + # Note: https://github.com/matrix-org/matrix-spec/issues/853 plans to forbid + # string power levels in new rooms. For this test to have a clean + # conscience, we ought to ensure it's upgrading from a sufficiently old + # version of room. + self.helper.send_state( + self.room_id, + "m.room.power_levels", + body=power_levels, + tok=self.creator_token, + ) + + # Upgrade the room. Check the homeserver reports success. + channel = self._upgrade_room() + self.assertEqual(200, channel.code, channel.result) + + # Extract the new room ID. + new_room_id = channel.json_body["replacement_room"] + + # Fetch the new room's power level event. + new_power_levels = self.helper.get_state( + new_room_id, + "m.room.power_levels", + tok=self.creator_token, + ) + + # We should now have an integer power level. + self.assertEqual(new_power_levels["users"][self.creator], 100, new_power_levels) + def test_space(self) -> None: """Test upgrading a space.""" From 0ce2201932488e98cab8a7d81788e5bcf8f8dd5e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Sat, 7 May 2022 13:40:58 +0100 Subject: [PATCH 131/263] Move `pympler` back into the `all` extras (#12652) * Move `pympler` back into the `all` extras Undoes a change I made in #12381. I can't fully remember my reasoning, but this changed the contents of the debian packages in a backwards incompatible way. We're not aware of anyone who's been bitten by this, but we still want to fix it. To the reviewer: please be convinced that the debian packages will still contain pympler after this change. * Debian changelog entry to keep the linter happy --- changelog.d/12652.misc | 1 + debian/build_virtualenv | 1 - debian/changelog | 8 ++++++++ poetry.lock | 4 ++-- pyproject.toml | 7 ++++--- 5 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12652.misc diff --git a/changelog.d/12652.misc b/changelog.d/12652.misc new file mode 100644 index 000000000000..7b7c1cf5ff2a --- /dev/null +++ b/changelog.d/12652.misc @@ -0,0 +1 @@ +Move `pympler` back in to the `all` extras. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index d2955f762845..f1ec60916325 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -41,7 +41,6 @@ poetry export \ --extras all \ --extras test \ --extras systemd \ - --extras cache_memory \ -o exported_requirements.txt deactivate rm -rf "$TEMP_VENV" diff --git a/debian/changelog b/debian/changelog index 5440f91bc0ca..5b21e0d369e0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +matrix-synapse-py3 (1.58.2) UNRELEASED; urgency=medium + + * Adjust how the `exported-requirements.txt` file is generated as part of + the process of building these packages. This affects the package + maintainers only; end-users are unaffected. + + -- Synapse Packaging team Fri, 06 May 2022 13:49:29 +0100 + matrix-synapse-py3 (1.58.1) stable; urgency=medium * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which diff --git a/poetry.lock b/poetry.lock index 564ba7ec02c4..ddafaaeba066 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1546,7 +1546,7 @@ docs = ["sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"] +all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis", "Pympler"] cache_memory = ["Pympler"] jwt = ["pyjwt"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] @@ -1563,7 +1563,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "f24699464828ac1a63f1034b4a18c841ef585737b9a802fd8311836444f1d702" +content-hash = "eebc9e1d720e2e866f5fddda98ce83d858949a6fdbe30c7e5aef4cf9d17be498" [metadata.files] attrs = [ diff --git a/pyproject.toml b/pyproject.toml index 877f19708d37..7348230fbab5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -231,10 +231,11 @@ all = [ "jaeger-client", "opentracing", # jwt "pyjwt", - #redis - "txredisapi", "hiredis" + # redis + "txredisapi", "hiredis", + # cache_memory + "pympler", # omitted: - # - cache_memory: this is an experimental option # - test: it's useful to have this separate from dev deps in the olddeps job # - systemd: this is a system-based requirement ] From 26c1ad71c555329dbef5f7989ecaf54448321db6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 9 May 2022 10:28:38 +0100 Subject: [PATCH 132/263] Use `Concatenate` to annotate `do_execute` (#12666) --- changelog.d/12666.misc | 1 + pyproject.toml | 2 +- synapse/storage/database.py | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12666.misc diff --git a/changelog.d/12666.misc b/changelog.d/12666.misc new file mode 100644 index 000000000000..96268e33f56f --- /dev/null +++ b/changelog.d/12666.misc @@ -0,0 +1 @@ +Use `Concatenate` to better annotate `_do_execute`. diff --git a/pyproject.toml b/pyproject.toml index 7348230fbab5..4c51b8c4a112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,7 +142,7 @@ netaddr = ">=0.7.18" # add a lower bound to the Jinja2 dependency. Jinja2 = ">=3.0" bleach = ">=1.4.3" -# We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0. +# We use `ParamSpec` and `Concatenate`, which were added in `typing-extensions` 3.10.0.0. typing-extensions = ">=3.10.0" # We enforce that we have a `cryptography` version that bundles an `openssl` # with the latest security patches. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index df1e9c1b831d..2255e55f6f87 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -38,7 +38,7 @@ import attr from prometheus_client import Histogram -from typing_extensions import Literal +from typing_extensions import Concatenate, Literal, ParamSpec from twisted.enterprise import adbapi @@ -194,7 +194,7 @@ def __getattr__(self, name): # The type of entry which goes on our after_callbacks and exception_callbacks lists. _CallbackListEntry = Tuple[Callable[..., object], Iterable[Any], Dict[str, Any]] - +P = ParamSpec("P") R = TypeVar("R") @@ -339,7 +339,13 @@ def _make_sql_one_line(self, sql: str) -> str: "Strip newlines out of SQL so that the loggers in the DB are on one line" return " ".join(line.strip() for line in sql.splitlines() if line.strip()) - def _do_execute(self, func: Callable[..., R], sql: str, *args: Any) -> R: + def _do_execute( + self, + func: Callable[Concatenate[str, P], R], + sql: str, + *args: P.args, + **kwargs: P.kwargs, + ) -> R: sql = self._make_sql_one_line(sql) # TODO(paul): Maybe use 'info' and 'debug' for values? @@ -348,7 +354,10 @@ def _do_execute(self, func: Callable[..., R], sql: str, *args: Any) -> R: sql = self.database_engine.convert_param_style(sql) if args: try: - sql_logger.debug("[SQL values] {%s} %r", self.name, args[0]) + # The type-ignore should be redundant once mypy releases a version with + # https://github.com/python/mypy/pull/12668. (`args` might be empty, + # (but we'll catch the index error if so.) + sql_logger.debug("[SQL values] {%s} %r", self.name, args[0]) # type: ignore[index] except Exception: # Don't let logging failures stop SQL from working pass @@ -363,7 +372,7 @@ def _do_execute(self, func: Callable[..., R], sql: str, *args: Any) -> R: opentracing.tags.DATABASE_STATEMENT: sql, }, ): - return func(sql, *args) + return func(sql, *args, **kwargs) except Exception as e: sql_logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise From 18d6c18aa16ab9809656d1175adf4617da44e36c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 9 May 2022 04:38:32 -0500 Subject: [PATCH 133/263] Fix docs on how to run specific Complement tests after recent `complement.sh` change (#12664) --- changelog.d/12664.doc | 1 + docs/development/contributing_guide.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12664.doc diff --git a/changelog.d/12664.doc b/changelog.d/12664.doc new file mode 100644 index 000000000000..142d18037a12 --- /dev/null +++ b/changelog.d/12664.doc @@ -0,0 +1 @@ +Fix docs on how to run specific Complement tests using the `complement.sh` test runner. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 3b5c77401863..d356c72bf780 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -270,13 +270,13 @@ COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh To run a specific test file, you can pass the test name at the end of the command. The name passed comes from the naming structure in your Complement tests. If you're unsure of the name, you can do a full run and copy it from the test output: ```sh -COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory +COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages ``` To run a specific test, you can specify the whole name structure: ```sh -COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory/parallel/Backfilled_historical_events_resolve_with_proper_state_in_correct_order +COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages/parallel/Historical_events_resolve_in_the_correct_order ``` From 77258b67257983d67f90270d3d8e04594fd512ba Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Mon, 9 May 2022 12:08:31 +0200 Subject: [PATCH 134/263] docs(contrib): Add link to documentation in dashboard (#12602) --- changelog.d/12602.misc | 1 + contrib/grafana/synapse.json | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12602.misc diff --git a/changelog.d/12602.misc b/changelog.d/12602.misc new file mode 100644 index 000000000000..cdccc5c31661 --- /dev/null +++ b/changelog.d/12602.misc @@ -0,0 +1 @@ +Add link to documentation in Grafana Dashboard. diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 2c839c30d036..819426b8ea2b 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -66,6 +66,18 @@ ], "title": "Dashboards", "type": "dashboards" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Synapse Documentation", + "tooltip": "Open Documentation", + "type": "link", + "url": "https://matrix-org.github.io/synapse/latest/" } ], "panels": [ @@ -10889,4 +10901,4 @@ "title": "Synapse", "uid": "000000012", "version": 100 -} \ No newline at end of file +} From c5969b346d2694fea32ddd063ce40fec41028b57 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 May 2022 11:09:19 +0100 Subject: [PATCH 135/263] Don't error on unknown receipt types (#12670) Fixes #12669 --- changelog.d/12670.feature | 1 + synapse/rest/client/read_marker.py | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 changelog.d/12670.feature diff --git a/changelog.d/12670.feature b/changelog.d/12670.feature new file mode 100644 index 000000000000..cd5c45029ee1 --- /dev/null +++ b/changelog.d/12670.feature @@ -0,0 +1 @@ +Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index 1583e903cd88..3644705e6abb 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, Tuple from synapse.api.constants import ReceiptTypes -from synapse.api.errors import SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -50,17 +49,21 @@ async def on_POST( body = parse_json_object_from_request(request) - valid_receipt_types = {ReceiptTypes.READ, ReceiptTypes.FULLY_READ} - if self.config.experimental.msc2285_enabled: - valid_receipt_types.add(ReceiptTypes.READ_PRIVATE) - - if set(body.keys()) > valid_receipt_types: - raise SynapseError( - 400, - "Receipt type must be 'm.read', 'org.matrix.msc2285.read.private' or 'm.fully_read'" - if self.config.experimental.msc2285_enabled - else "Receipt type must be 'm.read' or 'm.fully_read'", - ) + valid_receipt_types = { + ReceiptTypes.READ, + ReceiptTypes.FULLY_READ, + ReceiptTypes.READ_PRIVATE, + } + + unrecognized_types = set(body.keys()) - valid_receipt_types + if unrecognized_types: + # It's fine if there are unrecognized receipt types, but let's log + # it to help debug clients that have typoed the receipt type. + # + # We specifically *don't* error here, as a) it stops us processing + # the valid receipts, and b) we need to be extensible on receipt + # types. + logger.info("Ignoring unrecognized receipt types: %s", unrecognized_types) read_event_id = body.get(ReceiptTypes.READ, None) if read_event_id: From fa0eab9c8e159b698a31fc7cfaafed643f47e284 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 9 May 2022 11:27:39 +0100 Subject: [PATCH 136/263] Use `ParamSpec` in a few places (#12667) --- changelog.d/12667.misc | 1 + poetry.lock | 2 +- pyproject.toml | 4 ++- synapse/app/_base.py | 14 ++++++++--- synapse/events/presence_router.py | 15 +++++++++--- synapse/module_api/__init__.py | 17 +++++++------ synapse/rest/client/knock.py | 4 +-- synapse/rest/client/transactions.py | 19 +++++++++------ synapse/storage/database.py | 31 +++++++++++++++--------- synapse/storage/databases/main/events.py | 8 ++++-- synapse/util/async_helpers.py | 26 ++++++++++++++------ synapse/util/distributor.py | 29 +++++++++++++++++----- synapse/util/metrics.py | 31 ++++++++++++++++-------- synapse/util/patch_inline_callbacks.py | 15 +++++++----- 14 files changed, 148 insertions(+), 68 deletions(-) create mode 100644 changelog.d/12667.misc diff --git a/changelog.d/12667.misc b/changelog.d/12667.misc new file mode 100644 index 000000000000..2b17502d6b57 --- /dev/null +++ b/changelog.d/12667.misc @@ -0,0 +1 @@ +Use `ParamSpec` to refine type hints. diff --git a/poetry.lock b/poetry.lock index ddafaaeba066..f649efdf2b9c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1563,7 +1563,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "eebc9e1d720e2e866f5fddda98ce83d858949a6fdbe30c7e5aef4cf9d17be498" +content-hash = "d39d5ac5d51c014581186b7691999b861058b569084c525523baf70b77f292b1" [metadata.files] attrs = [ diff --git a/pyproject.toml b/pyproject.toml index 4c51b8c4a112..2c4b7eb08ecd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,7 +143,9 @@ netaddr = ">=0.7.18" Jinja2 = ">=3.0" bleach = ">=1.4.3" # We use `ParamSpec` and `Concatenate`, which were added in `typing-extensions` 3.10.0.0. -typing-extensions = ">=3.10.0" +# Additionally we need https://github.com/python/typing/pull/817 to allow types to be +# generic over ParamSpecs. +typing-extensions = ">=3.10.0.1" # We enforce that we have a `cryptography` version that bundles an `openssl` # with the latest security patches. cryptography = ">=3.4.7" diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d28b87a3f4d8..3623c1724ded 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -38,6 +38,7 @@ from cryptography.utils import CryptographyDeprecationWarning from matrix_common.versionstring import get_distribution_version_string +from typing_extensions import ParamSpec import twisted from twisted.internet import defer, error, reactor as _reactor @@ -81,11 +82,12 @@ # list of tuples of function, args list, kwargs dict _sighup_callbacks: List[ - Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]] + Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]] ] = [] +P = ParamSpec("P") -def register_sighup(func: Callable[..., None], *args: Any, **kwargs: Any) -> None: +def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: """ Register a function to be called when a SIGHUP occurs. @@ -93,7 +95,9 @@ def register_sighup(func: Callable[..., None], *args: Any, **kwargs: Any) -> Non func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ - _sighup_callbacks.append((func, args, kwargs)) + # This type-ignore should be redundant once we use a mypy release with + # https://github.com/python/mypy/pull/12668. + _sighup_callbacks.append((func, args, kwargs)) # type: ignore[arg-type] def start_worker_reactor( @@ -214,7 +218,9 @@ def redirect_stdio_to_logs() -> None: print("Redirected stdout/stderr to logs") -def register_start(cb: Callable[..., Awaitable], *args: Any, **kwargs: Any) -> None: +def register_start( + cb: Callable[P, Awaitable], *args: P.args, **kwargs: P.kwargs +) -> None: """Register a callback with the reactor, to be called once it is running This can be used to initialise parts of the system which require an asynchronous diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index 98555c8c0c65..8437ce52dc97 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -22,9 +22,12 @@ List, Optional, Set, + TypeVar, Union, ) +from typing_extensions import ParamSpec + from synapse.api.presence import UserPresenceState from synapse.util.async_helpers import maybe_awaitable @@ -40,6 +43,10 @@ logger = logging.getLogger(__name__) +P = ParamSpec("P") +R = TypeVar("R") + + def load_legacy_presence_router(hs: "HomeServer") -> None: """Wrapper that loads a presence router module configured using the old configuration, and registers the hooks they implement. @@ -63,13 +70,15 @@ def load_legacy_presence_router(hs: "HomeServer") -> None: # All methods that the module provides should be async, but this wasn't enforced # in the old module system, so we wrap them if needed - def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]: + def async_wrapper( + f: Optional[Callable[P, R]] + ) -> Optional[Callable[P, Awaitable[R]]]: # f might be None if the callback isn't implemented by the module. In this # case we don't want to register a callback at all so we return None. if f is None: return None - def run(*args: Any, **kwargs: Any) -> Awaitable: + def run(*args: P.args, **kwargs: P.kwargs) -> Awaitable[R]: # Assertion required because mypy can't prove we won't change `f` # back to `None`. See # https://mypy.readthedocs.io/en/latest/common_issues.html#narrowing-and-inner-functions @@ -80,7 +89,7 @@ def run(*args: Any, **kwargs: Any) -> Awaitable: return run # Register the hooks through the module API. - hooks = { + hooks: Dict[str, Optional[Callable[..., Any]]] = { hook: async_wrapper(getattr(presence_router, hook, None)) for hook in presence_router_methods } diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 834fe1b62c77..73f92d2df8d6 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -30,6 +30,7 @@ import attr import jinja2 +from typing_extensions import ParamSpec from twisted.internet import defer from twisted.web.resource import Resource @@ -129,6 +130,7 @@ T = TypeVar("T") +P = ParamSpec("P") """ This package defines the 'stable' API which can be used by extension modules which @@ -799,9 +801,9 @@ def invalidate_access_token( def run_db_interaction( self, desc: str, - func: Callable[..., T], - *args: Any, - **kwargs: Any, + func: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> "defer.Deferred[T]": """Run a function with a database connection @@ -817,8 +819,9 @@ def run_db_interaction( Returns: Deferred[object]: result of func """ + # type-ignore: See https://github.com/python/mypy/issues/8862 return defer.ensureDeferred( - self._store.db_pool.runInteraction(desc, func, *args, **kwargs) + self._store.db_pool.runInteraction(desc, func, *args, **kwargs) # type: ignore[arg-type] ) def complete_sso_login( @@ -1296,9 +1299,9 @@ async def get_room_state( async def defer_to_thread( self, - f: Callable[..., T], - *args: Any, - **kwargs: Any, + f: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> T: """Runs the given function in a separate thread from Synapse's thread pool. diff --git a/synapse/rest/client/knock.py b/synapse/rest/client/knock.py index 0152a0c66a50..ad025c8a4529 100644 --- a/synapse/rest/client/knock.py +++ b/synapse/rest/client/knock.py @@ -15,8 +15,6 @@ import logging from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple -from twisted.web.server import Request - from synapse.api.constants import Membership from synapse.api.errors import SynapseError from synapse.http.server import HttpServer @@ -97,7 +95,7 @@ async def on_POST( return 200, {"room_id": room_id} def on_PUT( - self, request: Request, room_identifier: str, txn_id: str + self, request: SynapseRequest, room_identifier: str, txn_id: str ) -> Awaitable[Tuple[int, JsonDict]]: set_tag("txn_id", txn_id) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 914fb3acf5aa..61375651bc15 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -15,7 +15,9 @@ """This module contains logic for storing HTTP PUT transactions. This is used to ensure idempotency when performing PUTs using the REST API.""" import logging -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Tuple +from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Tuple + +from typing_extensions import ParamSpec from twisted.python.failure import Failure from twisted.web.server import Request @@ -32,6 +34,9 @@ CLEANUP_PERIOD_MS = 1000 * 60 * 30 # 30 mins +P = ParamSpec("P") + + class HttpTransactionCache: def __init__(self, hs: "HomeServer"): self.hs = hs @@ -65,9 +70,9 @@ def _get_transaction_key(self, request: Request) -> str: def fetch_or_execute_request( self, request: Request, - fn: Callable[..., Awaitable[Tuple[int, JsonDict]]], - *args: Any, - **kwargs: Any, + fn: Callable[P, Awaitable[Tuple[int, JsonDict]]], + *args: P.args, + **kwargs: P.kwargs, ) -> Awaitable[Tuple[int, JsonDict]]: """A helper function for fetch_or_execute which extracts a transaction key from the given request. @@ -82,9 +87,9 @@ def fetch_or_execute_request( def fetch_or_execute( self, txn_key: str, - fn: Callable[..., Awaitable[Tuple[int, JsonDict]]], - *args: Any, - **kwargs: Any, + fn: Callable[P, Awaitable[Tuple[int, JsonDict]]], + *args: P.args, + **kwargs: P.kwargs, ) -> Awaitable[Tuple[int, JsonDict]]: """Fetches the response for this transaction, or executes the given function to produce a response for this transaction. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 2255e55f6f87..41f566b6487a 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -192,7 +192,7 @@ def __getattr__(self, name): # The type of entry which goes on our after_callbacks and exception_callbacks lists. -_CallbackListEntry = Tuple[Callable[..., object], Iterable[Any], Dict[str, Any]] +_CallbackListEntry = Tuple[Callable[..., object], Tuple[object, ...], Dict[str, object]] P = ParamSpec("P") R = TypeVar("R") @@ -239,7 +239,9 @@ def __init__( self.after_callbacks = after_callbacks self.exception_callbacks = exception_callbacks - def call_after(self, callback: Callable[..., object], *args: Any, **kwargs: Any): + def call_after( + self, callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs + ) -> None: """Call the given callback on the main twisted thread after the transaction has finished. @@ -256,11 +258,12 @@ def call_after(self, callback: Callable[..., object], *args: Any, **kwargs: Any) # LoggingTransaction isn't expecting there to be any callbacks; assert that # is not the case. assert self.after_callbacks is not None - self.after_callbacks.append((callback, args, kwargs)) + # type-ignore: need mypy containing https://github.com/python/mypy/pull/12668 + self.after_callbacks.append((callback, args, kwargs)) # type: ignore[arg-type] def call_on_exception( - self, callback: Callable[..., object], *args: Any, **kwargs: Any - ): + self, callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs + ) -> None: """Call the given callback on the main twisted thread after the transaction has failed. @@ -274,7 +277,8 @@ def call_on_exception( # LoggingTransaction isn't expecting there to be any callbacks; assert that # is not the case. assert self.exception_callbacks is not None - self.exception_callbacks.append((callback, args, kwargs)) + # type-ignore: need mypy containing https://github.com/python/mypy/pull/12668 + self.exception_callbacks.append((callback, args, kwargs)) # type: ignore[arg-type] def fetchone(self) -> Optional[Tuple]: return self.txn.fetchone() @@ -549,9 +553,9 @@ def new_transaction( desc: str, after_callbacks: List[_CallbackListEntry], exception_callbacks: List[_CallbackListEntry], - func: Callable[..., R], - *args: Any, - **kwargs: Any, + func: Callable[Concatenate[LoggingTransaction, P], R], + *args: P.args, + **kwargs: P.kwargs, ) -> R: """Start a new database transaction with the given connection. @@ -581,7 +585,10 @@ def new_transaction( # will fail if we have to repeat the transaction. # For now, we just log an error, and hope that it works on the first attempt. # TODO: raise an exception. - for i, arg in enumerate(args): + + # Type-ignore Mypy doesn't yet consider ParamSpec.args to be iterable; see + # https://github.com/python/mypy/pull/12668 + for i, arg in enumerate(args): # type: ignore[arg-type, var-annotated] if inspect.isgenerator(arg): logger.error( "Programming error: generator passed to new_transaction as " @@ -589,7 +596,9 @@ def new_transaction( i, func, ) - for name, val in kwargs.items(): + # Type-ignore Mypy doesn't yet consider ParamSpec.args to be a mapping; see + # https://github.com/python/mypy/pull/12668 + for name, val in kwargs.items(): # type: ignore[attr-defined] if inspect.isgenerator(val): logger.error( "Programming error: generator passed to new_transaction as " diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 9a6c2fd47a55..ed29a0a5e2db 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1648,8 +1648,12 @@ def prefill(): txn.call_after(prefill) def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None: - # Invalidate the caches for the redacted event, note that these caches - # are also cleared as part of event replication in _invalidate_caches_for_event. + """Invalidate the caches for the redacted event. + + Note that these caches are also cleared as part of event replication in + _invalidate_caches_for_event. + """ + assert event.redacts is not None txn.call_after(self.store._invalidate_get_event_cache, event.redacts) txn.call_after(self.store.get_relations_for_event.invalidate, (event.redacts,)) txn.call_after(self.store.get_applicable_edit.invalidate, (event.redacts,)) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index e27c5d298f6a..b91020117f41 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -42,7 +42,7 @@ ) import attr -from typing_extensions import AsyncContextManager, Literal +from typing_extensions import AsyncContextManager, Concatenate, Literal, ParamSpec from twisted.internet import defer from twisted.internet.defer import CancelledError @@ -237,9 +237,16 @@ async def _concurrently_execute_inner(value: T) -> None: ) +P = ParamSpec("P") +R = TypeVar("R") + + async def yieldable_gather_results( - func: Callable[..., Awaitable[T]], iter: Iterable, *args: Any, **kwargs: Any -) -> List[T]: + func: Callable[Concatenate[T, P], Awaitable[R]], + iter: Iterable[T], + *args: P.args, + **kwargs: P.kwargs, +) -> List[R]: """Executes the function with each argument concurrently. Args: @@ -255,7 +262,15 @@ async def yieldable_gather_results( try: return await make_deferred_yieldable( defer.gatherResults( - [run_in_background(func, item, *args, **kwargs) for item in iter], + # type-ignore: mypy reports two errors: + # error: Argument 1 to "run_in_background" has incompatible type + # "Callable[[T, **P], Awaitable[R]]"; expected + # "Callable[[T, **P], Awaitable[R]]" [arg-type] + # error: Argument 2 to "run_in_background" has incompatible type + # "T"; expected "[T, **P.args]" [arg-type] + # The former looks like a mypy bug, and the latter looks like a + # false positive. + [run_in_background(func, item, *args, **kwargs) for item in iter], # type: ignore[arg-type] consumeErrors=True, ) ) @@ -577,9 +592,6 @@ async def _ctx_manager() -> AsyncIterator[None]: return _ctx_manager() -R = TypeVar("R") - - def timeout_deferred( deferred: "defer.Deferred[_T]", timeout: float, reactor: IReactorTime ) -> "defer.Deferred[_T]": diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 91837655f8fa..b580bdd0deef 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -12,7 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Callable, Dict, List +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Generic, + List, + Optional, + TypeVar, + Union, +) + +from typing_extensions import ParamSpec from twisted.internet import defer @@ -75,7 +87,11 @@ def fire(self, name: str, *args: Any, **kwargs: Any) -> None: run_as_background_process(name, self.signals[name].fire, *args, **kwargs) -class Signal: +P = ParamSpec("P") +R = TypeVar("R") + + +class Signal(Generic[P]): """A Signal is a dispatch point that stores a list of callables as observers of it. @@ -87,16 +103,16 @@ class Signal: def __init__(self, name: str): self.name: str = name - self.observers: List[Callable] = [] + self.observers: List[Callable[P, Any]] = [] - def observe(self, observer: Callable) -> None: + def observe(self, observer: Callable[P, Any]) -> None: """Adds a new callable to the observer list which will be invoked by the 'fire' method. Each observer callable may return a Deferred.""" self.observers.append(observer) - def fire(self, *args: Any, **kwargs: Any) -> "defer.Deferred[List[Any]]": + def fire(self, *args: P.args, **kwargs: P.kwargs) -> "defer.Deferred[List[Any]]": """Invokes every callable in the observer list, passing in the args and kwargs. Exceptions thrown by observers are logged but ignored. It is not an error to fire a signal with no observers. @@ -104,7 +120,7 @@ def fire(self, *args: Any, **kwargs: Any) -> "defer.Deferred[List[Any]]": Returns a Deferred that will complete when all the observers have completed.""" - async def do(observer: Callable[..., Any]) -> Any: + async def do(observer: Callable[P, Union[R, Awaitable[R]]]) -> Optional[R]: try: return await maybe_awaitable(observer(*args, **kwargs)) except Exception as e: @@ -114,6 +130,7 @@ async def do(observer: Callable[..., Any]) -> Any: observer, e, ) + return None deferreds = [run_in_background(do, o) for o in self.observers] diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 98ee49af6eb6..bc3b4938ea15 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -15,10 +15,10 @@ import logging from functools import wraps from types import TracebackType -from typing import Any, Callable, Optional, Type, TypeVar, cast +from typing import Awaitable, Callable, Optional, Type, TypeVar from prometheus_client import Counter -from typing_extensions import Protocol +from typing_extensions import Concatenate, ParamSpec, Protocol from synapse.logging.context import ( ContextResourceUsage, @@ -72,16 +72,21 @@ class _InFlightMetric(Protocol): ) -T = TypeVar("T", bound=Callable[..., Any]) +P = ParamSpec("P") +R = TypeVar("R") class HasClock(Protocol): clock: Clock -def measure_func(name: Optional[str] = None) -> Callable[[T], T]: - """ - Used to decorate an async function with a `Measure` context manager. +def measure_func( + name: Optional[str] = None, +) -> Callable[[Callable[P, Awaitable[R]]], Callable[P, Awaitable[R]]]: + """Decorate an async method with a `Measure` context manager. + + The Measure is created using `self.clock`; it should only be used to decorate + methods in classes defining an instance-level `clock` attribute. Usage: @@ -97,18 +102,24 @@ async def foo(...): """ - def wrapper(func: T) -> T: + def wrapper( + func: Callable[Concatenate[HasClock, P], Awaitable[R]] + ) -> Callable[P, Awaitable[R]]: block_name = func.__name__ if name is None else name @wraps(func) - async def measured_func(self: HasClock, *args: Any, **kwargs: Any) -> Any: + async def measured_func(self: HasClock, *args: P.args, **kwargs: P.kwargs) -> R: with Measure(self.clock, block_name): r = await func(self, *args, **kwargs) return r - return cast(T, measured_func) + # There are some shenanigans here, because we're decorating a method but + # explicitly making use of the `self` parameter. The key thing here is that the + # return type within the return type for `measure_func` itself describes how the + # decorated function will be called. + return measured_func # type: ignore[return-value] - return wrapper + return wrapper # type: ignore[return-value] class Measure: diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py index dace68666c9c..f97f98a05742 100644 --- a/synapse/util/patch_inline_callbacks.py +++ b/synapse/util/patch_inline_callbacks.py @@ -16,6 +16,8 @@ import sys from typing import Any, Callable, Generator, List, TypeVar, cast +from typing_extensions import ParamSpec + from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -25,6 +27,7 @@ T = TypeVar("T") +P = ParamSpec("P") def do_patch() -> None: @@ -41,13 +44,13 @@ def do_patch() -> None: return def new_inline_callbacks( - f: Callable[..., Generator["Deferred[object]", object, T]] - ) -> Callable[..., "Deferred[T]"]: + f: Callable[P, Generator["Deferred[object]", object, T]] + ) -> Callable[P, "Deferred[T]"]: @functools.wraps(f) - def wrapped(*args: Any, **kwargs: Any) -> "Deferred[T]": + def wrapped(*args: P.args, **kwargs: P.kwargs) -> "Deferred[T]": start_context = current_context() changes: List[str] = [] - orig: Callable[..., "Deferred[T]"] = orig_inline_callbacks( + orig: Callable[P, "Deferred[T]"] = orig_inline_callbacks( _check_yield_points(f, changes) ) @@ -115,7 +118,7 @@ def check_ctx(r: T) -> T: def _check_yield_points( - f: Callable[..., Generator["Deferred[object]", object, T]], + f: Callable[P, Generator["Deferred[object]", object, T]], changes: List[str], ) -> Callable: """Wraps a generator that is about to be passed to defer.inlineCallbacks @@ -138,7 +141,7 @@ def _check_yield_points( @functools.wraps(f) def check_yield_points_inner( - *args: Any, **kwargs: Any + *args: P.args, **kwargs: P.kwargs ) -> Generator["Deferred[object]", object, T]: gen = f(*args, **kwargs) From 41a882e62d6d17f7296e4c77f5bde9692aa8aa8b Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 9 May 2022 11:34:39 +0100 Subject: [PATCH 137/263] Update changelog for #12587 to be more accurate (#12663) #12587 has fallen on the wrong side of the release cutoff to the rest of the related PRs. Signed-off-by: Sean Quah --- changelog.d/12587.misc | 2 +- changelog.d/12663.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12663.misc diff --git a/changelog.d/12587.misc b/changelog.d/12587.misc index d26e332305ce..3b466f1ddf16 100644 --- a/changelog.d/12587.misc +++ b/changelog.d/12587.misc @@ -1 +1 @@ -Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. +Log status code of cancelled requests as 499 and avoid logging stack traces for them. diff --git a/changelog.d/12663.misc b/changelog.d/12663.misc new file mode 100644 index 000000000000..3b466f1ddf16 --- /dev/null +++ b/changelog.d/12663.misc @@ -0,0 +1 @@ +Log status code of cancelled requests as 499 and avoid logging stack traces for them. From 8de0facaae08c422cbf1acefe898820d3bf5c632 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 9 May 2022 11:48:14 +0100 Subject: [PATCH 138/263] Fix mypy against latest pillow stubs (#12671) --- changelog.d/12671.misc | 1 + poetry.lock | 6 +++--- synapse/rest/media/v1/thumbnailer.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12671.misc diff --git a/changelog.d/12671.misc b/changelog.d/12671.misc new file mode 100644 index 000000000000..56df4e383154 --- /dev/null +++ b/changelog.d/12671.misc @@ -0,0 +1 @@ +Fix mypy against latest pillow stubs. diff --git a/poetry.lock b/poetry.lock index f649efdf2b9c..49a912a58962 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1371,7 +1371,7 @@ python-versions = "*" [[package]] name = "types-pillow" -version = "9.0.6" +version = "9.0.15" description = "Typing stubs for Pillow" category = "dev" optional = false @@ -2626,8 +2626,8 @@ types-opentracing = [ {file = "types_opentracing-2.4.7-py3-none-any.whl", hash = "sha256:861fb8103b07cf717f501dd400cb274ca9992552314d4d6c7a824b11a215e512"}, ] types-pillow = [ - {file = "types-Pillow-9.0.6.tar.gz", hash = "sha256:79b350b1188c080c27558429f1e119e69c9f020b877a82df761d9283070e0185"}, - {file = "types_Pillow-9.0.6-py3-none-any.whl", hash = "sha256:bd1e0a844fc718398aa265bf50fcad550fc520cc54f80e5ffeb7b3226b3cc507"}, + {file = "types-Pillow-9.0.15.tar.gz", hash = "sha256:d2e385fe5c192e75970f18accce69f5c2a9f186f3feb578a9b91cd6fdf64211d"}, + {file = "types_Pillow-9.0.15-py3-none-any.whl", hash = "sha256:c9646595dfafdf8b63d4b1443292ead17ee0fc7b18a143e497b68e0ea2dc1eb6"}, ] types-psycopg2 = [ {file = "types-psycopg2-2.9.9.tar.gz", hash = "sha256:4f9d4d52eeb343dc00fd5ed4f1513a8a5c18efba0a072eb82706d15cf4f20a2e"}, diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py index 5e17664b5b0e..390491eb83cc 100644 --- a/synapse/rest/media/v1/thumbnailer.py +++ b/synapse/rest/media/v1/thumbnailer.py @@ -121,10 +121,10 @@ def _resize(self, width: int, height: int) -> Image.Image: # # If the image has transparency, use RGBA instead. if self.image.mode in ["1", "L", "P"]: - mode = "RGB" if self.image.info.get("transparency", None) is not None: - mode = "RGBA" - self.image = self.image.convert(mode) + self.image = self.image.convert("RGBA") + else: + self.image = self.image.convert("RGB") return self.image.resize((width, height), Image.ANTIALIAS) def scale(self, width: int, height: int, output_type: str) -> BytesIO: From a00462dd9927558532b030593f8914ade53b7214 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 9 May 2022 12:31:14 +0100 Subject: [PATCH 139/263] Implement cancellation support/protection for module callbacks (#12568) There's no guarantee that module callbacks will handle cancellation appropriately. Protect module callbacks with read semantics from cancellation and avoid swallowing `CancelledError`s that arise. Other module callbacks, such as the `on_*` callbacks, are presumed to live on code paths that involve writes and aren't cancellation-friendly. These module callbacks have been left alone. Signed-off-by: Sean Quah --- changelog.d/12568.misc | 1 + synapse/events/presence_router.py | 12 +++++++--- synapse/events/spamcheck.py | 36 +++++++++++++++++++--------- synapse/events/third_party_rules.py | 36 +++++++++++++++++++++++----- synapse/handlers/account_validity.py | 3 ++- synapse/handlers/auth.py | 25 ++++++++++++++----- 6 files changed, 86 insertions(+), 27 deletions(-) create mode 100644 changelog.d/12568.misc diff --git a/changelog.d/12568.misc b/changelog.d/12568.misc new file mode 100644 index 000000000000..f64dc67c4f9a --- /dev/null +++ b/changelog.d/12568.misc @@ -0,0 +1 @@ +Protect module callbacks with read semantics against cancellation. diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index 8437ce52dc97..bb4a6bd9574a 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -28,8 +28,10 @@ from typing_extensions import ParamSpec +from twisted.internet.defer import CancelledError + from synapse.api.presence import UserPresenceState -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable if TYPE_CHECKING: from synapse.server import HomeServer @@ -158,7 +160,9 @@ async def get_users_for_states( try: # Note: result is an object here, because we don't trust modules to # return the types they're supposed to. - result: object = await callback(state_updates) + result: object = await delay_cancellation(callback(state_updates)) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue @@ -210,7 +214,9 @@ async def get_interested_users(self, user_id: str) -> Union[Set[str], str]: # run all the callbacks for get_interested_users and combine the results for callback in self._get_interested_users_callbacks: try: - result = await callback(user_id) + result = await delay_cancellation(callback(user_id)) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index cd80fcf9d13a..3b6795d40f6b 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -31,7 +31,7 @@ from synapse.rest.media.v1.media_storage import ReadableFileWrapper from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import RoomAlias, UserProfile -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable if TYPE_CHECKING: import synapse.events @@ -255,7 +255,7 @@ async def check_event_for_spam( will be used as the error message returned to the user. """ for callback in self._check_event_for_spam_callbacks: - res: Union[bool, str] = await callback(event) + res: Union[bool, str] = await delay_cancellation(callback(event)) if res: return res @@ -276,7 +276,10 @@ async def user_may_join_room( Whether the user may join the room """ for callback in self._user_may_join_room_callbacks: - if await callback(user_id, room_id, is_invited) is False: + may_join_room = await delay_cancellation( + callback(user_id, room_id, is_invited) + ) + if may_join_room is False: return False return True @@ -297,7 +300,10 @@ async def user_may_invite( True if the user may send an invite, otherwise False """ for callback in self._user_may_invite_callbacks: - if await callback(inviter_userid, invitee_userid, room_id) is False: + may_invite = await delay_cancellation( + callback(inviter_userid, invitee_userid, room_id) + ) + if may_invite is False: return False return True @@ -322,7 +328,10 @@ async def user_may_send_3pid_invite( True if the user may send the invite, otherwise False """ for callback in self._user_may_send_3pid_invite_callbacks: - if await callback(inviter_userid, medium, address, room_id) is False: + may_send_3pid_invite = await delay_cancellation( + callback(inviter_userid, medium, address, room_id) + ) + if may_send_3pid_invite is False: return False return True @@ -339,7 +348,8 @@ async def user_may_create_room(self, userid: str) -> bool: True if the user may create a room, otherwise False """ for callback in self._user_may_create_room_callbacks: - if await callback(userid) is False: + may_create_room = await delay_cancellation(callback(userid)) + if may_create_room is False: return False return True @@ -359,7 +369,10 @@ async def user_may_create_room_alias( True if the user may create a room alias, otherwise False """ for callback in self._user_may_create_room_alias_callbacks: - if await callback(userid, room_alias) is False: + may_create_room_alias = await delay_cancellation( + callback(userid, room_alias) + ) + if may_create_room_alias is False: return False return True @@ -377,7 +390,8 @@ async def user_may_publish_room(self, userid: str, room_id: str) -> bool: True if the user may publish the room, otherwise False """ for callback in self._user_may_publish_room_callbacks: - if await callback(userid, room_id) is False: + may_publish_room = await delay_cancellation(callback(userid, room_id)) + if may_publish_room is False: return False return True @@ -400,7 +414,7 @@ async def check_username_for_spam(self, user_profile: UserProfile) -> bool: for callback in self._check_username_for_spam_callbacks: # Make a copy of the user profile object to ensure the spam checker cannot # modify it. - if await callback(user_profile.copy()): + if await delay_cancellation(callback(user_profile.copy())): return True return False @@ -428,7 +442,7 @@ async def check_registration_for_spam( """ for callback in self._check_registration_for_spam_callbacks: - behaviour = await ( + behaviour = await delay_cancellation( callback(email_threepid, username, request_info, auth_provider_id) ) assert isinstance(behaviour, RegistrationBehaviour) @@ -472,7 +486,7 @@ async def check_media_file_for_spam( """ for callback in self._check_media_file_for_spam_callbacks: - spam = await callback(file_wrapper, file_info) + spam = await delay_cancellation(callback(file_wrapper, file_info)) if spam: return True diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py index ef68e2028220..9f4ff9799c00 100644 --- a/synapse/events/third_party_rules.py +++ b/synapse/events/third_party_rules.py @@ -14,12 +14,14 @@ import logging from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Optional, Tuple +from twisted.internet.defer import CancelledError + from synapse.api.errors import ModuleFailedException, SynapseError from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.storage.roommember import ProfileInfo from synapse.types import Requester, StateMap -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable if TYPE_CHECKING: from synapse.server import HomeServer @@ -263,7 +265,11 @@ async def check_event_allowed( for callback in self._check_event_allowed_callbacks: try: - res, replacement_data = await callback(event, state_events) + res, replacement_data = await delay_cancellation( + callback(event, state_events) + ) + except CancelledError: + raise except SynapseError as e: # FIXME: Being able to throw SynapseErrors is relied upon by # some modules. PR #10386 accidentally broke this ability. @@ -333,8 +339,13 @@ async def check_threepid_can_be_invited( for callback in self._check_threepid_can_be_invited_callbacks: try: - if await callback(medium, address, state_events) is False: + threepid_can_be_invited = await delay_cancellation( + callback(medium, address, state_events) + ) + if threepid_can_be_invited is False: return False + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) @@ -361,8 +372,13 @@ async def check_visibility_can_be_modified( for callback in self._check_visibility_can_be_modified_callbacks: try: - if await callback(room_id, state_events, new_visibility) is False: + visibility_can_be_modified = await delay_cancellation( + callback(room_id, state_events, new_visibility) + ) + if visibility_can_be_modified is False: return False + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) @@ -400,8 +416,11 @@ async def check_can_shutdown_room(self, user_id: str, room_id: str) -> bool: """ for callback in self._check_can_shutdown_room_callbacks: try: - if await callback(user_id, room_id) is False: + can_shutdown_room = await delay_cancellation(callback(user_id, room_id)) + if can_shutdown_room is False: return False + except CancelledError: + raise except Exception as e: logger.exception( "Failed to run module API callback %s: %s", callback, e @@ -422,8 +441,13 @@ async def check_can_deactivate_user( """ for callback in self._check_can_deactivate_user_callbacks: try: - if await callback(user_id, by_admin) is False: + can_deactivate_user = await delay_cancellation( + callback(user_id, by_admin) + ) + if can_deactivate_user is False: return False + except CancelledError: + raise except Exception as e: logger.exception( "Failed to run module API callback %s: %s", callback, e diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 05a138410e25..33e45e3a1136 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -23,6 +23,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.types import UserID from synapse.util import stringutils +from synapse.util.async_helpers import delay_cancellation if TYPE_CHECKING: from synapse.server import HomeServer @@ -150,7 +151,7 @@ async def is_user_expired(self, user_id: str) -> bool: Whether the user has expired. """ for callback in self._is_user_expired_callbacks: - expired = await callback(user_id) + expired = await delay_cancellation(callback(user_id)) if expired is not None: return expired diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ad41337b2892..1b9050ea96c8 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -41,6 +41,7 @@ import unpaddedbase64 from pymacaroons.exceptions import MacaroonVerificationFailedException +from twisted.internet.defer import CancelledError from twisted.web.server import Request from synapse.api.constants import LoginType @@ -67,7 +68,7 @@ from synapse.storage.roommember import ProfileInfo from synapse.types import JsonDict, Requester, UserID from synapse.util import stringutils as stringutils -from synapse.util.async_helpers import maybe_awaitable +from synapse.util.async_helpers import delay_cancellation, maybe_awaitable from synapse.util.macaroons import get_value_from_macaroon, satisfy_expiry from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.stringutils import base62_encode @@ -2202,7 +2203,11 @@ async def check_auth( # other than None (i.e. until a callback returns a success) for callback in self.auth_checker_callbacks[login_type]: try: - result = await callback(username, login_type, login_dict) + result = await delay_cancellation( + callback(username, login_type, login_dict) + ) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue @@ -2263,7 +2268,9 @@ async def check_3pid_auth( for callback in self.check_3pid_auth_callbacks: try: - result = await callback(medium, address, password) + result = await delay_cancellation(callback(medium, address, password)) + except CancelledError: + raise except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue @@ -2345,7 +2352,7 @@ async def get_username_for_registration( """ for callback in self.get_username_for_registration_callbacks: try: - res = await callback(uia_results, params) + res = await delay_cancellation(callback(uia_results, params)) if isinstance(res, str): return res @@ -2359,6 +2366,8 @@ async def get_username_for_registration( callback, res, ) + except CancelledError: + raise except Exception as e: logger.error( "Module raised an exception in get_username_for_registration: %s", @@ -2388,7 +2397,7 @@ async def get_displayname_for_registration( """ for callback in self.get_displayname_for_registration_callbacks: try: - res = await callback(uia_results, params) + res = await delay_cancellation(callback(uia_results, params)) if isinstance(res, str): return res @@ -2402,6 +2411,8 @@ async def get_displayname_for_registration( callback, res, ) + except CancelledError: + raise except Exception as e: logger.error( "Module raised an exception in get_displayname_for_registration: %s", @@ -2429,7 +2440,7 @@ async def is_3pid_allowed( """ for callback in self.is_3pid_allowed_callbacks: try: - res = await callback(medium, address, registration) + res = await delay_cancellation(callback(medium, address, registration)) if res is False: return res @@ -2443,6 +2454,8 @@ async def is_3pid_allowed( callback, res, ) + except CancelledError: + raise except Exception as e: logger.error("Module raised an exception in is_3pid_allowed: %s", e) raise SynapseError(code=500, msg="Internal Server Error") From bf0c3ca20a79136df962645d952ac75ce9d3615f Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Mon, 9 May 2022 22:29:07 +0200 Subject: [PATCH 140/263] Fix inconsistent spelling of 'M_UNRECOGNIZED'. (#12665) --- changelog.d/12665.misc | 1 + synapse/federation/federation_client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12665.misc diff --git a/changelog.d/12665.misc b/changelog.d/12665.misc new file mode 100644 index 000000000000..37b96fea37d3 --- /dev/null +++ b/changelog.d/12665.misc @@ -0,0 +1 @@ +Fix spelling of `M_UNRECOGNIZED` in comments. diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index b5e0b84cbc69..17eff60909a2 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -618,7 +618,7 @@ def _is_unknown_endpoint( # # Dendrite returns a 404 (with a body of "404 page not found"); # Conduit returns a 404 (with no body); and Synapse returns a 400 - # with M_UNRECOGNISED. + # with M_UNRECOGNIZED. # # This needs to be rather specific as some endpoints truly do return 404 # errors. From 34e84fee681e8c4503e17b28d6dc50dccfb6d84f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 9 May 2022 22:41:06 +0100 Subject: [PATCH 141/263] Tweaks to workers-under-complement (#12637) * Bump the HS startup timeout * Log prefixes for more processes * Bump the overall timeout --- changelog.d/12637.misc | 1 + docker/conf-workers/supervisord.conf.j2 | 6 +++--- scripts-dev/complement.sh | 13 +++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12637.misc diff --git a/changelog.d/12637.misc b/changelog.d/12637.misc new file mode 100644 index 000000000000..735257787fed --- /dev/null +++ b/changelog.d/12637.misc @@ -0,0 +1 @@ +Minor improvements to the scripts for running Synapse in worker mode under Complement. diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2 index 408ef727879d..ca1f7aef8e3f 100644 --- a/docker/conf-workers/supervisord.conf.j2 +++ b/docker/conf-workers/supervisord.conf.j2 @@ -9,7 +9,7 @@ user=root files = /etc/supervisor/conf.d/*.conf [program:nginx] -command=/usr/sbin/nginx -g "daemon off;" +command=/usr/local/bin/prefix-log /usr/sbin/nginx -g "daemon off;" priority=500 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 @@ -19,7 +19,7 @@ username=www-data autorestart=true [program:redis] -command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no +command=/usr/local/bin/prefix-log /usr/bin/redis-server /etc/redis/redis.conf --daemonize no priority=1 stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 @@ -29,7 +29,7 @@ username=redis autorestart=true [program:synapse_main] -command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml +command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml priority=10 # Log startup failures to supervisord's stdout/err # Regular synapse logs will still go in the configured data directory diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index e0feba05fa4f..190df6909a6a 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -43,6 +43,8 @@ fi # Build the base Synapse image from the local checkout docker build -t matrixdotorg/synapse -f "docker/Dockerfile" . +extra_test_args=() + # If we're using workers, modify the docker files slightly. if [[ -n "$WORKERS" ]]; then # Build the workers docker image (from the base Synapse image). @@ -52,7 +54,14 @@ if [[ -n "$WORKERS" ]]; then COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile # And provide some more configuration to complement. - export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=60 + + # It can take quite a while to spin up a worker-mode Synapse for the first + # time (the main problem is that we start 14 python processes for each test, + # and complement likes to do two of them in parallel). + export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=120 + + # ... and it takes longer than 10m to run the whole suite. + extra_test_args+=("-timeout=60m") else export COMPLEMENT_BASE_IMAGE=complement-synapse COMPLEMENT_DOCKERFILE=Dockerfile @@ -64,4 +73,4 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF # Run the tests! echo "Images built; running complement" cd "$COMPLEMENT_DIR" -go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/... +go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "${extra_test_args[@]}" "$@" ./tests/... From 615d96ad6e5bba6f260cb03f4ec119bef51a3309 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 9 May 2022 23:43:02 +0200 Subject: [PATCH 142/263] Update SQL statements in docs for Synapse Admins (#12536) --- changelog.d/12536.doc | 1 + .../administration/useful_sql_for_admins.md | 191 +++++++++++------- 2 files changed, 122 insertions(+), 70 deletions(-) create mode 100644 changelog.d/12536.doc diff --git a/changelog.d/12536.doc b/changelog.d/12536.doc new file mode 100644 index 000000000000..4034c42076c4 --- /dev/null +++ b/changelog.d/12536.doc @@ -0,0 +1 @@ +Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. diff --git a/docs/usage/administration/useful_sql_for_admins.md b/docs/usage/administration/useful_sql_for_admins.md index d4aada3272d0..f3b97f957677 100644 --- a/docs/usage/administration/useful_sql_for_admins.md +++ b/docs/usage/administration/useful_sql_for_admins.md @@ -1,7 +1,10 @@ ## Some useful SQL queries for Synapse Admins ## Size of full matrix db -`SELECT pg_size_pretty( pg_database_size( 'matrix' ) );` +```sql +SELECT pg_size_pretty( pg_database_size( 'matrix' ) ); +``` + ### Result example: ``` pg_size_pretty @@ -9,39 +12,19 @@ pg_size_pretty 6420 MB (1 row) ``` -## Show top 20 larger rooms by state events count -```sql -SELECT r.name, s.room_id, s.current_state_events - FROM room_stats_current s - LEFT JOIN room_stats_state r USING (room_id) - ORDER BY current_state_events DESC - LIMIT 20; -``` - -and by state_group_events count: -```sql -SELECT rss.name, s.room_id, count(s.room_id) FROM state_groups_state s -LEFT JOIN room_stats_state rss USING (room_id) -GROUP BY s.room_id, rss.name -ORDER BY count(s.room_id) DESC -LIMIT 20; -``` -plus same, but with join removed for performance reasons: -```sql -SELECT s.room_id, count(s.room_id) FROM state_groups_state s -GROUP BY s.room_id -ORDER BY count(s.room_id) DESC -LIMIT 20; -``` ## Show top 20 larger tables by row count ```sql -SELECT relname, n_live_tup as rows - FROM pg_stat_user_tables +SELECT relname, n_live_tup AS "rows" + FROM pg_stat_user_tables ORDER BY n_live_tup DESC LIMIT 20; ``` -This query is quick, but may be very approximate, for exact number of rows use `SELECT COUNT(*) FROM `. +This query is quick, but may be very approximate, for exact number of rows use: +```sql +SELECT COUNT(*) FROM ; +``` + ### Result example: ``` state_groups_state - 161687170 @@ -66,46 +49,19 @@ device_lists_stream - 326903 user_directory_search - 316433 ``` -## Show top 20 rooms by new events count in last 1 day: -```sql -SELECT e.room_id, r.name, COUNT(e.event_id) cnt FROM events e -LEFT JOIN room_stats_state r USING (room_id) -WHERE e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 GROUP BY e.room_id, r.name ORDER BY cnt DESC LIMIT 20; -``` - -## Show top 20 users on homeserver by sent events (messages) at last month: -```sql -SELECT user_id, SUM(total_events) - FROM user_stats_historical - WHERE TO_TIMESTAMP(end_ts/1000) AT TIME ZONE 'UTC' > date_trunc('day', now() - interval '1 month') - GROUP BY user_id - ORDER BY SUM(total_events) DESC - LIMIT 20; -``` - -## Show last 100 messages from needed user, with room names: -```sql -SELECT e.room_id, r.name, e.event_id, e.type, e.content, j.json FROM events e - LEFT JOIN event_json j USING (room_id) - LEFT JOIN room_stats_state r USING (room_id) - WHERE sender = '@LOGIN:example.com' - AND e.type = 'm.room.message' - ORDER BY stream_ordering DESC - LIMIT 100; -``` - ## Show top 20 larger tables by storage size ```sql SELECT nspname || '.' || relname AS "relation", - pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size" - FROM pg_class C - LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) + pg_size_pretty(pg_total_relation_size(c.oid)) AS "total_size" + FROM pg_class c + LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) WHERE nspname NOT IN ('pg_catalog', 'information_schema') - AND C.relkind <> 'i' + AND c.relkind <> 'i' AND nspname !~ '^pg_toast' - ORDER BY pg_total_relation_size(C.oid) DESC + ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 20; ``` + ### Result example: ``` public.state_groups_state - 27 GB @@ -130,8 +86,93 @@ public.device_lists_remote_cache - 124 MB public.state_group_edges - 122 MB ``` +## Show top 20 larger rooms by state events count +You get the same information when you use the +[admin API](../../admin_api/rooms.md#list-room-api) +and set parameter `order_by=state_events`. + +```sql +SELECT r.name, s.room_id, s.current_state_events + FROM room_stats_current s + LEFT JOIN room_stats_state r USING (room_id) + ORDER BY current_state_events DESC + LIMIT 20; +``` + +and by state_group_events count: +```sql +SELECT rss.name, s.room_id, COUNT(s.room_id) + FROM state_groups_state s + LEFT JOIN room_stats_state rss USING (room_id) + GROUP BY s.room_id, rss.name + ORDER BY COUNT(s.room_id) DESC + LIMIT 20; +``` + +plus same, but with join removed for performance reasons: +```sql +SELECT s.room_id, COUNT(s.room_id) + FROM state_groups_state s + GROUP BY s.room_id + ORDER BY COUNT(s.room_id) DESC + LIMIT 20; +``` + +## Show top 20 rooms by new events count in last 1 day: +```sql +SELECT e.room_id, r.name, COUNT(e.event_id) cnt + FROM events e + LEFT JOIN room_stats_state r USING (room_id) + WHERE e.origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 day') * 1000 + GROUP BY e.room_id, r.name + ORDER BY cnt DESC + LIMIT 20; +``` + +## Show top 20 users on homeserver by sent events (messages) at last month: +Caution. This query does not use any indexes, can be slow and create load on the database. +```sql +SELECT COUNT(*), sender + FROM events + WHERE (type = 'm.room.encrypted' OR type = 'm.room.message') + AND origin_server_ts >= DATE_PART('epoch', NOW() - INTERVAL '1 month') * 1000 + GROUP BY sender + ORDER BY COUNT(*) DESC + LIMIT 20; +``` + +## Show last 100 messages from needed user, with room names: +```sql +SELECT e.room_id, r.name, e.event_id, e.type, e.content, j.json + FROM events e + LEFT JOIN event_json j USING (room_id) + LEFT JOIN room_stats_state r USING (room_id) + WHERE sender = '@LOGIN:example.com' + AND e.type = 'm.room.message' + ORDER BY stream_ordering DESC + LIMIT 100; +``` + ## Show rooms with names, sorted by events in this rooms -`echo "select event_json.room_id,room_stats_state.name from event_json,room_stats_state where room_stats_state.room_id=event_json.room_id" | psql synapse | sort | uniq -c | sort -n` + +**Sort and order with bash** +```bash +echo "SELECT event_json.room_id, room_stats_state.name FROM event_json, room_stats_state \ +WHERE room_stats_state.room_id = event_json.room_id" | psql -d synapse -h localhost -U synapse_user -t \ +| sort | uniq -c | sort -n +``` +Documentation for `psql` command line parameters: https://www.postgresql.org/docs/current/app-psql.html + +**Sort and order with SQL** +```sql +SELECT COUNT(*), event_json.room_id, room_stats_state.name + FROM event_json, room_stats_state + WHERE room_stats_state.room_id = event_json.room_id + GROUP BY event_json.room_id, room_stats_state.name + ORDER BY COUNT(*) DESC + LIMIT 50; +``` + ### Result example: ``` 9459 !FPUfgzXYWTKgIrwKxW:matrix.org | This Week in Matrix @@ -145,12 +186,22 @@ public.state_group_edges - 122 MB ``` ## Lookup room state info by list of room_id +You get the same information when you use the +[admin API](../../admin_api/rooms.md#room-details-api). ```sql -SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption, rsc.joined_members, rsc.local_users_in_room, rss.join_rules -FROM room_stats_state rss -LEFT JOIN room_stats_current rsc USING (room_id) -WHERE room_id IN (WHERE room_id IN ( - '!OGEhHVWSdvArJzumhm:matrix.org', - '!YTvKGNlinIzlkMTVRl:matrix.org' -) -``` \ No newline at end of file +SELECT rss.room_id, rss.name, rss.canonical_alias, rss.topic, rss.encryption, + rsc.joined_members, rsc.local_users_in_room, rss.join_rules + FROM room_stats_state rss + LEFT JOIN room_stats_current rsc USING (room_id) + WHERE room_id IN ( WHERE room_id IN ( + '!OGEhHVWSdvArJzumhm:matrix.org', + '!YTvKGNlinIzlkMTVRl:matrix.org' + ); +``` + +## Show users and devices that have not been online for a while +```sql +SELECT user_id, device_id, user_agent, TO_TIMESTAMP(last_seen / 1000) AS "last_seen" + FROM devices + WHERE last_seen < DATE_PART('epoch', NOW() - INTERVAL '3 month') * 1000; +``` From d80a7ab151afd6919b4e14a258105ab59146d528 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 9 May 2022 14:46:43 -0700 Subject: [PATCH 143/263] Update `replication.md` with info on TCP module structure (#12621) --- changelog.d/12621.doc | 1 + docs/replication.md | 5 +++++ synapse/replication/tcp/__init__.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12621.doc diff --git a/changelog.d/12621.doc b/changelog.d/12621.doc new file mode 100644 index 000000000000..d29fb9cb995e --- /dev/null +++ b/changelog.d/12621.doc @@ -0,0 +1 @@ +Add information about the TCP replication module to docs. diff --git a/docs/replication.md b/docs/replication.md index e82df0de8a30..108da9a065d8 100644 --- a/docs/replication.md +++ b/docs/replication.md @@ -35,3 +35,8 @@ See [the TCP replication documentation](tcp_replication.md). There are read-only version of the synapse storage layer in `synapse/replication/slave/storage` that use the response of the replication API to invalidate their caches. + +### The TCP Replication Module +Information about how the tcp replication module is structured, including how +the classes interact, can be found in +`synapse/replication/tcp/__init__.py` diff --git a/synapse/replication/tcp/__init__.py b/synapse/replication/tcp/__init__.py index 1fa60af8e6bc..2c5f5f0bf867 100644 --- a/synapse/replication/tcp/__init__.py +++ b/synapse/replication/tcp/__init__.py @@ -15,7 +15,7 @@ """This module implements the TCP replication protocol used by synapse to communicate between the master process and its workers (when they're enabled). -Further details can be found in docs/tcp_replication.rst +Further details can be found in docs/tcp_replication.md Structure of the module: From ade30088212091284d066fc31a755a8a21050677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 10 May 2022 09:57:36 +0200 Subject: [PATCH 144/263] Implement MSC3786: Add a default push rule to ignore m.room.server_acl events (#12601) Fixes vector-im/element-web#20788 Implements matrix-org/matrix-spec-proposals#3786 --- changelog.d/12601.feature | 1 + synapse/config/experimental.py | 3 ++ synapse/push/baserules.py | 15 +++++++ synapse/storage/databases/main/push_rule.py | 48 ++++++++++++++++----- 4 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 changelog.d/12601.feature diff --git a/changelog.d/12601.feature b/changelog.d/12601.feature new file mode 100644 index 000000000000..c13360ff35c3 --- /dev/null +++ b/changelog.d/12601.feature @@ -0,0 +1 @@ +Implement MSC3786: Add a default push rule to ignore m.room.server_acl events. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index abed5e7edb3a..b20d949689ec 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -81,3 +81,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC2815 (allow room moderators to view redacted event content) self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) + + # MSC3786 (Add a default push rule to ignore m.room.server_acl events) + self.msc3786_enabled: bool = experimental.get("msc3786_enabled", False) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index f42f605f2383..a17b35a605fb 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -277,6 +277,21 @@ def make_base_prepend_rules( ], "actions": ["dont_notify"], }, + # XXX: This is an experimental rule that is only enabled if msc3786_enabled + # is enabled, if it is not the rule gets filtered out in _load_rules() in + # PushRulesWorkerStore + { + "rule_id": "global/override/.org.matrix.msc3786.rule.room.server_acl", + "conditions": [ + { + "kind": "event_match", + "key": "type", + "pattern": "m.room.server_acl", + "_cache_key": "_room_server_acl", + } + ], + "actions": ["dont_notify"], + }, ] diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index eb85bbd39243..4ed913e24879 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union from synapse.api.errors import StoreError +from synapse.config.homeserver import ExperimentalConfig from synapse.push.baserules import list_with_base_rules from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker from synapse.storage._base import SQLBaseStore, db_to_json @@ -42,7 +43,21 @@ logger = logging.getLogger(__name__) -def _load_rules(rawrules, enabled_map): +def _is_experimental_rule_enabled( + rule_id: str, experimental_config: ExperimentalConfig +) -> bool: + """Used by `_load_rules` to filter out experimental rules when they + have not been enabled. + """ + if ( + rule_id == "global/override/.org.matrix.msc3786.rule.room.server_acl" + and not experimental_config.msc3786_enabled + ): + return False + return True + + +def _load_rules(rawrules, enabled_map, experimental_config: ExperimentalConfig): ruleslist = [] for rawrule in rawrules: rule = dict(rawrule) @@ -51,17 +66,26 @@ def _load_rules(rawrules, enabled_map): rule["default"] = False ruleslist.append(rule) - # We're going to be mutating this a lot, so do a deep copy - rules = list(list_with_base_rules(ruleslist)) + # We're going to be mutating this a lot, so copy it. We also filter out + # any experimental default push rules that aren't enabled. + rules = [ + rule + for rule in list_with_base_rules(ruleslist) + if _is_experimental_rule_enabled(rule["rule_id"], experimental_config) + ] for i, rule in enumerate(rules): rule_id = rule["rule_id"] - if rule_id in enabled_map: - if rule.get("enabled", True) != bool(enabled_map[rule_id]): - # Rules are cached across users. - rule = dict(rule) - rule["enabled"] = bool(enabled_map[rule_id]) - rules[i] = rule + + if rule_id not in enabled_map: + continue + if rule.get("enabled", True) == bool(enabled_map[rule_id]): + continue + + # Rules are cached across users. + rule = dict(rule) + rule["enabled"] = bool(enabled_map[rule_id]) + rules[i] = rule return rules @@ -141,7 +165,7 @@ async def get_push_rules_for_user(self, user_id): enabled_map = await self.get_push_rules_enabled_for_user(user_id) - return _load_rules(rows, enabled_map) + return _load_rules(rows, enabled_map, self.hs.config.experimental) @cached(max_entries=5000) async def get_push_rules_enabled_for_user(self, user_id) -> Dict[str, bool]: @@ -200,7 +224,9 @@ async def bulk_get_push_rules(self, user_ids): enabled_map_by_user = await self.bulk_get_push_rules_enabled(user_ids) for user_id, rules in results.items(): - results[user_id] = _load_rules(rules, enabled_map_by_user.get(user_id, {})) + results[user_id] = _load_rules( + rules, enabled_map_by_user.get(user_id, {}), self.hs.config.experimental + ) return results From 8dd3e0e084304dfc02ff072a1beaed5266cf4e33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 May 2022 10:39:54 +0100 Subject: [PATCH 145/263] Immediately retry any requests that have backed off when a server comes back online. (#12500) Otherwise it can take up to a minute for any in-flight `/send` requests to be retried. --- changelog.d/12500.misc | 1 + synapse/http/matrixfederationclient.py | 15 ++++- synapse/notifier.py | 8 ++- synapse/util/async_helpers.py | 57 ++++++++++++++++++ synapse/util/retryutils.py | 24 +++++++- tests/util/test_async_helpers.py | 80 ++++++++++++++++++++++++++ 6 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12500.misc diff --git a/changelog.d/12500.misc b/changelog.d/12500.misc new file mode 100644 index 000000000000..dbe3f7f5d197 --- /dev/null +++ b/changelog.d/12500.misc @@ -0,0 +1 @@ +Immediately retry any requests that have backed off when a server comes back online. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index e68644595546..c2ec3caa0ea8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -73,7 +73,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.types import JsonDict from synapse.util import json_decoder -from synapse.util.async_helpers import timeout_deferred +from synapse.util.async_helpers import AwakenableSleeper, timeout_deferred from synapse.util.metrics import Measure if TYPE_CHECKING: @@ -353,6 +353,13 @@ def schedule(x): self._cooperator = Cooperator(scheduler=schedule) + self._sleeper = AwakenableSleeper(self.reactor) + + def wake_destination(self, destination: str) -> None: + """Called when the remote server may have come back online.""" + + self._sleeper.wake(destination) + async def _send_request_with_optional_trailing_slash( self, request: MatrixFederationRequest, @@ -474,6 +481,8 @@ async def _send_request( self._store, backoff_on_404=backoff_on_404, ignore_backoff=ignore_backoff, + notifier=self.hs.get_notifier(), + replication_client=self.hs.get_replication_command_handler(), ) method_bytes = request.method.encode("ascii") @@ -664,7 +673,9 @@ async def _send_request( delay, ) - await self.clock.sleep(delay) + # Sleep for the calculated delay, or wake up immediately + # if we get notified that the server is back up. + await self._sleeper.sleep(request.destination, delay * 1000) retries_left -= 1 else: raise diff --git a/synapse/notifier.py b/synapse/notifier.py index 16d15a1f3328..01a50b9d6226 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -228,9 +228,7 @@ def __init__(self, hs: "HomeServer"): # Called when there are new things to stream over replication self.replication_callbacks: List[Callable[[], None]] = [] - # Called when remote servers have come back online after having been - # down. - self.remote_server_up_callbacks: List[Callable[[str], None]] = [] + self._federation_client = hs.get_federation_http_client() self._third_party_rules = hs.get_third_party_event_rules() @@ -731,3 +729,7 @@ def notify_remote_server_up(self, server: str) -> None: # circular dependencies. if self.federation_sender: self.federation_sender.wake_destination(server) + + # Tell the federation client about the fact the server is back up, so + # that any in flight requests can be immediately retried. + self._federation_client.wake_destination(server) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index b91020117f41..7f1d41eb3c7a 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -778,3 +778,60 @@ def handle_cancel(new_deferred: "defer.Deferred[T]") -> None: new_deferred: "defer.Deferred[T]" = defer.Deferred(handle_cancel) deferred.chainDeferred(new_deferred) return new_deferred + + +class AwakenableSleeper: + """Allows explicitly waking up deferreds related to an entity that are + currently sleeping. + """ + + def __init__(self, reactor: IReactorTime) -> None: + self._streams: Dict[str, Set[defer.Deferred[None]]] = {} + self._reactor = reactor + + def wake(self, name: str) -> None: + """Wake everything related to `name` that is currently sleeping.""" + stream_set = self._streams.pop(name, set()) + for deferred in stream_set: + try: + with PreserveLoggingContext(): + deferred.callback(None) + except Exception: + pass + + async def sleep(self, name: str, delay_ms: int) -> None: + """Sleep for the given number of milliseconds, or return if the given + `name` is explicitly woken up. + """ + + # Create a deferred that gets called in N seconds + sleep_deferred: "defer.Deferred[None]" = defer.Deferred() + call = self._reactor.callLater(delay_ms / 1000, sleep_deferred.callback, None) + + # Create a deferred that will get called if `wake` is called with + # the same `name`. + stream_set = self._streams.setdefault(name, set()) + notify_deferred: "defer.Deferred[None]" = defer.Deferred() + stream_set.add(notify_deferred) + + try: + # Wait for either the delay or for `wake` to be called. + await make_deferred_yieldable( + defer.DeferredList( + [sleep_deferred, notify_deferred], + fireOnOneCallback=True, + fireOnOneErrback=True, + consumeErrors=True, + ) + ) + finally: + # Clean up the state + curr_stream_set = self._streams.get(name) + if curr_stream_set is not None: + curr_stream_set.discard(notify_deferred) + if len(curr_stream_set) == 0: + self._streams.pop(name) + + # Cancel the sleep if we were woken up + if call.active(): + call.cancel() diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index d81f2527d7be..81bfed268ee7 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -14,13 +14,17 @@ import logging import random from types import TracebackType -from typing import Any, Optional, Type +from typing import TYPE_CHECKING, Any, Optional, Type import synapse.logging.context from synapse.api.errors import CodeMessageException from synapse.storage import DataStore from synapse.util import Clock +if TYPE_CHECKING: + from synapse.notifier import Notifier + from synapse.replication.tcp.handler import ReplicationCommandHandler + logger = logging.getLogger(__name__) # the initial backoff, after the first transaction fails @@ -131,6 +135,8 @@ def __init__( retry_interval: int, backoff_on_404: bool = False, backoff_on_failure: bool = True, + notifier: Optional["Notifier"] = None, + replication_client: Optional["ReplicationCommandHandler"] = None, ): """Marks the destination as "down" if an exception is thrown in the context, except for CodeMessageException with code < 500. @@ -160,6 +166,9 @@ def __init__( self.backoff_on_404 = backoff_on_404 self.backoff_on_failure = backoff_on_failure + self.notifier = notifier + self.replication_client = replication_client + def __enter__(self) -> None: pass @@ -239,6 +248,19 @@ async def store_retry_timings() -> None: retry_last_ts, self.retry_interval, ) + + if self.notifier: + # Inform the relevant places that the remote server is back up. + self.notifier.notify_remote_server_up(self.destination) + + if self.replication_client: + # If we're on a worker we try and inform master about this. The + # replication client doesn't hook into the notifier to avoid + # infinite loops where we send a `REMOTE_SERVER_UP` command to + # master, which then echoes it back to us which in turn pokes + # the notifier. + self.replication_client.send_remote_server_up(self.destination) + except Exception: logger.exception("Failed to store destination_retry_timings") diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index daacc54c72bd..9d5010bf9231 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -28,6 +28,7 @@ make_deferred_yieldable, ) from synapse.util.async_helpers import ( + AwakenableSleeper, ObservableDeferred, concurrently_execute, delay_cancellation, @@ -35,6 +36,7 @@ timeout_deferred, ) +from tests.server import get_clock from tests.unittest import TestCase @@ -496,3 +498,81 @@ async def outer(): # logging context. blocking_d.callback(None) self.successResultOf(d) + + +class AwakenableSleeperTests(TestCase): + "Tests AwakenableSleeper" + + def test_sleep(self): + reactor, _ = get_clock() + sleeper = AwakenableSleeper(reactor) + + d = defer.ensureDeferred(sleeper.sleep("name", 1000)) + + reactor.pump([0.0]) + self.assertFalse(d.called) + + reactor.advance(0.5) + self.assertFalse(d.called) + + reactor.advance(0.6) + self.assertTrue(d.called) + + def test_explicit_wake(self): + reactor, _ = get_clock() + sleeper = AwakenableSleeper(reactor) + + d = defer.ensureDeferred(sleeper.sleep("name", 1000)) + + reactor.pump([0.0]) + self.assertFalse(d.called) + + reactor.advance(0.5) + self.assertFalse(d.called) + + sleeper.wake("name") + self.assertTrue(d.called) + + reactor.advance(0.6) + + def test_multiple_sleepers_timeout(self): + reactor, _ = get_clock() + sleeper = AwakenableSleeper(reactor) + + d1 = defer.ensureDeferred(sleeper.sleep("name", 1000)) + + reactor.advance(0.6) + self.assertFalse(d1.called) + + # Add another sleeper + d2 = defer.ensureDeferred(sleeper.sleep("name", 1000)) + + # Only the first sleep should time out now. + reactor.advance(0.6) + self.assertTrue(d1.called) + self.assertFalse(d2.called) + + reactor.advance(0.6) + self.assertTrue(d2.called) + + def test_multiple_sleepers_wake(self): + reactor, _ = get_clock() + sleeper = AwakenableSleeper(reactor) + + d1 = defer.ensureDeferred(sleeper.sleep("name", 1000)) + + reactor.advance(0.5) + self.assertFalse(d1.called) + + # Add another sleeper + d2 = defer.ensureDeferred(sleeper.sleep("name", 1000)) + + # Neither should fire yet + reactor.advance(0.3) + self.assertFalse(d1.called) + self.assertFalse(d2.called) + + # Explicitly waking both up works + sleeper.wake("name") + self.assertTrue(d1.called) + self.assertTrue(d2.called) From e5fd23fb6f02f655aa91b5a7be9e5bb839438e25 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 10:45:13 +0100 Subject: [PATCH 146/263] 1.59.0rc1 --- CHANGES.md | 92 +++++++++++++++++++++++++++++++++++++++ changelog.d/11507.feature | 1 - changelog.d/12168.feature | 1 - changelog.d/12273.bugfix | 1 - changelog.d/12356.misc | 1 - changelog.d/12406.feature | 1 - changelog.d/12452.feature | 1 - changelog.d/12480.misc | 1 - changelog.d/12485.misc | 1 - changelog.d/12500.misc | 1 - changelog.d/12505.misc | 1 - changelog.d/12526.feature | 1 - changelog.d/12531.misc | 1 - changelog.d/12536.doc | 1 - changelog.d/12541.docker | 1 - changelog.d/12544.bugfix | 1 - changelog.d/12556.misc | 1 - changelog.d/12564.misc | 1 - changelog.d/12568.misc | 1 - changelog.d/12570.bugfix | 1 - changelog.d/12573.docker | 1 - changelog.d/12576.misc | 1 - changelog.d/12577.misc | 1 - changelog.d/12579.doc | 1 - changelog.d/12580.bugfix | 1 - changelog.d/12581.misc | 1 - changelog.d/12582.misc | 1 - changelog.d/12587.misc | 1 - changelog.d/12589.misc | 1 - changelog.d/12594.bugfix | 1 - changelog.d/12596.removal | 1 - changelog.d/12597.removal | 2 - changelog.d/12599.misc | 1 - changelog.d/12601.feature | 1 - changelog.d/12602.misc | 1 - changelog.d/12608.misc | 1 - changelog.d/12610.misc | 1 - changelog.d/12612.bugfix | 1 - changelog.d/12613.removal | 1 - changelog.d/12614.misc | 1 - changelog.d/12616.misc | 1 - changelog.d/12619.feature | 1 - changelog.d/12620.misc | 1 - changelog.d/12621.doc | 1 - changelog.d/12624.misc | 1 - changelog.d/12627.doc | 1 - changelog.d/12632.misc | 1 - changelog.d/12633.bugfix | 1 - changelog.d/12635.feature | 1 - changelog.d/12636.feature | 1 - changelog.d/12637.misc | 1 - changelog.d/12639.bugfix | 1 - changelog.d/12650.misc | 1 - changelog.d/12652.misc | 1 - changelog.d/12656.misc | 1 - changelog.d/12657.bugfix | 1 - changelog.d/12663.misc | 1 - changelog.d/12664.doc | 1 - changelog.d/12665.misc | 1 - changelog.d/12666.misc | 1 - changelog.d/12667.misc | 1 - changelog.d/12670.feature | 1 - changelog.d/12671.misc | 1 - debian/changelog | 5 ++- pyproject.toml | 2 +- 65 files changed, 96 insertions(+), 66 deletions(-) delete mode 100644 changelog.d/11507.feature delete mode 100644 changelog.d/12168.feature delete mode 100644 changelog.d/12273.bugfix delete mode 100644 changelog.d/12356.misc delete mode 100644 changelog.d/12406.feature delete mode 100644 changelog.d/12452.feature delete mode 100644 changelog.d/12480.misc delete mode 100644 changelog.d/12485.misc delete mode 100644 changelog.d/12500.misc delete mode 100644 changelog.d/12505.misc delete mode 100644 changelog.d/12526.feature delete mode 100644 changelog.d/12531.misc delete mode 100644 changelog.d/12536.doc delete mode 100644 changelog.d/12541.docker delete mode 100644 changelog.d/12544.bugfix delete mode 100644 changelog.d/12556.misc delete mode 100644 changelog.d/12564.misc delete mode 100644 changelog.d/12568.misc delete mode 100644 changelog.d/12570.bugfix delete mode 100644 changelog.d/12573.docker delete mode 100644 changelog.d/12576.misc delete mode 100644 changelog.d/12577.misc delete mode 100644 changelog.d/12579.doc delete mode 100644 changelog.d/12580.bugfix delete mode 100644 changelog.d/12581.misc delete mode 100644 changelog.d/12582.misc delete mode 100644 changelog.d/12587.misc delete mode 100644 changelog.d/12589.misc delete mode 100644 changelog.d/12594.bugfix delete mode 100644 changelog.d/12596.removal delete mode 100644 changelog.d/12597.removal delete mode 100644 changelog.d/12599.misc delete mode 100644 changelog.d/12601.feature delete mode 100644 changelog.d/12602.misc delete mode 100644 changelog.d/12608.misc delete mode 100644 changelog.d/12610.misc delete mode 100644 changelog.d/12612.bugfix delete mode 100644 changelog.d/12613.removal delete mode 100644 changelog.d/12614.misc delete mode 100644 changelog.d/12616.misc delete mode 100644 changelog.d/12619.feature delete mode 100644 changelog.d/12620.misc delete mode 100644 changelog.d/12621.doc delete mode 100644 changelog.d/12624.misc delete mode 100644 changelog.d/12627.doc delete mode 100644 changelog.d/12632.misc delete mode 100644 changelog.d/12633.bugfix delete mode 100644 changelog.d/12635.feature delete mode 100644 changelog.d/12636.feature delete mode 100644 changelog.d/12637.misc delete mode 100644 changelog.d/12639.bugfix delete mode 100644 changelog.d/12650.misc delete mode 100644 changelog.d/12652.misc delete mode 100644 changelog.d/12656.misc delete mode 100644 changelog.d/12657.bugfix delete mode 100644 changelog.d/12663.misc delete mode 100644 changelog.d/12664.doc delete mode 100644 changelog.d/12665.misc delete mode 100644 changelog.d/12666.misc delete mode 100644 changelog.d/12667.misc delete mode 100644 changelog.d/12670.feature delete mode 100644 changelog.d/12671.misc diff --git a/CHANGES.md b/CHANGES.md index 88b053897e60..9376e838700e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,95 @@ +Synapse 1.59.0rc1 (2022-05-10) +============================== + +Features +-------- + +- Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. ([\#11507](https://github.com/matrix-org/synapse/issues/11507)) +- Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. ([\#12168](https://github.com/matrix-org/synapse/issues/12168), [\#12635](https://github.com/matrix-org/synapse/issues/12635), [\#12636](https://github.com/matrix-org/synapse/issues/12636), [\#12670](https://github.com/matrix-org/synapse/issues/12670)) +- Add a module API to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406)) +- Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) +- Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526)) +- Implement MSC3786: Add a default push rule to ignore m.room.server_acl events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) +- Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) +- Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) +- Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) +- Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580)) +- Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. ([\#12594](https://github.com/matrix-org/synapse/issues/12594)) +- Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612)) +- Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633)) +- Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12639](https://github.com/matrix-org/synapse/issues/12639)) +- Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. ([\#12657](https://github.com/matrix-org/synapse/issues/12657)) + + +Updates to the Docker image +--------------------------- + +- Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. ([\#12541](https://github.com/matrix-org/synapse/issues/12541)) +- Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by henryclw. ([\#12573](https://github.com/matrix-org/synapse/issues/12573)) + + +Improved Documentation +---------------------- + +- Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. ([\#12536](https://github.com/matrix-org/synapse/issues/12536)) +- Add missing linebreak to pipx install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579)) +- Add information about the TCP replication module to docs. ([\#12621](https://github.com/matrix-org/synapse/issues/12621)) +- Fixes to the formatting of README.rst. ([\#12627](https://github.com/matrix-org/synapse/issues/12627)) +- Fix docs on how to run specific Complement tests using the `complement.sh` test runner. ([\#12664](https://github.com/matrix-org/synapse/issues/12664)) + + +Deprecations and Removals +------------------------- + +- Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). ([\#12596](https://github.com/matrix-org/synapse/issues/12596)) +- Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from + [MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) +- Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. ([\#12613](https://github.com/matrix-org/synapse/issues/12613)) + + +Internal Changes +---------------- + +- Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356)) +- Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. ([\#12480](https://github.com/matrix-org/synapse/issues/12480)) +- Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485)) +- Immediately retry any requests that have backed off when a server comes back online. ([\#12500](https://github.com/matrix-org/synapse/issues/12500)) +- Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. ([\#12505](https://github.com/matrix-org/synapse/issues/12505)) +- Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531)) +- Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556)) +- Consistently check if an object is a `frozendict`. ([\#12564](https://github.com/matrix-org/synapse/issues/12564)) +- Protect module callbacks with read semantics against cancellation. ([\#12568](https://github.com/matrix-org/synapse/issues/12568)) +- Allow unused `#type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) +- Improve comments and error messages around access tokens. ([\#12577](https://github.com/matrix-org/synapse/issues/12577)) +- Improve docstrings for the receipts store. ([\#12581](https://github.com/matrix-org/synapse/issues/12581)) +- Use constants for read-receipts in tests. ([\#12582](https://github.com/matrix-org/synapse/issues/12582)) +- Log status code of cancelled requests as 499 and avoid logging stack traces for them. ([\#12587](https://github.com/matrix-org/synapse/issues/12587), [\#12663](https://github.com/matrix-org/synapse/issues/12663)) +- Remove special-case for `twisted` logger from default log config. ([\#12589](https://github.com/matrix-org/synapse/issues/12589)) +- Use `getClientAddress` instead of the deprecated `getClientIP`. ([\#12599](https://github.com/matrix-org/synapse/issues/12599)) +- Add link to documentation in Grafana Dashboard. ([\#12602](https://github.com/matrix-org/synapse/issues/12602)) +- Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608)) +- Reduce log spam when running multiple event persisters. ([\#12610](https://github.com/matrix-org/synapse/issues/12610)) +- Add extra debug logging to federation sender. ([\#12614](https://github.com/matrix-org/synapse/issues/12614)) +- Prevent remote homeservers from requesting local user device names by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- Add a consistency check on events which we read from the database. ([\#12620](https://github.com/matrix-org/synapse/issues/12620)) +- Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) +- Remove unused code related to receipts. ([\#12632](https://github.com/matrix-org/synapse/issues/12632)) +- Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637)) +- Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650)) +- Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652)) +- Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656)) +- Fix spelling of `M_UNRECOGNIZED` in comments. ([\#12665](https://github.com/matrix-org/synapse/issues/12665)) +- Use `Concatenate` to better annotate `_do_execute`. ([\#12666](https://github.com/matrix-org/synapse/issues/12666)) +- Use `ParamSpec` to refine type hints. ([\#12667](https://github.com/matrix-org/synapse/issues/12667)) +- Fix mypy against latest pillow stubs. ([\#12671](https://github.com/matrix-org/synapse/issues/12671)) + + Synapse 1.59.0 ============== diff --git a/changelog.d/11507.feature b/changelog.d/11507.feature deleted file mode 100644 index 72c5690cca4a..000000000000 --- a/changelog.d/11507.feature +++ /dev/null @@ -1 +0,0 @@ -Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. diff --git a/changelog.d/12168.feature b/changelog.d/12168.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12168.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12273.bugfix b/changelog.d/12273.bugfix deleted file mode 100644 index f8d7b6c88956..000000000000 --- a/changelog.d/12273.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations. diff --git a/changelog.d/12356.misc b/changelog.d/12356.misc deleted file mode 100644 index 43e192910602..000000000000 --- a/changelog.d/12356.misc +++ /dev/null @@ -1 +0,0 @@ -Fix scripts-dev to pass typechecking. \ No newline at end of file diff --git a/changelog.d/12406.feature b/changelog.d/12406.feature deleted file mode 100644 index e345afdee729..000000000000 --- a/changelog.d/12406.feature +++ /dev/null @@ -1 +0,0 @@ -Add a module API to allow modules to change actions for existing push rules of local users. diff --git a/changelog.d/12452.feature b/changelog.d/12452.feature deleted file mode 100644 index 22f054d34493..000000000000 --- a/changelog.d/12452.feature +++ /dev/null @@ -1 +0,0 @@ -Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. \ No newline at end of file diff --git a/changelog.d/12480.misc b/changelog.d/12480.misc deleted file mode 100644 index 18a85e7b1579..000000000000 --- a/changelog.d/12480.misc +++ /dev/null @@ -1 +0,0 @@ -Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. \ No newline at end of file diff --git a/changelog.d/12485.misc b/changelog.d/12485.misc deleted file mode 100644 index e793d08e5e3f..000000000000 --- a/changelog.d/12485.misc +++ /dev/null @@ -1 +0,0 @@ -Add some type hints to datastore. \ No newline at end of file diff --git a/changelog.d/12500.misc b/changelog.d/12500.misc deleted file mode 100644 index dbe3f7f5d197..000000000000 --- a/changelog.d/12500.misc +++ /dev/null @@ -1 +0,0 @@ -Immediately retry any requests that have backed off when a server comes back online. diff --git a/changelog.d/12505.misc b/changelog.d/12505.misc deleted file mode 100644 index a691d7962f89..000000000000 --- a/changelog.d/12505.misc +++ /dev/null @@ -1 +0,0 @@ -Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. diff --git a/changelog.d/12526.feature b/changelog.d/12526.feature deleted file mode 100644 index c01596282c9f..000000000000 --- a/changelog.d/12526.feature +++ /dev/null @@ -1 +0,0 @@ -Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. \ No newline at end of file diff --git a/changelog.d/12531.misc b/changelog.d/12531.misc deleted file mode 100644 index 412fc9b6dc70..000000000000 --- a/changelog.d/12531.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused `# type: ignore`s. diff --git a/changelog.d/12536.doc b/changelog.d/12536.doc deleted file mode 100644 index 4034c42076c4..000000000000 --- a/changelog.d/12536.doc +++ /dev/null @@ -1 +0,0 @@ -Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. diff --git a/changelog.d/12541.docker b/changelog.d/12541.docker deleted file mode 100644 index c3b9c31657cd..000000000000 --- a/changelog.d/12541.docker +++ /dev/null @@ -1 +0,0 @@ -Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. diff --git a/changelog.d/12544.bugfix b/changelog.d/12544.bugfix deleted file mode 100644 index b5169cd8311a..000000000000 --- a/changelog.d/12544.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. diff --git a/changelog.d/12556.misc b/changelog.d/12556.misc deleted file mode 100644 index dc245397fbcd..000000000000 --- a/changelog.d/12556.misc +++ /dev/null @@ -1 +0,0 @@ -Release script: confirm the commit to be tagged before tagging. diff --git a/changelog.d/12564.misc b/changelog.d/12564.misc deleted file mode 100644 index 207c3224645f..000000000000 --- a/changelog.d/12564.misc +++ /dev/null @@ -1 +0,0 @@ -Consistently check if an object is a `frozendict`. diff --git a/changelog.d/12568.misc b/changelog.d/12568.misc deleted file mode 100644 index f64dc67c4f9a..000000000000 --- a/changelog.d/12568.misc +++ /dev/null @@ -1 +0,0 @@ -Protect module callbacks with read semantics against cancellation. diff --git a/changelog.d/12570.bugfix b/changelog.d/12570.bugfix deleted file mode 100644 index 1038646f358d..000000000000 --- a/changelog.d/12570.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. diff --git a/changelog.d/12573.docker b/changelog.d/12573.docker deleted file mode 100644 index 5cc8de50acb3..000000000000 --- a/changelog.d/12573.docker +++ /dev/null @@ -1 +0,0 @@ -Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by henryclw. \ No newline at end of file diff --git a/changelog.d/12576.misc b/changelog.d/12576.misc deleted file mode 100644 index 71022c86337f..000000000000 --- a/changelog.d/12576.misc +++ /dev/null @@ -1 +0,0 @@ -Allow unused `#type: ignore` comments in bleeding edge CI jobs. diff --git a/changelog.d/12577.misc b/changelog.d/12577.misc deleted file mode 100644 index 8c4c47ad5292..000000000000 --- a/changelog.d/12577.misc +++ /dev/null @@ -1 +0,0 @@ -Improve comments and error messages around access tokens. \ No newline at end of file diff --git a/changelog.d/12579.doc b/changelog.d/12579.doc deleted file mode 100644 index bcec5fe1af24..000000000000 --- a/changelog.d/12579.doc +++ /dev/null @@ -1 +0,0 @@ -Add missing linebreak to pipx install instructions. diff --git a/changelog.d/12580.bugfix b/changelog.d/12580.bugfix deleted file mode 100644 index bedce405e2ae..000000000000 --- a/changelog.d/12580.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing. diff --git a/changelog.d/12581.misc b/changelog.d/12581.misc deleted file mode 100644 index 38d40b262b25..000000000000 --- a/changelog.d/12581.misc +++ /dev/null @@ -1 +0,0 @@ -Improve docstrings for the receipts store. diff --git a/changelog.d/12582.misc b/changelog.d/12582.misc deleted file mode 100644 index 5fa9c9afe86e..000000000000 --- a/changelog.d/12582.misc +++ /dev/null @@ -1 +0,0 @@ -Use constants for read-receipts in tests. diff --git a/changelog.d/12587.misc b/changelog.d/12587.misc deleted file mode 100644 index 3b466f1ddf16..000000000000 --- a/changelog.d/12587.misc +++ /dev/null @@ -1 +0,0 @@ -Log status code of cancelled requests as 499 and avoid logging stack traces for them. diff --git a/changelog.d/12589.misc b/changelog.d/12589.misc deleted file mode 100644 index d362828d2e59..000000000000 --- a/changelog.d/12589.misc +++ /dev/null @@ -1 +0,0 @@ -Remove special-case for `twisted` logger from default log config. diff --git a/changelog.d/12594.bugfix b/changelog.d/12594.bugfix deleted file mode 100644 index 7411d9c07934..000000000000 --- a/changelog.d/12594.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. diff --git a/changelog.d/12596.removal b/changelog.d/12596.removal deleted file mode 100644 index 14fbfb39540a..000000000000 --- a/changelog.d/12596.removal +++ /dev/null @@ -1 +0,0 @@ -Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069). diff --git a/changelog.d/12597.removal b/changelog.d/12597.removal deleted file mode 100644 index 7927f1d68d5f..000000000000 --- a/changelog.d/12597.removal +++ /dev/null @@ -1,2 +0,0 @@ -Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from -[MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778). diff --git a/changelog.d/12599.misc b/changelog.d/12599.misc deleted file mode 100644 index d01278bbce51..000000000000 --- a/changelog.d/12599.misc +++ /dev/null @@ -1 +0,0 @@ -Use `getClientAddress` instead of the deprecated `getClientIP`. diff --git a/changelog.d/12601.feature b/changelog.d/12601.feature deleted file mode 100644 index c13360ff35c3..000000000000 --- a/changelog.d/12601.feature +++ /dev/null @@ -1 +0,0 @@ -Implement MSC3786: Add a default push rule to ignore m.room.server_acl events. diff --git a/changelog.d/12602.misc b/changelog.d/12602.misc deleted file mode 100644 index cdccc5c31661..000000000000 --- a/changelog.d/12602.misc +++ /dev/null @@ -1 +0,0 @@ -Add link to documentation in Grafana Dashboard. diff --git a/changelog.d/12608.misc b/changelog.d/12608.misc deleted file mode 100644 index 38272118fbe8..000000000000 --- a/changelog.d/12608.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant lines of config from `mypy.ini`. \ No newline at end of file diff --git a/changelog.d/12610.misc b/changelog.d/12610.misc deleted file mode 100644 index 02efe0c72f51..000000000000 --- a/changelog.d/12610.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce log spam when running multiple event persisters. diff --git a/changelog.d/12612.bugfix b/changelog.d/12612.bugfix deleted file mode 100644 index c39e97f0cbaa..000000000000 --- a/changelog.d/12612.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a typo in the announcement text generated by the Synapse release development script. \ No newline at end of file diff --git a/changelog.d/12613.removal b/changelog.d/12613.removal deleted file mode 100644 index b1a9e207b00f..000000000000 --- a/changelog.d/12613.removal +++ /dev/null @@ -1 +0,0 @@ -Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk. diff --git a/changelog.d/12614.misc b/changelog.d/12614.misc deleted file mode 100644 index 79022df127d0..000000000000 --- a/changelog.d/12614.misc +++ /dev/null @@ -1 +0,0 @@ -Add extra debug logging to federation sender. diff --git a/changelog.d/12616.misc b/changelog.d/12616.misc deleted file mode 100644 index d17ce24cdf29..000000000000 --- a/changelog.d/12616.misc +++ /dev/null @@ -1 +0,0 @@ -Prevent remote homeservers from requesting local user device names by default. \ No newline at end of file diff --git a/changelog.d/12619.feature b/changelog.d/12619.feature deleted file mode 100644 index b0fc0f5fedf8..000000000000 --- a/changelog.d/12619.feature +++ /dev/null @@ -1 +0,0 @@ -Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. diff --git a/changelog.d/12620.misc b/changelog.d/12620.misc deleted file mode 100644 index 63f8e540c37c..000000000000 --- a/changelog.d/12620.misc +++ /dev/null @@ -1 +0,0 @@ -Add a consistency check on events which we read from the database. diff --git a/changelog.d/12621.doc b/changelog.d/12621.doc deleted file mode 100644 index d29fb9cb995e..000000000000 --- a/changelog.d/12621.doc +++ /dev/null @@ -1 +0,0 @@ -Add information about the TCP replication module to docs. diff --git a/changelog.d/12624.misc b/changelog.d/12624.misc deleted file mode 100644 index 8772d40fa70f..000000000000 --- a/changelog.d/12624.misc +++ /dev/null @@ -1 +0,0 @@ -Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh. diff --git a/changelog.d/12627.doc b/changelog.d/12627.doc deleted file mode 100644 index 3a787dfef275..000000000000 --- a/changelog.d/12627.doc +++ /dev/null @@ -1 +0,0 @@ -Fixes to the formatting of README.rst. diff --git a/changelog.d/12632.misc b/changelog.d/12632.misc deleted file mode 100644 index 9e4ba79c794f..000000000000 --- a/changelog.d/12632.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused code related to receipts. diff --git a/changelog.d/12633.bugfix b/changelog.d/12633.bugfix deleted file mode 100644 index 32332acd9a36..000000000000 --- a/changelog.d/12633.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. diff --git a/changelog.d/12635.feature b/changelog.d/12635.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12635.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12636.feature b/changelog.d/12636.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12636.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12637.misc b/changelog.d/12637.misc deleted file mode 100644 index 735257787fed..000000000000 --- a/changelog.d/12637.misc +++ /dev/null @@ -1 +0,0 @@ -Minor improvements to the scripts for running Synapse in worker mode under Complement. diff --git a/changelog.d/12639.bugfix b/changelog.d/12639.bugfix deleted file mode 100644 index c01596282c9f..000000000000 --- a/changelog.d/12639.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. \ No newline at end of file diff --git a/changelog.d/12650.misc b/changelog.d/12650.misc deleted file mode 100644 index 07bb4ce5a91a..000000000000 --- a/changelog.d/12650.misc +++ /dev/null @@ -1 +0,0 @@ -Update to mypy 0.950. \ No newline at end of file diff --git a/changelog.d/12652.misc b/changelog.d/12652.misc deleted file mode 100644 index 7b7c1cf5ff2a..000000000000 --- a/changelog.d/12652.misc +++ /dev/null @@ -1 +0,0 @@ -Move `pympler` back in to the `all` extras. diff --git a/changelog.d/12656.misc b/changelog.d/12656.misc deleted file mode 100644 index 8a8743e614f3..000000000000 --- a/changelog.d/12656.misc +++ /dev/null @@ -1 +0,0 @@ -Prevent memory leak from reoccurring when presence is disabled. diff --git a/changelog.d/12657.bugfix b/changelog.d/12657.bugfix deleted file mode 100644 index 7547ca40a724..000000000000 --- a/changelog.d/12657.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. diff --git a/changelog.d/12663.misc b/changelog.d/12663.misc deleted file mode 100644 index 3b466f1ddf16..000000000000 --- a/changelog.d/12663.misc +++ /dev/null @@ -1 +0,0 @@ -Log status code of cancelled requests as 499 and avoid logging stack traces for them. diff --git a/changelog.d/12664.doc b/changelog.d/12664.doc deleted file mode 100644 index 142d18037a12..000000000000 --- a/changelog.d/12664.doc +++ /dev/null @@ -1 +0,0 @@ -Fix docs on how to run specific Complement tests using the `complement.sh` test runner. diff --git a/changelog.d/12665.misc b/changelog.d/12665.misc deleted file mode 100644 index 37b96fea37d3..000000000000 --- a/changelog.d/12665.misc +++ /dev/null @@ -1 +0,0 @@ -Fix spelling of `M_UNRECOGNIZED` in comments. diff --git a/changelog.d/12666.misc b/changelog.d/12666.misc deleted file mode 100644 index 96268e33f56f..000000000000 --- a/changelog.d/12666.misc +++ /dev/null @@ -1 +0,0 @@ -Use `Concatenate` to better annotate `_do_execute`. diff --git a/changelog.d/12667.misc b/changelog.d/12667.misc deleted file mode 100644 index 2b17502d6b57..000000000000 --- a/changelog.d/12667.misc +++ /dev/null @@ -1 +0,0 @@ -Use `ParamSpec` to refine type hints. diff --git a/changelog.d/12670.feature b/changelog.d/12670.feature deleted file mode 100644 index cd5c45029ee1..000000000000 --- a/changelog.d/12670.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. diff --git a/changelog.d/12671.misc b/changelog.d/12671.misc deleted file mode 100644 index 56df4e383154..000000000000 --- a/changelog.d/12671.misc +++ /dev/null @@ -1 +0,0 @@ -Fix mypy against latest pillow stubs. diff --git a/debian/changelog b/debian/changelog index 5b21e0d369e0..fabc690bae93 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,11 @@ -matrix-synapse-py3 (1.58.2) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.59.0~rc1) stable; urgency=medium * Adjust how the `exported-requirements.txt` file is generated as part of the process of building these packages. This affects the package maintainers only; end-users are unaffected. + * New Synapse release 1.59.0rc1. - -- Synapse Packaging team Fri, 06 May 2022 13:49:29 +0100 + -- Synapse Packaging team Tue, 10 May 2022 10:45:08 +0100 matrix-synapse-py3 (1.58.1) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index 2c4b7eb08ecd..e3d81ae5f1e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.58.1" +version = "1.59.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 2cdac6f5854fa8cf2ace6db335a3459db5cde329 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:06:58 +0100 Subject: [PATCH 147/263] Adjust changelog --- CHANGES.md | 54 +++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9376e838700e..034045a58649 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,44 +1,52 @@ Synapse 1.59.0rc1 (2022-05-10) ============================== +This release makes several changes that server administrators should be aware of: + +- Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- The `synapse.app.appservice` is now deprecated. ([\#12452])(https://github.com/matrix-org/synapse/pull/12452)) + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. + +Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) + Features -------- - Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation. ([\#11507](https://github.com/matrix-org/synapse/issues/11507)) - Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. ([\#12168](https://github.com/matrix-org/synapse/issues/12168), [\#12635](https://github.com/matrix-org/synapse/issues/12635), [\#12636](https://github.com/matrix-org/synapse/issues/12636), [\#12670](https://github.com/matrix-org/synapse/issues/12670)) -- Add a module API to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406)) +- Extend the [module API](https://github.com/matrix-org/synapse/blob/release-v1.59/synapse/module_api/__init__.py) to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406)) - Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) - Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526)) -- Implement MSC3786: Add a default push rule to ignore m.room.server_acl events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) +- Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore m.room.server_acl events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) - Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619)) Bugfixes -------- -- Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) -- Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) -- Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) -- Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580)) +- Fix a bug introduced in Synapse v1.48.0 where the latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) +- Fix a bug introduced in Synapse v1.22.0 where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) +- Fix a bug introduced in Synapse v1.57.0 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) +- Fix a long-standing bug where status codes would almost always get logged as `200!`, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580)) - Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. ([\#12594](https://github.com/matrix-org/synapse/issues/12594)) -- Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612)) - Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633)) -- Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12639](https://github.com/matrix-org/synapse/issues/12639)) - Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. ([\#12657](https://github.com/matrix-org/synapse/issues/12657)) +- Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656)) Updates to the Docker image --------------------------- - Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments. ([\#12541](https://github.com/matrix-org/synapse/issues/12541)) -- Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by henryclw. ([\#12573](https://github.com/matrix-org/synapse/issues/12573)) +- Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by @henryclw. ([\#12573](https://github.com/matrix-org/synapse/issues/12573)) Improved Documentation ---------------------- - Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. ([\#12536](https://github.com/matrix-org/synapse/issues/12536)) -- Add missing linebreak to pipx install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579)) +- Add missing linebreak to `pipx` install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579)) - Add information about the TCP replication module to docs. ([\#12621](https://github.com/matrix-org/synapse/issues/12621)) - Fixes to the formatting of README.rst. ([\#12627](https://github.com/matrix-org/synapse/issues/12627)) - Fix docs on how to run specific Complement tests using the `complement.sh` test runner. ([\#12664](https://github.com/matrix-org/synapse/issues/12664)) @@ -56,16 +64,12 @@ Deprecations and Removals Internal Changes ---------------- -- Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356)) - Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. ([\#12480](https://github.com/matrix-org/synapse/issues/12480)) -- Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485)) - Immediately retry any requests that have backed off when a server comes back online. ([\#12500](https://github.com/matrix-org/synapse/issues/12500)) - Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. ([\#12505](https://github.com/matrix-org/synapse/issues/12505)) -- Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531)) - Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556)) - Consistently check if an object is a `frozendict`. ([\#12564](https://github.com/matrix-org/synapse/issues/12564)) - Protect module callbacks with read semantics against cancellation. ([\#12568](https://github.com/matrix-org/synapse/issues/12568)) -- Allow unused `#type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) - Improve comments and error messages around access tokens. ([\#12577](https://github.com/matrix-org/synapse/issues/12577)) - Improve docstrings for the receipts store. ([\#12581](https://github.com/matrix-org/synapse/issues/12581)) - Use constants for read-receipts in tests. ([\#12582](https://github.com/matrix-org/synapse/issues/12582)) @@ -73,29 +77,29 @@ Internal Changes - Remove special-case for `twisted` logger from default log config. ([\#12589](https://github.com/matrix-org/synapse/issues/12589)) - Use `getClientAddress` instead of the deprecated `getClientIP`. ([\#12599](https://github.com/matrix-org/synapse/issues/12599)) - Add link to documentation in Grafana Dashboard. ([\#12602](https://github.com/matrix-org/synapse/issues/12602)) -- Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608)) - Reduce log spam when running multiple event persisters. ([\#12610](https://github.com/matrix-org/synapse/issues/12610)) - Add extra debug logging to federation sender. ([\#12614](https://github.com/matrix-org/synapse/issues/12614)) - Prevent remote homeservers from requesting local user device names by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) - Add a consistency check on events which we read from the database. ([\#12620](https://github.com/matrix-org/synapse/issues/12620)) -- Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) +- Remove use of `constantly` library and switch to enums for `EventRedactBehaviour`. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) - Remove unused code related to receipts. ([\#12632](https://github.com/matrix-org/synapse/issues/12632)) - Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637)) -- Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650)) - Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652)) -- Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656)) - Fix spelling of `M_UNRECOGNIZED` in comments. ([\#12665](https://github.com/matrix-org/synapse/issues/12665)) +- Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612)) + +### Typechecking + +- Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356)) +- Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485)) +- Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531)) +- Allow unused `#type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) +- Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608)) +- Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650)) - Use `Concatenate` to better annotate `_do_execute`. ([\#12666](https://github.com/matrix-org/synapse/issues/12666)) - Use `ParamSpec` to refine type hints. ([\#12667](https://github.com/matrix-org/synapse/issues/12667)) - Fix mypy against latest pillow stubs. ([\#12671](https://github.com/matrix-org/synapse/issues/12671)) - -Synapse 1.59.0 -============== - -The non-standard `m.login.jwt` login type has been removed from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. - - Synapse 1.58.1 (2022-05-05) =========================== From 8ef0d85acd460a1e5f64feef89968acda9a9d140 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:07:44 +0100 Subject: [PATCH 148/263] Changelog typo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 034045a58649..cec3e29c4923 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -93,7 +93,7 @@ Internal Changes - Fix scripts-dev to pass typechecking. ([\#12356](https://github.com/matrix-org/synapse/issues/12356)) - Add some type hints to datastore. ([\#12485](https://github.com/matrix-org/synapse/issues/12485)) - Remove unused `# type: ignore`s. ([\#12531](https://github.com/matrix-org/synapse/issues/12531)) -- Allow unused `#type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) +- Allow unused `# type: ignore` comments in bleeding edge CI jobs. ([\#12576](https://github.com/matrix-org/synapse/issues/12576)) - Remove redundant lines of config from `mypy.ini`. ([\#12608](https://github.com/matrix-org/synapse/issues/12608)) - Update to mypy 0.950. ([\#12650](https://github.com/matrix-org/synapse/issues/12650)) - Use `Concatenate` to better annotate `_do_execute`. ([\#12666](https://github.com/matrix-org/synapse/issues/12666)) From 699192fc1a1055a4bec2345bc80f120f28470c73 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 10 May 2022 11:08:45 +0100 Subject: [PATCH 149/263] Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. (#12654) Co-authored-by: Shay --- changelog.d/12654.feature | 1 + docs/upgrade.md | 25 ++++++++++++++++--------- docs/workers.md | 26 +++++++++++++++++++++++--- synapse/app/admin_cmd.py | 2 +- synapse/app/generic_worker.py | 16 ---------------- synapse/config/server.py | 4 ---- synapse/config/workers.py | 7 +++++++ synapse/handlers/user_directory.py | 2 +- tests/config/test_workers.py | 27 +++++++++++++++++++++++++++ 9 files changed, 76 insertions(+), 34 deletions(-) create mode 100644 changelog.d/12654.feature diff --git a/changelog.d/12654.feature b/changelog.d/12654.feature new file mode 100644 index 000000000000..fd0a4bb0da86 --- /dev/null +++ b/changelog.d/12654.feature @@ -0,0 +1 @@ +Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index 18c33a4198ab..fa4b3ef5902d 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -101,29 +101,36 @@ To re-enable this functionality, set the homeserver config option to `true`. -## Deprecation of the `synapse.app.appservice` worker application type +## Deprecation of the `synapse.app.appservice` and `synapse.app.user_dir` worker application types The `synapse.app.appservice` worker application type allowed you to configure a single worker to use to notify application services of new events, as long as this functionality was disabled on the main process with `notify_appservices: False`. +Further, the `synapse.app.user_dir` worker application type allowed you to configure +a single worker to be responsible for updating the user directory, as long as this +was disabled on the main process with `update_user_directory: False`. To unify Synapse's worker types, the `synapse.app.appservice` worker application type and the `notify_appservices` configuration option have been deprecated. +The `synapse.app.user_dir` worker application type and `update_user_directory` +configuration option have also been deprecated. -To get the same functionality, it's now recommended that the `synapse.app.generic_worker` -worker application type is used and that the `notify_appservices_from_worker` option -is set to the name of a worker. +To get the same functionality as was provided by the deprecated options, it's now recommended that the `synapse.app.generic_worker` +worker application type is used and that the `notify_appservices_from_worker` and/or +`update_user_directory_from_worker` options are set to the name of a worker. -For the time being, `notify_appservices_from_worker` can be used alongside -`synapse.app.appservice` and `notify_appservices` to make it easier to transition -between the two configurations, however please note that: +For the time being, the old options can be used alongside the new options to make +it easier to transition between the two configurations, however please note that: - the options must not contradict each other (otherwise Synapse won't start); and -- the `notify_appservices` option will be removed in a future release of Synapse. +- the `notify_appservices` and `update_user_directory` options will be removed in a future release of Synapse. -Please see [the relevant section of the worker documentation][v1_59_notify_ases_from] for more information. +Please see the [*Notifying Application Services*][v1_59_notify_ases_from] and +[*Updating the User Directory*][v1_59_update_user_dir] sections of the worker +documentation for more information. [v1_59_notify_ases_from]: workers.md#notifying-application-services +[v1_59_update_user_dir]: workers.md#updating-the-user-directory # Upgrading to v1.58.0 diff --git a/docs/workers.md b/docs/workers.md index 1d049b6c4f28..553792d2384c 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -426,7 +426,7 @@ the shared configuration would include: run_background_tasks_on: background_worker ``` -You might also wish to investigate the `update_user_directory` and +You might also wish to investigate the `update_user_directory_from_worker` and `media_instance_running_background_jobs` settings. An example for a dedicated background worker instance: @@ -435,9 +435,26 @@ An example for a dedicated background worker instance: {{#include systemd-with-workers/workers/background_worker.yaml}} ``` +#### Updating the User Directory + +You can designate one generic worker to update the user directory. + +Specify its name in the shared configuration as follows: + +```yaml +update_user_directory_from_worker: worker_name +``` + +This work cannot be load-balanced; please ensure the main process is restarted +after setting this option in the shared configuration! + +This style of configuration supersedes the legacy `synapse.app.user_dir` +worker application type. + + #### Notifying Application Services -You can designate one worker to send output traffic to Application Services. +You can designate one generic worker to send output traffic to Application Services. Specify its name in the shared configuration as follows: @@ -470,7 +487,7 @@ pusher_instances: ### `synapse.app.appservice` -**Deprecated as of Synapse v1.58.** [Use `synapse.app.generic_worker` with the +**Deprecated as of Synapse v1.59.** [Use `synapse.app.generic_worker` with the `notify_appservices_from_worker` option instead.](#notifying-application-services) Handles sending output traffic to Application Services. Doesn't handle any @@ -540,6 +557,9 @@ Note that if a reverse proxy is used , then `/_matrix/media/` must be routed for ### `synapse.app.user_dir` +**Deprecated as of Synapse v1.59.** [Use `synapse.app.generic_worker` with the +`update_user_directory_from_worker` option instead.](#updating-the-user-directory) + Handles searches in the user directory. It can handle REST endpoints matching the following regular expressions: diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 2b0d92cbaedc..2a4c2e59cda3 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -210,7 +210,7 @@ def start(config_options: List[str]) -> None: config.logging.no_redirect_stdio = True # Explicitly disable background processes - config.server.update_user_directory = False + config.worker.should_update_user_directory = False config.worker.run_background_tasks = False config.worker.start_pushers = False config.worker.pusher_shard_config.instances = [] diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 07dddc0b1326..2a9480a5c161 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -441,22 +441,6 @@ def start(config_options: List[str]) -> None: "synapse.app.user_dir", ) - if config.worker.worker_app == "synapse.app.user_dir": - if config.server.update_user_directory: - sys.stderr.write( - "\nThe update_user_directory must be disabled in the main synapse process" - "\nbefore they can be run in a separate worker." - "\nPlease add ``update_user_directory: false`` to the main config" - "\n" - ) - sys.exit(1) - - # Force the pushers to start since they will be disabled in the main config - config.server.update_user_directory = True - else: - # For other worker types we force this to off. - config.server.update_user_directory = False - synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage diff --git a/synapse/config/server.py b/synapse/config/server.py index 1e709c7cf519..005a3ee48ce4 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -319,10 +319,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.presence_router_config, ) = load_module(presence_router_config, ("presence", "presence_router")) - # Whether to update the user directory or not. This should be set to - # false only if we are updating the user directory in a worker - self.update_user_directory = config.get("update_user_directory", True) - # whether to enable the media repository endpoints. This should be set # to false if the media repository is running as a separate endpoint; # doing so ensures that we will not run cache cleanup jobs on the diff --git a/synapse/config/workers.py b/synapse/config/workers.py index a9dbcc6d3dc7..e1569b3c14f9 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -311,6 +311,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: new_option_name="notify_appservices_from_worker", ) + self.should_update_user_directory = self._should_this_worker_perform_duty( + config, + legacy_master_option_name="update_user_directory", + legacy_worker_app_name="synapse.app.user_dir", + new_option_name="update_user_directory_from_worker", + ) + def _should_this_worker_perform_duty( self, config: Dict[str, Any], diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 048fd4bb8225..74f7fdfe6ce5 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -60,7 +60,7 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.notifier = hs.get_notifier() self.is_mine_id = hs.is_mine_id - self.update_user_directory = hs.config.server.update_user_directory + self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users self.spam_checker = hs.get_spam_checker() # The current position in the current_state_delta stream diff --git a/tests/config/test_workers.py b/tests/config/test_workers.py index da81bb9655fc..ef6294ecb2ce 100644 --- a/tests/config/test_workers.py +++ b/tests/config/test_workers.py @@ -286,3 +286,30 @@ def test_new_configs_appservice_worker(self) -> None: "notify_appservices_from_worker", ) ) + + def test_worker_duty_configs(self) -> None: + """ + Additional tests for the worker duties + """ + + worker1_config = self._make_worker_config( + worker_app="synapse.app.generic_worker", + worker_name="worker1", + extras={ + "notify_appservices_from_worker": "worker2", + "update_user_directory_from_worker": "worker1", + }, + ) + self.assertFalse(worker1_config.should_notify_appservices) + self.assertTrue(worker1_config.should_update_user_directory) + + worker2_config = self._make_worker_config( + worker_app="synapse.app.generic_worker", + worker_name="worker2", + extras={ + "notify_appservices_from_worker": "worker2", + "update_user_directory_from_worker": "worker1", + }, + ) + self.assertTrue(worker2_config.should_notify_appservices) + self.assertFalse(worker2_config.should_update_user_directory) From 464fe99f52e6384fe8a0a422977087e70d300db2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:12:13 +0100 Subject: [PATCH 150/263] Fix changelog link --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cec3e29c4923..eb2cfb18f14f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.59.0rc1 (2022-05-10) This release makes several changes that server administrators should be aware of: - Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) -- The `synapse.app.appservice` is now deprecated. ([\#12452])(https://github.com/matrix-org/synapse/pull/12452)) +- The `synapse.app.appservice` is now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. From 946b8437cfcf6c434b04eaa25d7899d3d738322f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:12:20 +0100 Subject: [PATCH 151/263] Group release script changes --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index eb2cfb18f14f..3b761c14e471 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -67,7 +67,6 @@ Internal Changes - Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. ([\#12480](https://github.com/matrix-org/synapse/issues/12480)) - Immediately retry any requests that have backed off when a server comes back online. ([\#12500](https://github.com/matrix-org/synapse/issues/12500)) - Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests. ([\#12505](https://github.com/matrix-org/synapse/issues/12505)) -- Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556)) - Consistently check if an object is a `frozendict`. ([\#12564](https://github.com/matrix-org/synapse/issues/12564)) - Protect module callbacks with read semantics against cancellation. ([\#12568](https://github.com/matrix-org/synapse/issues/12568)) - Improve comments and error messages around access tokens. ([\#12577](https://github.com/matrix-org/synapse/issues/12577)) @@ -86,6 +85,7 @@ Internal Changes - Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637)) - Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652)) - Fix spelling of `M_UNRECOGNIZED` in comments. ([\#12665](https://github.com/matrix-org/synapse/issues/12665)) +- Release script: confirm the commit to be tagged before tagging. ([\#12556](https://github.com/matrix-org/synapse/issues/12556)) - Fix a typo in the announcement text generated by the Synapse release development script. ([\#12612](https://github.com/matrix-org/synapse/issues/12612)) ### Typechecking From 239da21c1ae02f310da863dbb4a3da1a5404fbfd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:12:31 +0100 Subject: [PATCH 152/263] Add Olivier's last-minute merge --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 3b761c14e471..8f3a3909219c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ Features - Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner. ([\#12168](https://github.com/matrix-org/synapse/issues/12168), [\#12635](https://github.com/matrix-org/synapse/issues/12635), [\#12636](https://github.com/matrix-org/synapse/issues/12636), [\#12670](https://github.com/matrix-org/synapse/issues/12670)) - Extend the [module API](https://github.com/matrix-org/synapse/blob/release-v1.59/synapse/module_api/__init__.py) to allow modules to change actions for existing push rules of local users. ([\#12406](https://github.com/matrix-org/synapse/issues/12406)) - Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) +- Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. ([\#12654](https://github.com/matrix-org/synapse/issues/12654)) - Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526)) - Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore m.room.server_acl events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) - Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619)) From 2bae6d93c94e47e7ce9459bf17ee3ae18b2a0923 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:17:42 +0100 Subject: [PATCH 153/263] I manually added O's change, remove newsfile --- changelog.d/12654.feature | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/12654.feature diff --git a/changelog.d/12654.feature b/changelog.d/12654.feature deleted file mode 100644 index fd0a4bb0da86..000000000000 --- a/changelog.d/12654.feature +++ /dev/null @@ -1 +0,0 @@ -Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. \ No newline at end of file From 80b3246528d23329b7d3dcd230bd4728ba26068c Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:29:40 +0100 Subject: [PATCH 154/263] Fix deprecation notice --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8f3a3909219c..82cdf2b59379 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.59.0rc1 (2022-05-10) This release makes several changes that server administrators should be aware of: - Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) -- The `synapse.app.appservice` is now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) +- The `synapse.app.appservice` and `synapse.app.user_dir` worker application types are now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452), [\#12654](https://github.com/matrix-org/synapse/issues/12654)) See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. From c707ea736a2401ad898b1e4d43c4d75fbc93846d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:29:49 +0100 Subject: [PATCH 155/263] v1 -> 1 --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 82cdf2b59379..ea6005cc379c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,12 +26,12 @@ Features Bugfixes -------- -- Fix a bug introduced in Synapse v1.48.0 where the latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) -- Fix a bug introduced in Synapse v1.22.0 where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) -- Fix a bug introduced in Synapse v1.57.0 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) +- Fix a bug introduced in Synapse 1.48.0 where the latest thread reply provided failed to include the proper bundled aggregations. ([\#12273](https://github.com/matrix-org/synapse/issues/12273)) +- Fix a bug introduced in Synapse 1.22.0 where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper. ([\#12544](https://github.com/matrix-org/synapse/issues/12544)) +- Fix a bug introduced in Synapse 1.57.0 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation. ([\#12570](https://github.com/matrix-org/synapse/issues/12570)) - Fix a long-standing bug where status codes would almost always get logged as `200!`, irrespective of the actual status code, when clients disconnect before a request has finished processing. ([\#12580](https://github.com/matrix-org/synapse/issues/12580)) - Fix race when persisting an event and deleting a room that could lead to outbound federation breaking. ([\#12594](https://github.com/matrix-org/synapse/issues/12594)) -- Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633)) +- Fix a bug introduced in Synapse 1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated. ([\#12633](https://github.com/matrix-org/synapse/issues/12633)) - Fix a long-standing bug where rooms containing power levels with string values could not be upgraded. ([\#12657](https://github.com/matrix-org/synapse/issues/12657)) - Prevent memory leak from reoccurring when presence is disabled. ([\#12656](https://github.com/matrix-org/synapse/issues/12656)) From 735faab2b899f230bafd8c321ebb1e2486d22e6e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:30:20 +0100 Subject: [PATCH 156/263] backquote `m.room.server_acl` --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ea6005cc379c..dc4a30f8bc6c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,7 +19,7 @@ Features - Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services. ([\#12452](https://github.com/matrix-org/synapse/issues/12452)) - Add the `update_user_directory_from_worker` configuration option (superseding `update_user_directory`) to allow a generic worker to be designated as the worker to update the user directory. ([\#12654](https://github.com/matrix-org/synapse/issues/12654)) - Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid. ([\#12526](https://github.com/matrix-org/synapse/issues/12526)) -- Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore m.room.server_acl events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) +- Implement [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786): Add a default push rule to ignore `m.room.server_acl` events. ([\#12601](https://github.com/matrix-org/synapse/issues/12601)) - Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice. ([\#12619](https://github.com/matrix-org/synapse/issues/12619)) From efcd899f697a02c80e6ab5ef8e6985ce67e08e79 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 10 May 2022 11:31:10 +0100 Subject: [PATCH 157/263] other fixes --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dc4a30f8bc6c..c625e4d56119 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -49,7 +49,7 @@ Improved Documentation - Update SQL statements and replace use of old table `user_stats_historical` in docs for Synapse Admins. ([\#12536](https://github.com/matrix-org/synapse/issues/12536)) - Add missing linebreak to `pipx` install instructions. ([\#12579](https://github.com/matrix-org/synapse/issues/12579)) - Add information about the TCP replication module to docs. ([\#12621](https://github.com/matrix-org/synapse/issues/12621)) -- Fixes to the formatting of README.rst. ([\#12627](https://github.com/matrix-org/synapse/issues/12627)) +- Fixes to the formatting of `README.rst`. ([\#12627](https://github.com/matrix-org/synapse/issues/12627)) - Fix docs on how to run specific Complement tests using the `complement.sh` test runner. ([\#12664](https://github.com/matrix-org/synapse/issues/12664)) @@ -81,7 +81,7 @@ Internal Changes - Add extra debug logging to federation sender. ([\#12614](https://github.com/matrix-org/synapse/issues/12614)) - Prevent remote homeservers from requesting local user device names by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) - Add a consistency check on events which we read from the database. ([\#12620](https://github.com/matrix-org/synapse/issues/12620)) -- Remove use of `constantly` library and switch to enums for `EventRedactBehaviour`. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) +- Remove use of the `constantly` library and switch to enums for `EventRedactBehaviour`. Contributed by @andrewdoh. ([\#12624](https://github.com/matrix-org/synapse/issues/12624)) - Remove unused code related to receipts. ([\#12632](https://github.com/matrix-org/synapse/issues/12632)) - Minor improvements to the scripts for running Synapse in worker mode under Complement. ([\#12637](https://github.com/matrix-org/synapse/issues/12637)) - Move `pympler` back in to the `all` extras. ([\#12652](https://github.com/matrix-org/synapse/issues/12652)) From 02cdace707a129823c37d650b13b67fb3a7d7e24 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 10 May 2022 07:43:34 -0400 Subject: [PATCH 158/263] Add class-diagrams and notes for push. (#12676) --- changelog.d/12676.misc | 1 + synapse/push/__init__.py | 79 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 changelog.d/12676.misc diff --git a/changelog.d/12676.misc b/changelog.d/12676.misc new file mode 100644 index 000000000000..26490af00dee --- /dev/null +++ b/changelog.d/12676.misc @@ -0,0 +1 @@ +Improve documentation of the `synapse.push` module. diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index a1b771109848..d1dfb406d43a 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -12,6 +12,85 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module implements the push rules & notifications portion of the Matrix +specification. + +There's a few related features: + +* Push notifications (i.e. email or outgoing requests to a Push Gateway). +* Calculation of unread notifications (for /sync and /notifications). + +When Synapse receives a new event (locally, via the Client-Server API, or via +federation), the following occurs: + +1. The push rules get evaluated to generate a set of per-user actions. +2. The event is persisted into the database. +3. (In the background) The notifier is notified about the new event. + +The per-user actions are initially stored in the event_push_actions_staging table, +before getting moved into the event_push_actions table when the event is persisted. +The event_push_actions table is periodically summarised into the event_push_summary +and event_push_summary_stream_ordering tables. + +Since push actions block an event from being persisted the generation of push +actions is performance sensitive. + +The general interaction of the classes are: + + +---------------------------------------------+ + | FederationEventHandler/EventCreationHandler | + +---------------------------------------------+ + | + v + +-----------------+ + | ActionGenerator | + +-----------------+ + | + v + +-----------------------+ +---------------------------+ + | BulkPushRuleEvaluator |---->| PushRuleEvaluatorForEvent | + +-----------------------+ +---------------------------+ + | + v + +-----------------------------+ + | EventPushActionsWorkerStore | + +-----------------------------+ + +The notifier notifies the pusher pool of the new event, which checks for affected +users. Each user-configured pusher of the affected users then performs the +previously calculated action. + +The general interaction of the classes are: + + +----------+ + | Notifier | + +----------+ + | + v + +------------+ +--------------+ + | PusherPool |---->| PusherConfig | + +------------+ +--------------+ + | + | +---------------+ + +<--->| PusherFactory | + | +---------------+ + v + +------------------------+ +-----------------------------------------------+ + | EmailPusher/HttpPusher |---->| EventPushActionsWorkerStore/PusherWorkerStore | + +------------------------+ +-----------------------------------------------+ + | + v + +-------------------------+ + | Mailer/SimpleHttpClient | + +-------------------------+ + +The Pusher instance also calls out to various utilities for generating payloads +(or email templates), but those interactions are not detailed in this diagram +(and are specific to the type of pusher). + +""" + import abc from typing import TYPE_CHECKING, Any, Dict, Optional From b44fbdffa44cee752853ee16ad5604ec67667f92 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 10 May 2022 07:54:30 -0400 Subject: [PATCH 159/263] Move free functions into PushRuleEvaluatorForEvent. (#12677) * Move `_condition_checker` into `PushRuleEvaluatorForEvent`. * Move the condition cache into `PushRuleEvaluatorForEvent`. * Improve docstrings. * Inline a method which is only called once. --- changelog.d/12677.misc | 1 + synapse/push/bulk_push_rule_evaluator.py | 32 +---------- synapse/push/push_rule_evaluator.py | 70 ++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 34 deletions(-) create mode 100644 changelog.d/12677.misc diff --git a/changelog.d/12677.misc b/changelog.d/12677.misc new file mode 100644 index 000000000000..eed12e69e9ba --- /dev/null +++ b/changelog.d/12677.misc @@ -0,0 +1 @@ +Refactor functions to on `PushRuleEvaluatorForEvent`. diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index b07cf2eee705..85ddb56c6eb4 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -208,8 +208,6 @@ async def action_for_event_by_user( event, len(room_members), sender_power_level, power_levels ) - condition_cache: Dict[str, bool] = {} - # If the event is not a state event check if any users ignore the sender. if not event.is_state(): ignorers = await self.store.ignored_by(event.sender) @@ -247,8 +245,8 @@ async def action_for_event_by_user( if "enabled" in rule and not rule["enabled"]: continue - matches = _condition_checker( - evaluator, rule["conditions"], uid, display_name, condition_cache + matches = evaluator.check_conditions( + rule["conditions"], uid, display_name ) if matches: actions = [x for x in rule["actions"] if x != "dont_notify"] @@ -267,32 +265,6 @@ async def action_for_event_by_user( ) -def _condition_checker( - evaluator: PushRuleEvaluatorForEvent, - conditions: List[dict], - uid: str, - display_name: Optional[str], - cache: Dict[str, bool], -) -> bool: - for cond in conditions: - _cache_key = cond.get("_cache_key", None) - if _cache_key: - res = cache.get(_cache_key, None) - if res is False: - return False - elif res is True: - continue - - res = evaluator.matches(cond, uid, display_name) - if _cache_key: - cache[_cache_key] = bool(res) - - if not res: - return False - - return True - - MemberMap = Dict[str, Optional[EventIdMembership]] Rule = Dict[str, dict] RulesByUser = Dict[str, List[Rule]] diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index f617c759e6cf..54db6b5612a3 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -129,9 +129,55 @@ def __init__( # Maps strings of e.g. 'content.body' -> event["content"]["body"] self._value_cache = _flatten_dict(event) + # Maps cache keys to final values. + self._condition_cache: Dict[str, bool] = {} + + def check_conditions( + self, conditions: List[dict], uid: str, display_name: Optional[str] + ) -> bool: + """ + Returns true if a user's conditions/user ID/display name match the event. + + Args: + conditions: The user's conditions to match. + uid: The user's MXID. + display_name: The display name. + + Returns: + True if all conditions match the event, False otherwise. + """ + for cond in conditions: + _cache_key = cond.get("_cache_key", None) + if _cache_key: + res = self._condition_cache.get(_cache_key, None) + if res is False: + return False + elif res is True: + continue + + res = self.matches(cond, uid, display_name) + if _cache_key: + self._condition_cache[_cache_key] = bool(res) + + if not res: + return False + + return True + def matches( self, condition: Dict[str, Any], user_id: str, display_name: Optional[str] ) -> bool: + """ + Returns true if a user's condition/user ID/display name match the event. + + Args: + condition: The user's condition to match. + uid: The user's MXID. + display_name: The display name, or None if there is not one. + + Returns: + True if the condition matches the event, False otherwise. + """ if condition["kind"] == "event_match": return self._event_match(condition, user_id) elif condition["kind"] == "contains_display_name": @@ -146,6 +192,16 @@ def matches( return True def _event_match(self, condition: dict, user_id: str) -> bool: + """ + Check an "event_match" push rule condition. + + Args: + condition: The "event_match" push rule condition to match. + user_id: The user's MXID. + + Returns: + True if the condition matches the event, False otherwise. + """ pattern = condition.get("pattern", None) if not pattern: @@ -167,13 +223,22 @@ def _event_match(self, condition: dict, user_id: str) -> bool: return _glob_matches(pattern, body, word_boundary=True) else: - haystack = self._get_value(condition["key"]) + haystack = self._value_cache.get(condition["key"], None) if haystack is None: return False return _glob_matches(pattern, haystack) def _contains_display_name(self, display_name: Optional[str]) -> bool: + """ + Check an "event_match" push rule condition. + + Args: + display_name: The display name, or None if there is not one. + + Returns: + True if the display name is found in the event body, False otherwise. + """ if not display_name: return False @@ -191,9 +256,6 @@ def _contains_display_name(self, display_name: Optional[str]) -> bool: return bool(r.search(body)) - def _get_value(self, dotted_key: str) -> Optional[str]: - return self._value_cache.get(dotted_key, None) - # Caches (string, is_glob, word_boundary) -> regex for push. See _glob_matches regex_cache: LruCache[Tuple[str, bool, bool], Pattern] = LruCache( From 5c00151c28367cb091c408d02b275e7859bd4ace Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Tue, 10 May 2022 14:05:22 +0100 Subject: [PATCH 160/263] Add `@cancellable` decorator, for use on request handlers (#12586) Signed-off-by: Sean Quah --- changelog.d/12586.misc | 1 + synapse/http/server.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 changelog.d/12586.misc diff --git a/changelog.d/12586.misc b/changelog.d/12586.misc new file mode 100644 index 000000000000..d26e332305ce --- /dev/null +++ b/changelog.d/12586.misc @@ -0,0 +1 @@ +Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. diff --git a/synapse/http/server.py b/synapse/http/server.py index 657bffcddd88..8c96f2196eda 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,6 +33,7 @@ Optional, Pattern, Tuple, + TypeVar, Union, ) @@ -92,6 +93,66 @@ HTTP_STATUS_REQUEST_CANCELLED = 499 +F = TypeVar("F", bound=Callable[..., Any]) + + +_cancellable_method_names = frozenset( + { + # `RestServlet`, `BaseFederationServlet` and `BaseFederationServerServlet` + # methods + "on_GET", + "on_PUT", + "on_POST", + "on_DELETE", + # `_AsyncResource`, `DirectServeHtmlResource` and `DirectServeJsonResource` + # methods + "_async_render_GET", + "_async_render_PUT", + "_async_render_POST", + "_async_render_DELETE", + "_async_render_OPTIONS", + # `ReplicationEndpoint` methods + "_handle_request", + } +) + + +def cancellable(method: F) -> F: + """Marks a servlet method as cancellable. + + Methods with this decorator will be cancelled if the client disconnects before we + finish processing the request. + + During cancellation, `Deferred.cancel()` will be invoked on the `Deferred` wrapping + the method. The `cancel()` call will propagate down to the `Deferred` that is + currently being waited on. That `Deferred` will raise a `CancelledError`, which will + propagate up, as per normal exception handling. + + Before applying this decorator to a new endpoint, you MUST recursively check + that all `await`s in the function are on `async` functions or `Deferred`s that + handle cancellation cleanly, otherwise a variety of bugs may occur, ranging from + premature logging context closure, to stuck requests, to database corruption. + + Usage: + class SomeServlet(RestServlet): + @cancellable + async def on_GET(self, request: SynapseRequest) -> ...: + ... + """ + if method.__name__ not in _cancellable_method_names: + raise ValueError( + "@cancellable decorator can only be applied to servlet methods." + ) + + method.cancellable = True # type: ignore[attr-defined] + return method + + +def is_method_cancellable(method: Callable[..., Any]) -> bool: + """Checks whether a servlet method has the `@cancellable` flag.""" + return getattr(method, "cancellable", False) + + def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: """Sends a JSON error response to clients.""" From 5cfb0045955f8b5e9e8a1e0505fa2b5ed4f7bde2 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Tue, 10 May 2022 14:06:08 +0100 Subject: [PATCH 161/263] Add ability to cancel disconnected requests to `SynapseRequest` (#12588) Signed-off-by: Sean Quah --- changelog.d/12588.misc | 1 + synapse/http/site.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12588.misc diff --git a/changelog.d/12588.misc b/changelog.d/12588.misc new file mode 100644 index 000000000000..f62d5c8e210c --- /dev/null +++ b/changelog.d/12588.misc @@ -0,0 +1 @@ +Add ability to cancel disconnected requests to `SynapseRequest`. diff --git a/synapse/http/site.py b/synapse/http/site.py index 0b85a57d7787..f7f1c5704285 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -19,6 +19,7 @@ import attr from zope.interface import implementer +from twisted.internet.defer import Deferred from twisted.internet.interfaces import IAddress, IReactorTime from twisted.python.failure import Failure from twisted.web.http import HTTPChannel @@ -91,6 +92,13 @@ def __init__( # we can't yet create the logcontext, as we don't know the method. self.logcontext: Optional[LoggingContext] = None + # The `Deferred` to cancel if the client disconnects early. Expected to be set + # by `Resource.render`. + self.render_deferred: Optional["Deferred[None]"] = None + # A boolean indicating whether `_render_deferred` should be cancelled if the + # client disconnects early. Expected to be set during `Resource.render`. + self.is_render_cancellable = False + global _next_request_seq self.request_seq = _next_request_seq _next_request_seq += 1 @@ -357,7 +365,21 @@ def connectionLost(self, reason: Union[Failure, Exception]) -> None: {"event": "client connection lost", "reason": str(reason.value)} ) - if not self._is_processing: + if self._is_processing: + if self.is_render_cancellable: + if self.render_deferred is not None: + # Throw a cancellation into the request processing, in the hope + # that it will finish up sooner than it normally would. + # The `self.processing()` context manager will call + # `_finished_processing()` when done. + with PreserveLoggingContext(): + self.render_deferred.cancel() + else: + logger.error( + "Connection from client lost, but have no Deferred to " + "cancel even though the request is marked as cancellable." + ) + else: self._finished_processing() def _started_processing(self, servlet_name: str) -> None: From dbb12a0b547914024316b6eb510069e900680e42 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Tue, 10 May 2022 14:06:56 +0100 Subject: [PATCH 162/263] Add helper class for testing request cancellation (#12630) Also expose the `SynapseRequest` from `FakeChannel` in tests, so that we can call `Request.connectionLost` to simulate a client disconnecting. Signed-off-by: Sean Quah --- changelog.d/12630.misc | 1 + tests/http/server/__init__.py | 13 +++++ tests/http/server/_base.py | 100 ++++++++++++++++++++++++++++++++++ tests/server.py | 13 +++++ 4 files changed, 127 insertions(+) create mode 100644 changelog.d/12630.misc create mode 100644 tests/http/server/__init__.py create mode 100644 tests/http/server/_base.py diff --git a/changelog.d/12630.misc b/changelog.d/12630.misc new file mode 100644 index 000000000000..43e12603e2d8 --- /dev/null +++ b/changelog.d/12630.misc @@ -0,0 +1 @@ +Add a helper class for testing request cancellation. diff --git a/tests/http/server/__init__.py b/tests/http/server/__init__.py new file mode 100644 index 000000000000..3a5f22c02235 --- /dev/null +++ b/tests/http/server/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/http/server/_base.py b/tests/http/server/_base.py new file mode 100644 index 000000000000..b9f1a381aa2b --- /dev/null +++ b/tests/http/server/_base.py @@ -0,0 +1,100 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unles4s required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from http import HTTPStatus +from typing import Any, Callable, Optional, Union +from unittest import mock + +from twisted.internet.error import ConnectionDone + +from synapse.http.server import ( + HTTP_STATUS_REQUEST_CANCELLED, + respond_with_html_bytes, + respond_with_json, +) +from synapse.types import JsonDict + +from tests import unittest +from tests.server import FakeChannel, ThreadedMemoryReactorClock + + +class EndpointCancellationTestHelperMixin(unittest.TestCase): + """Provides helper methods for testing cancellation of endpoints.""" + + def _test_disconnect( + self, + reactor: ThreadedMemoryReactorClock, + channel: FakeChannel, + expect_cancellation: bool, + expected_body: Union[bytes, JsonDict], + expected_code: Optional[int] = None, + ) -> None: + """Disconnects an in-flight request and checks the response. + + Args: + reactor: The twisted reactor running the request handler. + channel: The `FakeChannel` for the request. + expect_cancellation: `True` if request processing is expected to be + cancelled, `False` if the request should run to completion. + expected_body: The expected response for the request. + expected_code: The expected status code for the request. Defaults to `200` + or `499` depending on `expect_cancellation`. + """ + # Determine the expected status code. + if expected_code is None: + if expect_cancellation: + expected_code = HTTP_STATUS_REQUEST_CANCELLED + else: + expected_code = HTTPStatus.OK + + request = channel.request + self.assertFalse( + channel.is_finished(), + "Request finished before we could disconnect - " + "was `await_result=False` passed to `make_request`?", + ) + + # We're about to disconnect the request. This also disconnects the channel, so + # we have to rely on mocks to extract the response. + respond_method: Callable[..., Any] + if isinstance(expected_body, bytes): + respond_method = respond_with_html_bytes + else: + respond_method = respond_with_json + + with mock.patch( + f"synapse.http.server.{respond_method.__name__}", wraps=respond_method + ) as respond_mock: + # Disconnect the request. + request.connectionLost(reason=ConnectionDone()) + + if expect_cancellation: + # An immediate cancellation is expected. + respond_mock.assert_called_once() + args, _kwargs = respond_mock.call_args + code, body = args[1], args[2] + self.assertEqual(code, expected_code) + self.assertEqual(request.code, expected_code) + self.assertEqual(body, expected_body) + else: + respond_mock.assert_not_called() + + # The handler is expected to run to completion. + reactor.pump([1.0]) + respond_mock.assert_called_once() + args, _kwargs = respond_mock.call_args + code, body = args[1], args[2] + self.assertEqual(code, expected_code) + self.assertEqual(request.code, expected_code) + self.assertEqual(body, expected_body) diff --git a/tests/server.py b/tests/server.py index 8f30e250c83c..aaefcfc46cd9 100644 --- a/tests/server.py +++ b/tests/server.py @@ -109,6 +109,17 @@ class FakeChannel: _ip: str = "127.0.0.1" _producer: Optional[Union[IPullProducer, IPushProducer]] = None resource_usage: Optional[ContextResourceUsage] = None + _request: Optional[Request] = None + + @property + def request(self) -> Request: + assert self._request is not None + return self._request + + @request.setter + def request(self, request: Request) -> None: + assert self._request is None + self._request = request @property def json_body(self): @@ -322,6 +333,8 @@ def make_request( channel = FakeChannel(site, reactor, ip=client_ip) req = request(channel, site) + channel.request = req + req.content = BytesIO(content) # Twisted expects to be at the end of the content when parsing the request. req.content.seek(0, SEEK_END) From 147f098fb4ac7ae435bae7d29c05f93b43472854 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 10 May 2022 15:35:08 +0100 Subject: [PATCH 163/263] Stop writing to `event_reference_hashes` (#12679) This table is never read, since #11794. We stop writing to it; in future we can drop it altogether. --- changelog.d/12679.misc | 1 + synapse/storage/databases/main/events.py | 25 ------------------- .../storage/databases/main/purge_events.py | 3 --- synapse/storage/schema/__init__.py | 5 +++- tests/storage/test_event_federation.py | 9 ------- 5 files changed, 5 insertions(+), 38 deletions(-) create mode 100644 changelog.d/12679.misc diff --git a/changelog.d/12679.misc b/changelog.d/12679.misc new file mode 100644 index 000000000000..6df1116b49ee --- /dev/null +++ b/changelog.d/12679.misc @@ -0,0 +1 @@ +Preparation for database schema simplifications: stop writing to `event_reference_hashes`. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index ed29a0a5e2db..ad611b2c0bb2 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -36,7 +36,6 @@ import synapse.metrics from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.room_versions import RoomVersions -from synapse.crypto.event_signing import compute_event_reference_hash from synapse.events import EventBase # noqa: F401 from synapse.events.snapshot import EventContext # noqa: F401 from synapse.storage._base import db_to_json, make_in_list_sql_clause @@ -1600,11 +1599,6 @@ def _update_metadata_tables_txn( inhibit_local_membership_updates=inhibit_local_membership_updates, ) - # Insert event_reference_hashes table. - self._store_event_reference_hashes_txn( - txn, [event for event, _ in events_and_contexts] - ) - # Prefill the event cache self._add_to_cache(txn, events_and_contexts) @@ -1704,25 +1698,6 @@ def _insert_event_expiry_txn(self, txn, event_id, expiry_ts): values={"event_id": event_id, "expiry_ts": expiry_ts}, ) - def _store_event_reference_hashes_txn(self, txn, events): - """Store a hash for a PDU - Args: - txn (cursor): - events (list): list of Events. - """ - - vals = [] - for event in events: - ref_alg, ref_hash_bytes = compute_event_reference_hash(event) - vals.append((event.event_id, ref_alg, memoryview(ref_hash_bytes))) - - self.db_pool.simple_insert_many_txn( - txn, - table="event_reference_hashes", - keys=("event_id", "algorithm", "hash"), - values=vals, - ) - def _store_room_members_txn( self, txn, events, *, inhibit_local_membership_updates: bool = False ): diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index bfc85b3add98..38ba91af4c47 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -69,7 +69,6 @@ def _purge_history_txn( # event_forward_extremities # event_json # event_push_actions - # event_reference_hashes # event_relations # event_search # event_to_state_groups @@ -220,7 +219,6 @@ def _purge_history_txn( "event_auth", "event_edges", "event_forward_extremities", - "event_reference_hashes", "event_relations", "event_search", "rejections", @@ -369,7 +367,6 @@ def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: "event_edges", "event_json", "event_push_actions_staging", - "event_reference_hashes", "event_relations", "event_to_state_groups", "event_auth_chains", diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 871d4ace123c..20c344faeab3 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCHEMA_VERSION = 69 # remember to update the list below when updating +SCHEMA_VERSION = 70 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -62,6 +62,9 @@ Changes in SCHEMA_VERSION = 69: - We now write to `device_lists_changes_in_room` table. - Use sequence to generate future `application_services_txns.txn_id`s + +Changes in SCHEMA_VERSION = 70: + - event_reference_hashes is no longer written to. """ diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 645d564d1c40..d92a9ac5b798 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -58,15 +58,6 @@ def insert_event(txn, i): (room_id, event_id), ) - txn.execute( - ( - "INSERT INTO event_reference_hashes " - "(event_id, algorithm, hash) " - "VALUES (?, 'sha256', ?)" - ), - (event_id, bytearray(b"ffff")), - ) - for i in range(0, 20): self.get_success( self.store.db_pool.runInteraction("insert", insert_event, i) From 989fa3309655e2ebd5416f4b09a98edfb1b2caa8 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 10 May 2022 20:07:48 +0200 Subject: [PATCH 164/263] Add some type hints to datastore. (#12477) --- changelog.d/12477.misc | 1 + synapse/events/snapshot.py | 3 +- synapse/storage/databases/main/events.py | 156 ++++++++++++++--------- synapse/storage/databases/main/search.py | 33 +++-- 4 files changed, 122 insertions(+), 71 deletions(-) create mode 100644 changelog.d/12477.misc diff --git a/changelog.d/12477.misc b/changelog.d/12477.misc new file mode 100644 index 000000000000..e793d08e5e3f --- /dev/null +++ b/changelog.d/12477.misc @@ -0,0 +1 @@ +Add some type hints to datastore. \ No newline at end of file diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 46042b2bf7af..8120c305df14 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -15,6 +15,7 @@ import attr from frozendict import frozendict +from typing_extensions import Literal from twisted.internet.defer import Deferred @@ -106,7 +107,7 @@ class EventContext: incomplete state. """ - rejected: Union[bool, str] = False + rejected: Union[Literal[False], str] = False _state_group: Optional[int] = None state_group_before_event: Optional[int] = None prev_group: Optional[int] = None diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index ad611b2c0bb2..6c12653bb3c6 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -49,7 +49,7 @@ from synapse.storage.engines.postgres import PostgresEngine from synapse.storage.util.id_generators import AbstractStreamIdGenerator from synapse.storage.util.sequence import SequenceGenerator -from synapse.types import StateMap, get_domain_from_id +from synapse.types import JsonDict, StateMap, get_domain_from_id from synapse.util import json_encoder from synapse.util.iterutils import batch_iter, sorted_topologically @@ -235,7 +235,9 @@ async def _get_events_which_are_prevs(self, event_ids: Iterable[str]) -> List[st """ results: List[str] = [] - def _get_events_which_are_prevs_txn(txn, batch): + def _get_events_which_are_prevs_txn( + txn: LoggingTransaction, batch: Collection[str] + ) -> None: sql = """ SELECT prev_event_id, internal_metadata FROM event_edges @@ -285,7 +287,9 @@ async def _get_prevs_before_rejected(self, event_ids: Iterable[str]) -> Set[str] # and their prev events. existing_prevs = set() - def _get_prevs_before_rejected_txn(txn, batch): + def _get_prevs_before_rejected_txn( + txn: LoggingTransaction, batch: Collection[str] + ) -> None: to_recursively_check = batch while to_recursively_check: @@ -515,7 +519,7 @@ def _persist_event_auth_chain_txn( @classmethod def _add_chain_cover_index( cls, - txn, + txn: LoggingTransaction, db_pool: DatabasePool, event_chain_id_gen: SequenceGenerator, event_to_room_id: Dict[str, str], @@ -809,7 +813,7 @@ def _add_chain_cover_index( @staticmethod def _allocate_chain_ids( - txn, + txn: LoggingTransaction, db_pool: DatabasePool, event_chain_id_gen: SequenceGenerator, event_to_room_id: Dict[str, str], @@ -943,7 +947,7 @@ def _persist_transaction_ids_txn( self, txn: LoggingTransaction, events_and_contexts: List[Tuple[EventBase, EventContext]], - ): + ) -> None: """Persist the mapping from transaction IDs to event IDs (if defined).""" to_insert = [] @@ -997,7 +1001,7 @@ def _update_current_state_txn( txn: LoggingTransaction, state_delta_by_room: Dict[str, DeltaState], stream_id: int, - ): + ) -> None: for room_id, delta_state in state_delta_by_room.items(): to_delete = delta_state.to_delete to_insert = delta_state.to_insert @@ -1155,7 +1159,7 @@ def _update_current_state_txn( txn, room_id, members_changed ) - def _upsert_room_version_txn(self, txn: LoggingTransaction, room_id: str): + def _upsert_room_version_txn(self, txn: LoggingTransaction, room_id: str) -> None: """Update the room version in the database based off current state events. @@ -1189,7 +1193,7 @@ def _update_forward_extremities_txn( txn: LoggingTransaction, new_forward_extremities: Dict[str, Set[str]], max_stream_order: int, - ): + ) -> None: for room_id in new_forward_extremities.keys(): self.db_pool.simple_delete_txn( txn, table="event_forward_extremities", keyvalues={"room_id": room_id} @@ -1254,9 +1258,9 @@ def _filter_events_and_contexts_for_duplicates( def _update_room_depths_txn( self, - txn, + txn: LoggingTransaction, events_and_contexts: List[Tuple[EventBase, EventContext]], - ): + ) -> None: """Update min_depth for each room Args: @@ -1385,7 +1389,7 @@ def _store_event_txn( # nothing to do here return - def event_dict(event): + def event_dict(event: EventBase) -> JsonDict: d = event.get_dict() d.pop("redacted", None) d.pop("redacted_because", None) @@ -1476,18 +1480,20 @@ def event_dict(event): ), ) - def _store_rejected_events_txn(self, txn, events_and_contexts): + def _store_rejected_events_txn( + self, + txn: LoggingTransaction, + events_and_contexts: List[Tuple[EventBase, EventContext]], + ) -> List[Tuple[EventBase, EventContext]]: """Add rows to the 'rejections' table for received events which were rejected Args: - txn (twisted.enterprise.adbapi.Connection): db connection - events_and_contexts (list[(EventBase, EventContext)]): events - we are persisting + txn: db connection + events_and_contexts: events we are persisting Returns: - list[(EventBase, EventContext)] new list, without the rejected - events. + new list, without the rejected events. """ # Remove the rejected events from the list now that we've added them # to the events table and the events_json table. @@ -1508,7 +1514,7 @@ def _update_metadata_tables_txn( events_and_contexts: List[Tuple[EventBase, EventContext]], all_events_and_contexts: List[Tuple[EventBase, EventContext]], inhibit_local_membership_updates: bool = False, - ): + ) -> None: """Update all the miscellaneous tables for new events Args: @@ -1602,7 +1608,11 @@ def _update_metadata_tables_txn( # Prefill the event cache self._add_to_cache(txn, events_and_contexts) - def _add_to_cache(self, txn, events_and_contexts): + def _add_to_cache( + self, + txn: LoggingTransaction, + events_and_contexts: List[Tuple[EventBase, EventContext]], + ) -> None: to_prefill = [] rows = [] @@ -1633,7 +1643,7 @@ def _add_to_cache(self, txn, events_and_contexts): if not row["rejects"] and not row["redacts"]: to_prefill.append(EventCacheEntry(event=event, redacted_event=None)) - def prefill(): + def prefill() -> None: for cache_entry in to_prefill: self.store._get_event_cache.set( (cache_entry.event.event_id,), cache_entry @@ -1663,19 +1673,24 @@ def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None: ) def insert_labels_for_event_txn( - self, txn, event_id, labels, room_id, topological_ordering - ): + self, + txn: LoggingTransaction, + event_id: str, + labels: List[str], + room_id: str, + topological_ordering: int, + ) -> None: """Store the mapping between an event's ID and its labels, with one row per (event_id, label) tuple. Args: - txn (LoggingTransaction): The transaction to execute. - event_id (str): The event's ID. - labels (list[str]): A list of text labels. - room_id (str): The ID of the room the event was sent to. - topological_ordering (int): The position of the event in the room's topology. + txn: The transaction to execute. + event_id: The event's ID. + labels: A list of text labels. + room_id: The ID of the room the event was sent to. + topological_ordering: The position of the event in the room's topology. """ - return self.db_pool.simple_insert_many_txn( + self.db_pool.simple_insert_many_txn( txn=txn, table="event_labels", keys=("event_id", "label", "room_id", "topological_ordering"), @@ -1684,25 +1699,32 @@ def insert_labels_for_event_txn( ], ) - def _insert_event_expiry_txn(self, txn, event_id, expiry_ts): + def _insert_event_expiry_txn( + self, txn: LoggingTransaction, event_id: str, expiry_ts: int + ) -> None: """Save the expiry timestamp associated with a given event ID. Args: - txn (LoggingTransaction): The database transaction to use. - event_id (str): The event ID the expiry timestamp is associated with. - expiry_ts (int): The timestamp at which to expire (delete) the event. + txn: The database transaction to use. + event_id: The event ID the expiry timestamp is associated with. + expiry_ts: The timestamp at which to expire (delete) the event. """ - return self.db_pool.simple_insert_txn( + self.db_pool.simple_insert_txn( txn=txn, table="event_expiry", values={"event_id": event_id, "expiry_ts": expiry_ts}, ) def _store_room_members_txn( - self, txn, events, *, inhibit_local_membership_updates: bool = False - ): + self, + txn: LoggingTransaction, + events: List[EventBase], + *, + inhibit_local_membership_updates: bool = False, + ) -> None: """ Store a room member in the database. + Args: txn: The transaction to use. events: List of events to store. @@ -1742,6 +1764,7 @@ def non_null_str_or_none(val: Any) -> Optional[str]: ) for event in events: + assert event.internal_metadata.stream_ordering is not None txn.call_after( self.store._membership_stream_cache.entity_has_changed, event.state_key, @@ -1838,7 +1861,9 @@ def _handle_event_relations( (parent_id, event.sender), ) - def _handle_insertion_event(self, txn: LoggingTransaction, event: EventBase): + def _handle_insertion_event( + self, txn: LoggingTransaction, event: EventBase + ) -> None: """Handles keeping track of insertion events and edges/connections. Part of MSC2716. @@ -1899,7 +1924,7 @@ def _handle_insertion_event(self, txn: LoggingTransaction, event: EventBase): }, ) - def _handle_batch_event(self, txn: LoggingTransaction, event: EventBase): + def _handle_batch_event(self, txn: LoggingTransaction, event: EventBase) -> None: """Handles inserting the batch edges/connections between the batch event and an insertion event. Part of MSC2716. @@ -1999,25 +2024,29 @@ def _handle_redact_relations( txn, table="event_relations", keyvalues={"event_id": redacted_event_id} ) - def _store_room_topic_txn(self, txn: LoggingTransaction, event: EventBase): + def _store_room_topic_txn(self, txn: LoggingTransaction, event: EventBase) -> None: if isinstance(event.content.get("topic"), str): self.store_event_search_txn( txn, event, "content.topic", event.content["topic"] ) - def _store_room_name_txn(self, txn: LoggingTransaction, event: EventBase): + def _store_room_name_txn(self, txn: LoggingTransaction, event: EventBase) -> None: if isinstance(event.content.get("name"), str): self.store_event_search_txn( txn, event, "content.name", event.content["name"] ) - def _store_room_message_txn(self, txn: LoggingTransaction, event: EventBase): + def _store_room_message_txn( + self, txn: LoggingTransaction, event: EventBase + ) -> None: if isinstance(event.content.get("body"), str): self.store_event_search_txn( txn, event, "content.body", event.content["body"] ) - def _store_retention_policy_for_room_txn(self, txn, event): + def _store_retention_policy_for_room_txn( + self, txn: LoggingTransaction, event: EventBase + ) -> None: if not event.is_state(): logger.debug("Ignoring non-state m.room.retention event") return @@ -2077,8 +2106,11 @@ def store_event_search_txn( ) def _set_push_actions_for_event_and_users_txn( - self, txn, events_and_contexts, all_events_and_contexts - ): + self, + txn: LoggingTransaction, + events_and_contexts: List[Tuple[EventBase, EventContext]], + all_events_and_contexts: List[Tuple[EventBase, EventContext]], + ) -> None: """Handles moving push actions from staging table to main event_push_actions table for all events in `events_and_contexts`. @@ -2086,12 +2118,10 @@ def _set_push_actions_for_event_and_users_txn( from the push action staging area. Args: - events_and_contexts (list[(EventBase, EventContext)]): events - we are persisting - all_events_and_contexts (list[(EventBase, EventContext)]): all - events that we were going to persist. This includes events - we've already persisted, etc, that wouldn't appear in - events_and_context. + events_and_contexts: events we are persisting + all_events_and_contexts: all events that we were going to persist. + This includes events we've already persisted, etc, that wouldn't + appear in events_and_context. """ # Only non outlier events will have push actions associated with them, @@ -2160,7 +2190,9 @@ def _set_push_actions_for_event_and_users_txn( ), ) - def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): + def _remove_push_actions_for_event_id_txn( + self, txn: LoggingTransaction, room_id: str, event_id: str + ) -> None: # Sad that we have to blow away the cache for the whole room here txn.call_after( self.store.get_unread_event_push_actions_by_room_for_user.invalidate, @@ -2171,7 +2203,9 @@ def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): (room_id, event_id), ) - def _store_rejections_txn(self, txn, event_id, reason): + def _store_rejections_txn( + self, txn: LoggingTransaction, event_id: str, reason: str + ) -> None: self.db_pool.simple_insert_txn( txn, table="rejections", @@ -2183,8 +2217,10 @@ def _store_rejections_txn(self, txn, event_id, reason): ) def _store_event_state_mappings_txn( - self, txn, events_and_contexts: Iterable[Tuple[EventBase, EventContext]] - ): + self, + txn: LoggingTransaction, + events_and_contexts: Collection[Tuple[EventBase, EventContext]], + ) -> None: state_groups = {} for event, context in events_and_contexts: if event.internal_metadata.is_outlier(): @@ -2241,7 +2277,9 @@ def _store_event_state_mappings_txn( state_group_id, ) - def _update_min_depth_for_room_txn(self, txn, room_id, depth): + def _update_min_depth_for_room_txn( + self, txn: LoggingTransaction, room_id: str, depth: int + ) -> None: min_depth = self.store._get_min_depth_interaction(txn, room_id) if min_depth is not None and depth >= min_depth: @@ -2254,7 +2292,9 @@ def _update_min_depth_for_room_txn(self, txn, room_id, depth): values={"min_depth": depth}, ) - def _handle_mult_prev_events(self, txn, events): + def _handle_mult_prev_events( + self, txn: LoggingTransaction, events: List[EventBase] + ) -> None: """ For the given event, update the event edges table and forward and backward extremities tables. @@ -2272,7 +2312,9 @@ def _handle_mult_prev_events(self, txn, events): self._update_backward_extremeties(txn, events) - def _update_backward_extremeties(self, txn, events): + def _update_backward_extremeties( + self, txn: LoggingTransaction, events: List[EventBase] + ) -> None: """Updates the event_backward_extremities tables based on the new/updated events being persisted. diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 3c49e7ec98e2..78e0773b2a88 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -14,7 +14,7 @@ import logging import re -from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Set +from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Set, Tuple import attr @@ -27,7 +27,7 @@ LoggingTransaction, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour -from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.types import JsonDict if TYPE_CHECKING: @@ -149,7 +149,9 @@ def __init__( self.EVENT_SEARCH_DELETE_NON_STRINGS, self._background_delete_non_strings ) - async def _background_reindex_search(self, progress, batch_size): + async def _background_reindex_search( + self, progress: JsonDict, batch_size: int + ) -> int: # we work through the events table from highest stream id to lowest target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] @@ -157,7 +159,7 @@ async def _background_reindex_search(self, progress, batch_size): TYPES = ["m.room.name", "m.room.message", "m.room.topic"] - def reindex_search_txn(txn): + def reindex_search_txn(txn: LoggingTransaction) -> int: sql = ( "SELECT stream_ordering, event_id, room_id, type, json, " " origin_server_ts FROM events" @@ -255,12 +257,14 @@ def reindex_search_txn(txn): return result - async def _background_reindex_gin_search(self, progress, batch_size): + async def _background_reindex_gin_search( + self, progress: JsonDict, batch_size: int + ) -> int: """This handles old synapses which used GIST indexes, if any; converting them back to be GIN as per the actual schema. """ - def create_index(conn): + def create_index(conn: LoggingDatabaseConnection) -> None: conn.rollback() # we have to set autocommit, because postgres refuses to @@ -299,7 +303,9 @@ def create_index(conn): ) return 1 - async def _background_reindex_search_order(self, progress, batch_size): + async def _background_reindex_search_order( + self, progress: JsonDict, batch_size: int + ) -> int: target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] rows_inserted = progress.get("rows_inserted", 0) @@ -307,7 +313,7 @@ async def _background_reindex_search_order(self, progress, batch_size): if not have_added_index: - def create_index(conn): + def create_index(conn: LoggingDatabaseConnection) -> None: conn.rollback() conn.set_session(autocommit=True) c = conn.cursor() @@ -336,7 +342,7 @@ def create_index(conn): pg, ) - def reindex_search_txn(txn): + def reindex_search_txn(txn: LoggingTransaction) -> Tuple[int, bool]: sql = ( "UPDATE event_search AS es SET stream_ordering = e.stream_ordering," " origin_server_ts = e.origin_server_ts" @@ -644,7 +650,8 @@ async def search_rooms( else: raise Exception("Unrecognized database engine") - args.append(limit) + # mypy expects to append only a `str`, not an `int` + args.append(limit) # type: ignore[arg-type] results = await self.db_pool.execute( "search_rooms", self.db_pool.cursor_to_dict, sql, *args @@ -705,7 +712,7 @@ async def _find_highlights_in_postgres( A set of strings. """ - def f(txn): + def f(txn: LoggingTransaction) -> Set[str]: highlight_words = set() for event in events: # As a hack we simply join values of all possible keys. This is @@ -759,11 +766,11 @@ def f(txn): return await self.db_pool.runInteraction("_find_highlights", f) -def _to_postgres_options(options_dict): +def _to_postgres_options(options_dict: JsonDict) -> str: return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),) -def _parse_query(database_engine, search_term): +def _parse_query(database_engine: BaseDatabaseEngine, search_term: str) -> str: """Takes a plain unicode string from the user and converts it into a form that can be passed to database. We use this so that we can add prefix matching, which isn't something From 29f06704b8871a44926f7c99e73cf4a978fb8e81 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 10 May 2022 14:10:22 -0400 Subject: [PATCH 165/263] Fix incorrect type hint in filtering code. (#12695) --- changelog.d/12695.misc | 1 + synapse/api/filtering.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12695.misc diff --git a/changelog.d/12695.misc b/changelog.d/12695.misc new file mode 100644 index 000000000000..1b39d969a4c5 --- /dev/null +++ b/changelog.d/12695.misc @@ -0,0 +1 @@ +Fixes an incorrect type hint for `Filter._check_event_relations`. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 4a808e33fee1..b91ce06de7c3 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -19,6 +19,7 @@ TYPE_CHECKING, Awaitable, Callable, + Collection, Dict, Iterable, List, @@ -444,9 +445,9 @@ def filter_rooms(self, room_ids: Iterable[str]) -> Set[str]: return room_ids async def _check_event_relations( - self, events: Iterable[FilterEvent] + self, events: Collection[FilterEvent] ) -> List[FilterEvent]: - # The event IDs to check, mypy doesn't understand the ifinstance check. + # The event IDs to check, mypy doesn't understand the isinstance check. event_ids = [event.event_id for event in events if isinstance(event, EventBase)] # type: ignore[attr-defined] event_ids_to_keep = set( await self._store.events_have_relations( From c997bfb926a29f0ec894fca889cc5eae603f4027 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Tue, 10 May 2022 20:39:05 +0100 Subject: [PATCH 166/263] Capture the `Deferred` for request cancellation in `_AsyncResource` (#12694) All async request processing goes through `_AsyncResource`, so this is the only place where a `Deferred` needs to be captured for cancellation. Unfortunately, the same isn't true for determining whether a request can be cancelled. Each of `RestServlet`, `BaseFederationServlet`, `DirectServe{Html,Json}Resource` and `ReplicationEndpoint` have different wrappers around the method doing the request handling and they all need to be handled separately. Signed-off-by: Sean Quah --- changelog.d/12694.misc | 1 + synapse/http/server.py | 4 +++- synapse/http/site.py | 9 +++++---- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12694.misc diff --git a/changelog.d/12694.misc b/changelog.d/12694.misc new file mode 100644 index 000000000000..e1e956a51301 --- /dev/null +++ b/changelog.d/12694.misc @@ -0,0 +1 @@ +Capture the `Deferred` for request cancellation in `_AsyncResource`. diff --git a/synapse/http/server.py b/synapse/http/server.py index 8c96f2196eda..4b4debc5cd2b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -344,7 +344,9 @@ def __init__(self, extract_context: bool = False): def render(self, request: SynapseRequest) -> int: """This gets called by twisted every time someone sends us a request.""" - defer.ensureDeferred(self._async_render_wrapper(request)) + request.render_deferred = defer.ensureDeferred( + self._async_render_wrapper(request) + ) return NOT_DONE_YET @wrap_async_request_handler diff --git a/synapse/http/site.py b/synapse/http/site.py index f7f1c5704285..eeec74b78ae5 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -92,11 +92,12 @@ def __init__( # we can't yet create the logcontext, as we don't know the method. self.logcontext: Optional[LoggingContext] = None - # The `Deferred` to cancel if the client disconnects early. Expected to be set - # by `Resource.render`. + # The `Deferred` to cancel if the client disconnects early and + # `is_render_cancellable` is set. Expected to be set by `Resource.render`. self.render_deferred: Optional["Deferred[None]"] = None - # A boolean indicating whether `_render_deferred` should be cancelled if the - # client disconnects early. Expected to be set during `Resource.render`. + # A boolean indicating whether `render_deferred` should be cancelled if the + # client disconnects early. Expected to be set by the coroutine started by + # `Resource.render`, if rendering is asynchronous. self.is_render_cancellable = False global _next_request_seq From c72d26c1e1e997e63cef1c474010a7db783f8022 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 May 2022 20:43:13 +0100 Subject: [PATCH 167/263] Refactor `EventContext` (#12689) Refactor how the `EventContext` class works, with the intention of reducing the amount of state we fetch from the DB during event processing. The idea here is to get rid of the cached `current_state_ids` and `prev_state_ids` that live in the `EventContext`, and instead defer straight to the database (and its caching). One change that may have a noticeable effect is that we now no longer prefill the `get_current_state_ids` cache on a state change. However, that query is relatively light, since its just a case of reading a table from the DB (unlike fetching state at an event which is more heavyweight). For deployments with workers this cache isn't even used. Part of #12684 --- changelog.d/12689.misc | 1 + synapse/events/snapshot.py | 177 ++++------------------- synapse/handlers/federation.py | 6 +- synapse/handlers/federation_event.py | 6 +- synapse/handlers/message.py | 6 +- synapse/push/action_generator.py | 4 + synapse/state/__init__.py | 9 +- synapse/storage/databases/main/events.py | 6 - synapse/storage/persist_events.py | 42 +----- tests/handlers/test_federation_event.py | 4 +- tests/storage/test_event_chain.py | 2 +- tests/test_state.py | 3 + tests/test_visibility.py | 4 +- 13 files changed, 70 insertions(+), 200 deletions(-) create mode 100644 changelog.d/12689.misc diff --git a/changelog.d/12689.misc b/changelog.d/12689.misc new file mode 100644 index 000000000000..daa484ea3019 --- /dev/null +++ b/changelog.d/12689.misc @@ -0,0 +1 @@ +Refactor `EventContext` class. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 8120c305df14..9ccd24b298bb 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -17,11 +17,8 @@ from frozendict import frozendict from typing_extensions import Literal -from twisted.internet.defer import Deferred - from synapse.appservice import ApplicationService from synapse.events import EventBase -from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.types import JsonDict, StateMap if TYPE_CHECKING: @@ -61,6 +58,9 @@ class EventContext: If ``state_group`` is None (ie, the event is an outlier), ``state_group_before_event`` will always also be ``None``. + state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None + then this is the delta of the state between the two groups. + prev_group: If it is known, ``state_group``'s prev_group. Note that this being None does not necessarily mean that ``state_group`` does not have a prev_group! @@ -79,73 +79,47 @@ class EventContext: app_service: If this event is being sent by a (local) application service, that app service. - _current_state_ids: The room state map, including this event - ie, the state - in ``state_group``. - - (type, state_key) -> event_id - - For an outlier, this is {} - - Note that this is a private attribute: it should be accessed via - ``get_current_state_ids``. _AsyncEventContext impl calculates this - on-demand: it will be None until that happens. - - _prev_state_ids: The room state map, excluding this event - ie, the state - in ``state_group_before_event``. For a non-state - event, this will be the same as _current_state_events. - - Note that it is a completely different thing to prev_group! - - (type, state_key) -> event_id - - For an outlier, this is {} - - As with _current_state_ids, this is a private attribute. It should be - accessed via get_prev_state_ids. - partial_state: if True, we may be storing this event with a temporary, incomplete state. """ + _storage: "Storage" rejected: Union[Literal[False], str] = False _state_group: Optional[int] = None state_group_before_event: Optional[int] = None + _state_delta_due_to_event: Optional[StateMap[str]] = None prev_group: Optional[int] = None delta_ids: Optional[StateMap[str]] = None app_service: Optional[ApplicationService] = None - _current_state_ids: Optional[StateMap[str]] = None - _prev_state_ids: Optional[StateMap[str]] = None - partial_state: bool = False @staticmethod def with_state( + storage: "Storage", state_group: Optional[int], state_group_before_event: Optional[int], - current_state_ids: Optional[StateMap[str]], - prev_state_ids: Optional[StateMap[str]], + state_delta_due_to_event: Optional[StateMap[str]], partial_state: bool, prev_group: Optional[int] = None, delta_ids: Optional[StateMap[str]] = None, ) -> "EventContext": return EventContext( - current_state_ids=current_state_ids, - prev_state_ids=prev_state_ids, + storage=storage, state_group=state_group, state_group_before_event=state_group_before_event, + state_delta_due_to_event=state_delta_due_to_event, prev_group=prev_group, delta_ids=delta_ids, partial_state=partial_state, ) @staticmethod - def for_outlier() -> "EventContext": + def for_outlier( + storage: "Storage", + ) -> "EventContext": """Return an EventContext instance suitable for persisting an outlier event""" - return EventContext( - current_state_ids={}, - prev_state_ids={}, - ) + return EventContext(storage=storage) async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: """Converts self to a type that can be serialized as JSON, and then @@ -158,24 +132,14 @@ async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict: The serialized event. """ - # We don't serialize the full state dicts, instead they get pulled out - # of the DB on the other side. However, the other side can't figure out - # the prev_state_ids, so if we're a state event we include the event - # id that we replaced in the state. - if event.is_state(): - prev_state_ids = await self.get_prev_state_ids() - prev_state_id = prev_state_ids.get((event.type, event.state_key)) - else: - prev_state_id = None - return { - "prev_state_id": prev_state_id, - "event_type": event.type, - "event_state_key": event.get_state_key(), "state_group": self._state_group, "state_group_before_event": self.state_group_before_event, "rejected": self.rejected, "prev_group": self.prev_group, + "state_delta_due_to_event": _encode_state_dict( + self._state_delta_due_to_event + ), "delta_ids": _encode_state_dict(self.delta_ids), "app_service_id": self.app_service.id if self.app_service else None, "partial_state": self.partial_state, @@ -193,16 +157,16 @@ def deserialize(storage: "Storage", input: JsonDict) -> "EventContext": Returns: The event context. """ - context = _AsyncEventContextImpl( + context = EventContext( # We use the state_group and prev_state_id stuff to pull the # current_state_ids out of the DB and construct prev_state_ids. storage=storage, - prev_state_id=input["prev_state_id"], - event_type=input["event_type"], - event_state_key=input["event_state_key"], state_group=input["state_group"], state_group_before_event=input["state_group_before_event"], prev_group=input["prev_group"], + state_delta_due_to_event=_decode_state_dict( + input["state_delta_due_to_event"] + ), delta_ids=_decode_state_dict(input["delta_ids"]), rejected=input["rejected"], partial_state=input.get("partial_state", False), @@ -250,8 +214,15 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: if self.rejected: raise RuntimeError("Attempt to access state_ids of rejected event") - await self._ensure_fetched() - return self._current_state_ids + assert self._state_delta_due_to_event is not None + + prev_state_ids = await self.get_prev_state_ids() + + if self._state_delta_due_to_event: + prev_state_ids = dict(prev_state_ids) + prev_state_ids.update(self._state_delta_due_to_event) + + return prev_state_ids async def get_prev_state_ids(self) -> StateMap[str]: """ @@ -266,94 +237,10 @@ async def get_prev_state_ids(self) -> StateMap[str]: Maps a (type, state_key) to the event ID of the state event matching this tuple. """ - await self._ensure_fetched() - # There *should* be previous state IDs now. - assert self._prev_state_ids is not None - return self._prev_state_ids - - def get_cached_current_state_ids(self) -> Optional[StateMap[str]]: - """Gets the current state IDs if we have them already cached. - - It is an error to access this for a rejected event, since rejected state should - not make it into the room state. This method will raise an exception if - ``rejected`` is set. - - Returns: - Returns None if we haven't cached the state or if state_group is None - (which happens when the associated event is an outlier). - - Otherwise, returns the the current state IDs. - """ - if self.rejected: - raise RuntimeError("Attempt to access state_ids of rejected event") - - return self._current_state_ids - - async def _ensure_fetched(self) -> None: - return None - - -@attr.s(slots=True) -class _AsyncEventContextImpl(EventContext): - """ - An implementation of EventContext which fetches _current_state_ids and - _prev_state_ids from the database on demand. - - Attributes: - - _storage - - _fetching_state_deferred: Resolves when *_state_ids have been calculated. - None if we haven't started calculating yet - - _event_type: The type of the event the context is associated with. - - _event_state_key: The state_key of the event the context is associated with. - - _prev_state_id: If the event associated with the context is a state event, - then `_prev_state_id` is the event_id of the state that was replaced. - """ - - # This needs to have a default as we're inheriting - _storage: "Storage" = attr.ib(default=None) - _prev_state_id: Optional[str] = attr.ib(default=None) - _event_type: str = attr.ib(default=None) - _event_state_key: Optional[str] = attr.ib(default=None) - _fetching_state_deferred: Optional["Deferred[None]"] = attr.ib(default=None) - - async def _ensure_fetched(self) -> None: - if not self._fetching_state_deferred: - self._fetching_state_deferred = run_in_background(self._fill_out_state) - - await make_deferred_yieldable(self._fetching_state_deferred) - - async def _fill_out_state(self) -> None: - """Called to populate the _current_state_ids and _prev_state_ids - attributes by loading from the database. - """ - if self.state_group is None: - # No state group means the event is an outlier. Usually the state_ids dicts are also - # pre-set to empty dicts, but they get reset when the context is serialized, so set - # them to empty dicts again here. - self._current_state_ids = {} - self._prev_state_ids = {} - return - - current_state_ids = await self._storage.state.get_state_ids_for_group( - self.state_group + assert self.state_group_before_event is not None + return await self._storage.state.get_state_ids_for_group( + self.state_group_before_event ) - # Set this separately so mypy knows current_state_ids is not None. - self._current_state_ids = current_state_ids - if self._event_state_key is not None: - self._prev_state_ids = dict(current_state_ids) - - key = (self._event_type, self._event_state_key) - if self._prev_state_id: - self._prev_state_ids[key] = self._prev_state_id - else: - self._prev_state_ids.pop(key, None) - else: - self._prev_state_ids = current_state_ids def _encode_state_dict( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 38dc5b1f6edf..be5099b507f6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -659,7 +659,7 @@ async def do_knock( # in the invitee's sync stream. It is stripped out for all other local users. event.unsigned["knock_room_state"] = stripped_room_state["knock_state_events"] - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) stream_id = await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -848,7 +848,7 @@ async def on_invite_request( ) ) - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) @@ -877,7 +877,7 @@ async def do_remotely_reject_invite( await self.federation_client.send_leave(host_list, event) - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) stream_id = await self._federation_event_handler.persist_events_and_notify( event.room_id, [(event, context)] ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 6cf927e4ff7b..6d11b32b61db 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1423,7 +1423,7 @@ def prep(event: EventBase) -> Optional[Tuple[EventBase, EventContext]]: # we're not bothering about room state, so flag the event as an outlier. event.internal_metadata.outlier = True - context = EventContext.for_outlier() + context = EventContext.for_outlier(self._storage) try: validate_event_for_room_version(room_version_obj, event) check_auth_rules_for_event(room_version_obj, event, auth) @@ -1874,10 +1874,10 @@ async def _update_context_for_auth_events( ) return EventContext.with_state( + storage=self._storage, state_group=state_group, state_group_before_event=context.state_group_before_event, - current_state_ids=current_state_ids, - prev_state_ids=prev_state_ids, + state_delta_due_to_event=state_updates, prev_group=prev_group, delta_ids=state_updates, partial_state=context.partial_state, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c28b792e6fe2..e47799e7f962 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -757,6 +757,10 @@ async def deduplicate_state_event( The previous version of the event is returned, if it is found in the event context. Otherwise, None is returned. """ + if event.internal_metadata.is_outlier(): + # This can happen due to out of band memberships + return None + prev_state_ids = await context.get_prev_state_ids() prev_event_id = prev_state_ids.get((event.type, event.state_key)) if not prev_event_id: @@ -1001,7 +1005,7 @@ async def create_new_client_event( # after it is created if builder.internal_metadata.outlier: event.internal_metadata.outlier = True - context = EventContext.for_outlier() + context = EventContext.for_outlier(self.storage) elif ( event.type == EventTypes.MSC2716_INSERTION and state_event_ids diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 60758df01664..730d9cd35463 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -40,5 +40,9 @@ def __init__(self, hs: "HomeServer"): async def handle_push_actions_for_event( self, event: EventBase, context: EventContext ) -> None: + if event.internal_metadata.is_outlier(): + # This can happen due to out of band memberships + return + with Measure(self.clock, "action_for_event_by_user"): await self.bulk_evaluator.action_for_event_by_user(event, context) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index cad3b4264007..54e41d537584 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -130,6 +130,7 @@ def __init__(self, hs: "HomeServer"): self.state_store = hs.get_storage().state self.hs = hs self._state_resolution_handler = hs.get_state_resolution_handler() + self._storage = hs.get_storage() @overload async def get_current_state( @@ -361,10 +362,10 @@ async def compute_event_context( if not event.is_state(): return EventContext.with_state( + storage=self._storage, state_group_before_event=state_group_before_event, state_group=state_group_before_event, - current_state_ids=state_ids_before_event, - prev_state_ids=state_ids_before_event, + state_delta_due_to_event={}, prev_group=state_group_before_event_prev_group, delta_ids=deltas_to_state_group_before_event, partial_state=partial_state, @@ -393,10 +394,10 @@ async def compute_event_context( ) return EventContext.with_state( + storage=self._storage, state_group=state_group_after_event, state_group_before_event=state_group_before_event, - current_state_ids=state_ids_after_event, - prev_state_ids=state_ids_before_event, + state_delta_due_to_event=delta_ids, prev_group=state_group_before_event, delta_ids=delta_ids, partial_state=partial_state, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 6c12653bb3c6..f544bcfff07f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -128,7 +128,6 @@ async def _persist_events_and_state_updates( self, events_and_contexts: List[Tuple[EventBase, EventContext]], *, - current_state_for_room: Dict[str, StateMap[str]], state_delta_for_room: Dict[str, DeltaState], new_forward_extremities: Dict[str, Set[str]], use_negative_stream_ordering: bool = False, @@ -139,8 +138,6 @@ async def _persist_events_and_state_updates( Args: events_and_contexts: - current_state_for_room: Map from room_id to the current state of - the room based on forward extremities state_delta_for_room: Map from room_id to the delta to apply to room state new_forward_extremities: Map from room_id to set of event IDs @@ -215,9 +212,6 @@ async def _persist_events_and_state_updates( event_counter.labels(event.type, origin_type, origin_entity).inc() - for room_id, new_state in current_state_for_room.items(): - self.store.get_current_state_ids.prefill((room_id,), new_state) - for room_id, latest_event_ids in new_forward_extremities.items(): self.store.get_latest_event_ids_in_room.prefill( (room_id,), list(latest_event_ids) diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index 97118045a1ad..a7f6338e058d 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -487,12 +487,6 @@ async def _persist_event_batch( # extremities in each room new_forward_extremities: Dict[str, Set[str]] = {} - # map room_id->(type,state_key)->event_id tracking the full - # state in each room after adding these events. - # This is simply used to prefill the get_current_state_ids - # cache - current_state_for_room: Dict[str, StateMap[str]] = {} - # map room_id->(to_delete, to_insert) where to_delete is a list # of type/state keys to remove from current state, and to_insert # is a map (type,key)->event_id giving the state delta in each @@ -628,14 +622,8 @@ async def _persist_event_batch( state_delta_for_room[room_id] = delta - # If we have the current_state then lets prefill - # the cache with it. - if current_state is not None: - current_state_for_room[room_id] = current_state - await self.persist_events_store._persist_events_and_state_updates( chunk, - current_state_for_room=current_state_for_room, state_delta_for_room=state_delta_for_room, new_forward_extremities=new_forward_extremities, use_negative_stream_ordering=backfilled, @@ -733,7 +721,8 @@ async def _get_new_state_after_events( The first state map is the full new current state and the second is the delta to the existing current state. If both are None then - there has been no change. + there has been no change. Either or neither can be None if there + has been a change. The function may prune some old entries from the set of new forward extremities if it's safe to do so. @@ -743,9 +732,6 @@ async def _get_new_state_after_events( the new current state is only returned if we've already calculated it. """ - # map from state_group to ((type, key) -> event_id) state map - state_groups_map = {} - # Map from (prev state group, new state group) -> delta state dict state_group_deltas = {} @@ -759,16 +745,6 @@ async def _get_new_state_after_events( ) continue - if ctx.state_group in state_groups_map: - continue - - # We're only interested in pulling out state that has already - # been cached in the context. We'll pull stuff out of the DB later - # if necessary. - current_state_ids = ctx.get_cached_current_state_ids() - if current_state_ids is not None: - state_groups_map[ctx.state_group] = current_state_ids - if ctx.prev_group: state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids @@ -826,18 +802,14 @@ async def _get_new_state_after_events( delta_ids = state_group_deltas.get((old_state_group, new_state_group), None) if delta_ids is not None: # We have a delta from the existing to new current state, - # so lets just return that. If we happen to already have - # the current state in memory then lets also return that, - # but it doesn't matter if we don't. - new_state = state_groups_map.get(new_state_group) - return new_state, delta_ids, new_latest_event_ids + # so lets just return that. + return None, delta_ids, new_latest_event_ids # Now that we have calculated new_state_groups we need to get # their state IDs so we can resolve to a single state set. - missing_state = new_state_groups - set(state_groups_map) - if missing_state: - group_to_state = await self.state_store._get_state_for_groups(missing_state) - state_groups_map.update(group_to_state) + state_groups_map = await self.state_store._get_state_for_groups( + new_state_groups + ) if len(new_state_groups) == 1: # If there is only one state group, then we know what the current diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 489ba5773672..e64b28f28b86 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -148,7 +148,9 @@ def _test_process_pulled_event_with_missing_state( prev_event.internal_metadata.outlier = True persistence = self.hs.get_storage().persistence self.get_success( - persistence.persist_event(prev_event, EventContext.for_outlier()) + persistence.persist_event( + prev_event, EventContext.for_outlier(self.hs.get_storage()) + ) ) else: diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py index 401020fd6361..c7661e71868f 100644 --- a/tests/storage/test_event_chain.py +++ b/tests/storage/test_event_chain.py @@ -393,7 +393,7 @@ def _persist(txn): # We need to persist the events to the events and state_events # tables. persist_events_store._store_event_txn( - txn, [(e, EventContext()) for e in events] + txn, [(e, EventContext(self.hs.get_storage())) for e in events] ) # Actually call the function that calculates the auth chain stuff. diff --git a/tests/test_state.py b/tests/test_state.py index e4baa6913746..651ec1c7d4bd 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -88,6 +88,9 @@ async def get_state_groups_ids(self, room_id, event_ids): return groups + async def get_state_ids_for_group(self, state_group): + return self._group_to_state[state_group] + async def store_state_group( self, event_id, room_id, prev_group, delta_ids, current_state_ids ): diff --git a/tests/test_visibility.py b/tests/test_visibility.py index d0230f9ebbc5..7a9b01ef9d44 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -234,7 +234,9 @@ def _inject_outlier(self) -> EventBase: event = self.get_success(builder.build(prev_event_ids=[], auth_event_ids=[])) event.internal_metadata.outlier = True self.get_success( - self.storage.persistence.persist_event(event, EventContext.for_outlier()) + self.storage.persistence.persist_event( + event, EventContext.for_outlier(self.storage) + ) ) return event From 84facf769eb79112be5f21942c18047b2b85f0bd Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 May 2022 23:39:14 -0500 Subject: [PATCH 168/263] Fix `/messages` throwing a 500 when querying for non-existent room (#12683) Fix https://github.com/matrix-org/synapse/issues/12678 Complement test added: https://github.com/matrix-org/complement/pull/369 **Before:** 500 internal server error **After:** According to the [spec](https://spec.matrix.org/latest/client-server-api/#get_matrixclientv3roomsroomidmessages), calling `/messages` against a non-existent `room_id` should throw a 403 forbidden (since you're not part of the room). This also matches the behavior before https://github.com/matrix-org/synapse/pull/12370 which regressed Synapse to the 500 behavior. ```json { "errcode": "M_FORBIDDEN", "error": "User @test:my.synapse.server not in room !dne:my.synapse.server, and room previews are disabled" } ``` --- changelog.d/12683.bugfix | 1 + synapse/handlers/pagination.py | 2 +- synapse/storage/databases/main/stream.py | 26 ++++++++++-------------- 3 files changed, 13 insertions(+), 16 deletions(-) create mode 100644 changelog.d/12683.bugfix diff --git a/changelog.d/12683.bugfix b/changelog.d/12683.bugfix new file mode 100644 index 000000000000..2ce84a223a37 --- /dev/null +++ b/changelog.d/12683.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.57.0 where `/messages` would throw a 500 error when querying for a non-existent room. diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 7ee334037376..2e30180094d2 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -448,7 +448,7 @@ async def get_messages( ) # We expect `/messages` to use historic pagination tokens by default but # `/messages` should still works with live tokens when manually provided. - assert from_token.room_key.topological + assert from_token.room_key.topological is not None if pagin_config.limit is None: # This shouldn't happen as we've set a default limit before this diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 793e906630e8..4e1d9647b7b8 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -785,22 +785,14 @@ async def get_last_event_in_room_before_stream_ordering( return None async def get_current_room_stream_token_for_room_id( - self, room_id: Optional[str] = None + self, room_id: str ) -> RoomStreamToken: - """Returns the current position of the rooms stream. - - By default, it returns a live token with the current global stream - token. Specifying a `room_id` causes it to return a historic token with - the room specific topological token. - """ + """Returns the current position of the rooms stream (historic token).""" stream_ordering = self.get_room_max_stream_ordering() - if room_id is None: - return RoomStreamToken(None, stream_ordering) - else: - topo = await self.db_pool.runInteraction( - "_get_max_topological_txn", self._get_max_topological_txn, room_id - ) - return RoomStreamToken(topo, stream_ordering) + topo = await self.db_pool.runInteraction( + "_get_max_topological_txn", self._get_max_topological_txn, room_id + ) + return RoomStreamToken(topo, stream_ordering) def get_stream_id_for_event_txn( self, @@ -870,7 +862,11 @@ def _get_max_topological_txn(self, txn: LoggingTransaction, room_id: str) -> int ) rows = txn.fetchall() - return rows[0][0] if rows else 0 + # An aggregate function like MAX() will always return one row per group + # so we can safely rely on the lookup here. For example, when a we + # lookup a `room_id` which does not exist, `rows` will look like + # `[(None,)]` + return rows[0][0] if rows[0][0] is not None else 0 @staticmethod def _set_before_and_after( From a4c75918b3e9cf48fa2bb91e9861f5f6fd74bd2e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 11 May 2022 07:15:21 -0400 Subject: [PATCH 169/263] Remove unneeded `ActionGenerator` class. (#12691) It simply passes through to `BulkPushRuleEvaluator`, which can be called directly instead. --- changelog.d/12691.misc | 1 + synapse/handlers/federation_event.py | 4 +- synapse/handlers/message.py | 6 ++- synapse/push/__init__.py | 5 --- synapse/push/action_generator.py | 48 ------------------------ synapse/push/bulk_push_rule_evaluator.py | 7 ++++ synapse/server.py | 6 +-- 7 files changed, 17 insertions(+), 60 deletions(-) create mode 100644 changelog.d/12691.misc delete mode 100644 synapse/push/action_generator.py diff --git a/changelog.d/12691.misc b/changelog.d/12691.misc new file mode 100644 index 000000000000..c63543421111 --- /dev/null +++ b/changelog.d/12691.misc @@ -0,0 +1 @@ +Remove an unneeded class in the push code. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 6d11b32b61db..761caa04b726 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -103,7 +103,7 @@ def __init__(self, hs: "HomeServer"): self._event_creation_handler = hs.get_event_creation_handler() self._event_auth_handler = hs.get_event_auth_handler() self._message_handler = hs.get_message_handler() - self._action_generator = hs.get_action_generator() + self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() self._state_resolution_handler = hs.get_state_resolution_handler() # avoid a circular dependency by deferring execution here self._get_room_member_handler = hs.get_room_member_handler @@ -1913,7 +1913,7 @@ async def _run_push_actions_and_persist_event( min_depth, ) else: - await self._action_generator.handle_push_actions_for_event( + await self._bulk_push_rule_evaluator.action_for_event_by_user( event, context ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e47799e7f962..4a4b535bae6a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -426,7 +426,7 @@ def __init__(self, hs: "HomeServer"): # This is to stop us from diverging history *too* much. self.limiter = Linearizer(max_count=5, name="room_event_creation_limit") - self.action_generator = hs.get_action_generator() + self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() self.spam_checker = hs.get_spam_checker() self.third_party_event_rules: "ThirdPartyEventRules" = ( @@ -1249,7 +1249,9 @@ async def _persist_event( # and `state_groups` because they have `prev_events` that aren't persisted yet # (historical messages persisted in reverse-chronological order). if not event.internal_metadata.is_historical(): - await self.action_generator.handle_push_actions_for_event(event, context) + await self._bulk_push_rule_evaluator.action_for_event_by_user( + event, context + ) try: # If we're a worker we need to hit out to the master. diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index d1dfb406d43a..57c4d70466b6 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -43,11 +43,6 @@ +---------------------------------------------+ | v - +-----------------+ - | ActionGenerator | - +-----------------+ - | - v +-----------------------+ +---------------------------+ | BulkPushRuleEvaluator |---->| PushRuleEvaluatorForEvent | +-----------------------+ +---------------------------+ diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py deleted file mode 100644 index 730d9cd35463..000000000000 --- a/synapse/push/action_generator.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import TYPE_CHECKING - -from synapse.events import EventBase -from synapse.events.snapshot import EventContext -from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator -from synapse.util.metrics import Measure - -if TYPE_CHECKING: - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - - -class ActionGenerator: - def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() - self.bulk_evaluator = BulkPushRuleEvaluator(hs) - # really we want to get all user ids and all profile tags too, - # since we want the actions for each profile tag for every user and - # also actions for a client with no profile tag for each user. - # Currently the event stream doesn't support profile tags on an - # event stream, so we just run the rules for a client with no profile - # tag (ie. we just need all the users). - - async def handle_push_actions_for_event( - self, event: EventBase, context: EventContext - ) -> None: - if event.internal_metadata.is_outlier(): - # This can happen due to out of band memberships - return - - with Measure(self.clock, "action_for_event_by_user"): - await self.bulk_evaluator.action_for_event_by_user(event, context) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 85ddb56c6eb4..0ffafc882b65 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -29,6 +29,7 @@ from synapse.util.caches import CacheMetric, register_cache from synapse.util.caches.descriptors import lru_cache from synapse.util.caches.lrucache import LruCache +from synapse.util.metrics import measure_func from .push_rule_evaluator import PushRuleEvaluatorForEvent @@ -105,6 +106,7 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs self.store = hs.get_datastores().main + self.clock = hs.get_clock() self._event_auth_handler = hs.get_event_auth_handler() # Used by `RulesForRoom` to ensure only one thing mutates the cache at a @@ -185,6 +187,7 @@ async def _get_power_levels_and_sender_level( return pl_event.content if pl_event else {}, sender_level + @measure_func("action_for_event_by_user") async def action_for_event_by_user( self, event: EventBase, context: EventContext ) -> None: @@ -192,6 +195,10 @@ async def action_for_event_by_user( should increment the unread count, and insert the results into the event_push_actions_staging table. """ + if event.internal_metadata.is_outlier(): + # This can happen due to out of band memberships + return + count_as_unread = _should_count_as_unread(event, context) rules_by_user = await self._get_rules_for_event(event, context) diff --git a/synapse/server.py b/synapse/server.py index d49c76518a8d..7daa7b9334c8 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -119,7 +119,7 @@ from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.module_api import ModuleApi from synapse.notifier import Notifier -from synapse.push.action_generator import ActionGenerator +from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator from synapse.push.pusherpool import PusherPool from synapse.replication.tcp.client import ReplicationDataHandler from synapse.replication.tcp.external_cache import ExternalCache @@ -644,8 +644,8 @@ def get_replication_command_handler(self) -> ReplicationCommandHandler: return ReplicationCommandHandler(self) @cache_in_self - def get_action_generator(self) -> ActionGenerator: - return ActionGenerator(self) + def get_bulk_push_rule_evaluator(self) -> BulkPushRuleEvaluator: + return BulkPushRuleEvaluator(self) @cache_in_self def get_user_directory_handler(self) -> UserDirectoryHandler: From dffecade7df8a88caced2a7707c51e2de3407c0d Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 11 May 2022 12:24:48 +0100 Subject: [PATCH 170/263] Respect the `@cancellable` flag for `DirectServe{Html,Json}Resource`s (#12698) `DirectServeHtmlResource` and `DirectServeJsonResource` both inherit from `_AsyncResource`. These classes expect to be subclassed with `_async_render_*` methods. This commit has no effect on `JsonResource`, despite inheriting from `_AsyncResource`. `JsonResource` has its own `_async_render` override which will need to be updated separately. Signed-off-by: Sean Quah --- changelog.d/12698.misc | 1 + synapse/http/server.py | 2 + tests/test_server.py | 111 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12698.misc diff --git a/changelog.d/12698.misc b/changelog.d/12698.misc new file mode 100644 index 000000000000..5d626352f9c2 --- /dev/null +++ b/changelog.d/12698.misc @@ -0,0 +1 @@ +Respect the `@cancellable` flag for `DirectServe{Html,Json}Resource`s. diff --git a/synapse/http/server.py b/synapse/http/server.py index 4b4debc5cd2b..f6d4d8db86fa 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -382,6 +382,8 @@ async def _async_render(self, request: SynapseRequest) -> Optional[Tuple[int, An method_handler = getattr(self, "_async_render_%s" % (request_method,), None) if method_handler: + request.is_render_cancellable = is_method_cancellable(method_handler) + raw_callback_return = method_handler(request) # Is it synchronous? We'll allow this for now. diff --git a/tests/test_server.py b/tests/test_server.py index f2ffbc895b88..0f1eb43cbced 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -13,18 +13,28 @@ # limitations under the License. import re +from http import HTTPStatus +from typing import Tuple from twisted.internet.defer import Deferred from twisted.web.resource import Resource from synapse.api.errors import Codes, RedirectException, SynapseError from synapse.config.server import parse_listener_def -from synapse.http.server import DirectServeHtmlResource, JsonResource, OptionsResource -from synapse.http.site import SynapseSite +from synapse.http.server import ( + DirectServeHtmlResource, + DirectServeJsonResource, + JsonResource, + OptionsResource, + cancellable, +) +from synapse.http.site import SynapseRequest, SynapseSite from synapse.logging.context import make_deferred_yieldable +from synapse.types import JsonDict from synapse.util import Clock from tests import unittest +from tests.http.server._base import EndpointCancellationTestHelperMixin from tests.server import ( FakeSite, ThreadedMemoryReactorClock, @@ -363,3 +373,100 @@ async def callback(request): self.assertEqual(channel.result["code"], b"200") self.assertNotIn("body", channel.result) + + +class CancellableDirectServeJsonResource(DirectServeJsonResource): + def __init__(self, clock: Clock): + super().__init__() + self.clock = clock + + @cancellable + async def _async_render_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + async def _async_render_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + +class CancellableDirectServeHtmlResource(DirectServeHtmlResource): + ERROR_TEMPLATE = "{code} {msg}" + + def __init__(self, clock: Clock): + super().__init__() + self.clock = clock + + @cancellable + async def _async_render_GET(self, request: SynapseRequest) -> Tuple[int, bytes]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, b"ok" + + async def _async_render_POST(self, request: SynapseRequest) -> Tuple[int, bytes]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, b"ok" + + +class DirectServeJsonResourceCancellationTests(EndpointCancellationTestHelperMixin): + """Tests for `DirectServeJsonResource` cancellation.""" + + def setUp(self): + self.reactor = ThreadedMemoryReactorClock() + self.clock = Clock(self.reactor) + self.resource = CancellableDirectServeJsonResource(self.clock) + self.site = FakeSite(self.resource, self.reactor) + + def test_cancellable_disconnect(self) -> None: + """Test that handlers with the `@cancellable` flag can be cancelled.""" + channel = make_request( + self.reactor, self.site, "GET", "/sleep", await_result=False + ) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=True, + expected_body={"error": "Request cancelled", "errcode": Codes.UNKNOWN}, + ) + + def test_uncancellable_disconnect(self) -> None: + """Test that handlers without the `@cancellable` flag cannot be cancelled.""" + channel = make_request( + self.reactor, self.site, "POST", "/sleep", await_result=False + ) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=False, + expected_body={"result": True}, + ) + + +class DirectServeHtmlResourceCancellationTests(EndpointCancellationTestHelperMixin): + """Tests for `DirectServeHtmlResource` cancellation.""" + + def setUp(self): + self.reactor = ThreadedMemoryReactorClock() + self.clock = Clock(self.reactor) + self.resource = CancellableDirectServeHtmlResource(self.clock) + self.site = FakeSite(self.resource, self.reactor) + + def test_cancellable_disconnect(self) -> None: + """Test that handlers with the `@cancellable` flag can be cancelled.""" + channel = make_request( + self.reactor, self.site, "GET", "/sleep", await_result=False + ) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=True, + expected_body=b"499 Request cancelled", + ) + + def test_uncancellable_disconnect(self) -> None: + """Test that handlers without the `@cancellable` flag cannot be cancelled.""" + channel = make_request( + self.reactor, self.site, "POST", "/sleep", await_result=False + ) + self._test_disconnect( + self.reactor, channel, expect_cancellation=False, expected_body=b"ok" + ) From 9d8e380d2e8267129de921b9b926257c36417cd2 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 11 May 2022 12:25:13 +0100 Subject: [PATCH 171/263] Respect the `@cancellable` flag for `RestServlet`s and `BaseFederationServlet`s (#12699) Both `RestServlet`s and `BaseFederationServlet`s register their handlers with `HttpServer.register_paths` / `JsonResource.register_paths`. Update `JsonResource` to respect the `@cancellable` flag on handlers registered in this way. Although `ReplicationEndpoint` also registers itself using `register_paths`, it does not pass the handler method that would have the `@cancellable` flag directly, and so needs separate handling. Signed-off-by: Sean Quah --- changelog.d/12699.misc | 1 + synapse/http/server.py | 5 + tests/federation/transport/server/__init__.py | 13 ++ .../federation/transport/server/test__base.py | 112 ++++++++++++++++++ tests/http/test_servlet.py | 60 +++++++++- tests/unittest.py | 2 +- 6 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12699.misc create mode 100644 tests/federation/transport/server/__init__.py create mode 100644 tests/federation/transport/server/test__base.py diff --git a/changelog.d/12699.misc b/changelog.d/12699.misc new file mode 100644 index 000000000000..d278a956c7a9 --- /dev/null +++ b/changelog.d/12699.misc @@ -0,0 +1 @@ +Respect the `@cancellable` flag for `RestServlet`s and `BaseFederationServlet`s. diff --git a/synapse/http/server.py b/synapse/http/server.py index f6d4d8db86fa..756c6e1aeeda 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -314,6 +314,9 @@ def register_paths( If the regex contains groups these gets passed to the callback via an unpacked tuple. + The callback may be marked with the `@cancellable` decorator, which will + cause request processing to be cancelled when clients disconnect early. + Args: method: The HTTP method to listen to. path_patterns: The regex used to match requests. @@ -544,6 +547,8 @@ def _get_handler_for_request( async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]: callback, servlet_classname, group_dict = self._get_handler_for_request(request) + request.is_render_cancellable = is_method_cancellable(callback) + # Make sure we have an appropriate name for this handler in prometheus # (rather than the default of JsonResource). request.request_metrics.name = servlet_classname diff --git a/tests/federation/transport/server/__init__.py b/tests/federation/transport/server/__init__.py new file mode 100644 index 000000000000..3a5f22c02235 --- /dev/null +++ b/tests/federation/transport/server/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/federation/transport/server/test__base.py b/tests/federation/transport/server/test__base.py new file mode 100644 index 000000000000..98a951f03e07 --- /dev/null +++ b/tests/federation/transport/server/test__base.py @@ -0,0 +1,112 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from http import HTTPStatus +from typing import Dict, List, Tuple + +from synapse.api.errors import Codes +from synapse.federation.transport.server import BaseFederationServlet +from synapse.federation.transport.server._base import Authenticator +from synapse.http.server import JsonResource, cancellable +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.ratelimitutils import FederationRateLimiter + +from tests import unittest +from tests.http.server._base import EndpointCancellationTestHelperMixin + + +class CancellableFederationServlet(BaseFederationServlet): + PATH = "/sleep" + + def __init__( + self, + hs: HomeServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + server_name: str, + ): + super().__init__(hs, authenticator, ratelimiter, server_name) + self.clock = hs.get_clock() + + @cancellable + async def on_GET( + self, origin: str, content: None, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + async def on_POST( + self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + +class BaseFederationServletCancellationTests( + unittest.FederatingHomeserverTestCase, EndpointCancellationTestHelperMixin +): + """Tests for `BaseFederationServlet` cancellation.""" + + path = f"{CancellableFederationServlet.PREFIX}{CancellableFederationServlet.PATH}" + + def create_test_resource(self): + """Overrides `HomeserverTestCase.create_test_resource`.""" + resource = JsonResource(self.hs) + + CancellableFederationServlet( + hs=self.hs, + authenticator=Authenticator(self.hs), + ratelimiter=self.hs.get_federation_ratelimiter(), + server_name=self.hs.hostname, + ).register(resource) + + return resource + + def test_cancellable_disconnect(self) -> None: + """Test that handlers with the `@cancellable` flag can be cancelled.""" + channel = self.make_signed_federation_request( + "GET", self.path, await_result=False + ) + + # Advance past all the rate limiting logic. If we disconnect too early, the + # request won't be processed. + self.pump() + + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=True, + expected_body={"error": "Request cancelled", "errcode": Codes.UNKNOWN}, + ) + + def test_uncancellable_disconnect(self) -> None: + """Test that handlers without the `@cancellable` flag cannot be cancelled.""" + channel = self.make_signed_federation_request( + "POST", + self.path, + content={}, + await_result=False, + ) + + # Advance past all the rate limiting logic. If we disconnect too early, the + # request won't be processed. + self.pump() + + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=False, + expected_body={"result": True}, + ) diff --git a/tests/http/test_servlet.py b/tests/http/test_servlet.py index a80bfb9f4eb5..ad521525cfaa 100644 --- a/tests/http/test_servlet.py +++ b/tests/http/test_servlet.py @@ -12,16 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +from http import HTTPStatus from io import BytesIO +from typing import Tuple from unittest.mock import Mock -from synapse.api.errors import SynapseError +from synapse.api.errors import Codes, SynapseError +from synapse.http.server import cancellable from synapse.http.servlet import ( + RestServlet, parse_json_object_from_request, parse_json_value_from_request, ) +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.server import HomeServer +from synapse.types import JsonDict from tests import unittest +from tests.http.server._base import EndpointCancellationTestHelperMixin def make_request(content): @@ -76,3 +85,52 @@ def test_parse_json_object(self): # Test not an object with self.assertRaises(SynapseError): parse_json_object_from_request(make_request(b'["foo"]')) + + +class CancellableRestServlet(RestServlet): + """A `RestServlet` with a mix of cancellable and uncancellable handlers.""" + + PATTERNS = client_patterns("/sleep$") + + def __init__(self, hs: HomeServer): + super().__init__() + self.clock = hs.get_clock() + + @cancellable + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + +class TestRestServletCancellation( + unittest.HomeserverTestCase, EndpointCancellationTestHelperMixin +): + """Tests for `RestServlet` cancellation.""" + + servlets = [ + lambda hs, http_server: CancellableRestServlet(hs).register(http_server) + ] + + def test_cancellable_disconnect(self) -> None: + """Test that handlers with the `@cancellable` flag can be cancelled.""" + channel = self.make_request("GET", "/sleep", await_result=False) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=True, + expected_body={"error": "Request cancelled", "errcode": Codes.UNKNOWN}, + ) + + def test_uncancellable_disconnect(self) -> None: + """Test that handlers without the `@cancellable` flag cannot be cancelled.""" + channel = self.make_request("POST", "/sleep", await_result=False) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=False, + expected_body={"result": True}, + ) diff --git a/tests/unittest.py b/tests/unittest.py index 9afa68c164ad..e7f255b4fa09 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -831,7 +831,7 @@ def make_signed_federation_request( self.site, method=method, path=path, - content=content or "", + content=content if content is not None else "", shorthand=False, await_result=await_result, custom_headers=custom_headers, From a559c8b0d939670b9d58dbeda6f3b1dd2f21937b Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 11 May 2022 12:25:39 +0100 Subject: [PATCH 172/263] Respect the `@cancellable` flag for `ReplicationEndpoint`s (#12700) While `ReplicationEndpoint`s register themselves via `JsonResource`, they pass a method that calls the handler, instead of the handler itself, to `register_paths`. As a result, `JsonResource` will not correctly pick up the `@cancellable` flag and we have to apply it ourselves. Signed-off-by: Sean Quah --- changelog.d/12700.misc | 1 + synapse/replication/http/_base.py | 21 +++++- tests/replication/http/__init__.py | 13 ++++ tests/replication/http/test__base.py | 106 +++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12700.misc create mode 100644 tests/replication/http/__init__.py create mode 100644 tests/replication/http/test__base.py diff --git a/changelog.d/12700.misc b/changelog.d/12700.misc new file mode 100644 index 000000000000..d93eb5dada74 --- /dev/null +++ b/changelog.d/12700.misc @@ -0,0 +1 @@ +Respect the `@cancellable` flag for `ReplicationEndpoint`s. diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 2bd244ed79df..a4ae4040c353 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -26,7 +26,8 @@ from synapse.api.errors import HttpResponseException, SynapseError from synapse.http import RequestTimedOutError -from synapse.http.server import HttpServer +from synapse.http.server import HttpServer, is_method_cancellable +from synapse.http.site import SynapseRequest from synapse.logging import opentracing from synapse.logging.opentracing import trace from synapse.types import JsonDict @@ -310,6 +311,12 @@ def register(self, http_server: HttpServer) -> None: url_args = list(self.PATH_ARGS) method = self.METHOD + if self.CACHE and is_method_cancellable(self._handle_request): + raise Exception( + f"{self.__class__.__name__} has been marked as cancellable, but CACHE " + "is set. The cancellable flag would have no effect." + ) + if self.CACHE: url_args.append("txn_id") @@ -324,7 +331,7 @@ def register(self, http_server: HttpServer) -> None: ) async def _check_auth_and_handle( - self, request: Request, **kwargs: Any + self, request: SynapseRequest, **kwargs: Any ) -> Tuple[int, JsonDict]: """Called on new incoming requests when caching is enabled. Checks if there is a cached response for the request and returns that, @@ -340,8 +347,18 @@ async def _check_auth_and_handle( if self.CACHE: txn_id = kwargs.pop("txn_id") + # We ignore the `@cancellable` flag, since cancellation wouldn't interupt + # `_handle_request` and `ResponseCache` does not handle cancellation + # correctly yet. In particular, there may be issues to do with logging + # context lifetimes. + return await self.response_cache.wrap( txn_id, self._handle_request, request, **kwargs ) + # The `@cancellable` decorator may be applied to `_handle_request`. But we + # told `HttpServer.register_paths` that our handler is `_check_auth_and_handle`, + # so we have to set up the cancellable flag ourselves. + request.is_render_cancellable = is_method_cancellable(self._handle_request) + return await self._handle_request(request, **kwargs) diff --git a/tests/replication/http/__init__.py b/tests/replication/http/__init__.py new file mode 100644 index 000000000000..3a5f22c02235 --- /dev/null +++ b/tests/replication/http/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/replication/http/test__base.py b/tests/replication/http/test__base.py new file mode 100644 index 000000000000..a5ab093a2722 --- /dev/null +++ b/tests/replication/http/test__base.py @@ -0,0 +1,106 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from http import HTTPStatus +from typing import Tuple + +from twisted.web.server import Request + +from synapse.api.errors import Codes +from synapse.http.server import JsonResource, cancellable +from synapse.replication.http import REPLICATION_PREFIX +from synapse.replication.http._base import ReplicationEndpoint +from synapse.server import HomeServer +from synapse.types import JsonDict + +from tests import unittest +from tests.http.server._base import EndpointCancellationTestHelperMixin + + +class CancellableReplicationEndpoint(ReplicationEndpoint): + NAME = "cancellable_sleep" + PATH_ARGS = () + CACHE = False + + def __init__(self, hs: HomeServer): + super().__init__(hs) + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload() -> JsonDict: + return {} + + @cancellable + async def _handle_request( # type: ignore[override] + self, request: Request + ) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + +class UncancellableReplicationEndpoint(ReplicationEndpoint): + NAME = "uncancellable_sleep" + PATH_ARGS = () + CACHE = False + + def __init__(self, hs: HomeServer): + super().__init__(hs) + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload() -> JsonDict: + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request + ) -> Tuple[int, JsonDict]: + await self.clock.sleep(1.0) + return HTTPStatus.OK, {"result": True} + + +class ReplicationEndpointCancellationTestCase( + unittest.HomeserverTestCase, EndpointCancellationTestHelperMixin +): + """Tests for `ReplicationEndpoint` cancellation.""" + + def create_test_resource(self): + """Overrides `HomeserverTestCase.create_test_resource`.""" + resource = JsonResource(self.hs) + + CancellableReplicationEndpoint(self.hs).register(resource) + UncancellableReplicationEndpoint(self.hs).register(resource) + + return resource + + def test_cancellable_disconnect(self) -> None: + """Test that handlers with the `@cancellable` flag can be cancelled.""" + path = f"{REPLICATION_PREFIX}/{CancellableReplicationEndpoint.NAME}/" + channel = self.make_request("POST", path, await_result=False) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=True, + expected_body={"error": "Request cancelled", "errcode": Codes.UNKNOWN}, + ) + + def test_uncancellable_disconnect(self) -> None: + """Test that handlers without the `@cancellable` flag cannot be cancelled.""" + path = f"{REPLICATION_PREFIX}/{UncancellableReplicationEndpoint.NAME}/" + channel = self.make_request("POST", path, await_result=False) + self._test_disconnect( + self.reactor, + channel, + expect_cancellation=False, + expected_body={"result": True}, + ) From d38d242411b8910dfacde1e61fd3a0ec5cbcaa66 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 11 May 2022 14:43:22 +0100 Subject: [PATCH 173/263] Reload cache factors from disk on SIGHUP (#12673) --- changelog.d/12673.feature | 1 + docs/sample_config.yaml | 6 ++ .../configuration/config_documentation.md | 17 ++++ synapse/app/_base.py | 44 ++++++++++ synapse/app/homeserver.py | 36 +-------- synapse/config/_base.py | 81 +++++++++++++++++-- synapse/config/_base.pyi | 15 +++- synapse/config/cache.py | 49 ++++++----- synapse/http/client.py | 2 +- tests/config/test_cache.py | 8 ++ tests/server.py | 1 + 11 files changed, 199 insertions(+), 61 deletions(-) create mode 100644 changelog.d/12673.feature diff --git a/changelog.d/12673.feature b/changelog.d/12673.feature new file mode 100644 index 000000000000..f2bddd6e1c27 --- /dev/null +++ b/changelog.d/12673.feature @@ -0,0 +1 @@ +Synapse will now reload [cache config](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#caching) when it receives a [SIGHUP](https://en.wikipedia.org/wiki/SIGHUP) signal. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index a803b8261dcd..e7b57f5a0bdf 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -730,6 +730,12 @@ retention: # A cache 'factor' is a multiplier that can be applied to each of # Synapse's caches in order to increase or decrease the maximum # number of entries that can be stored. +# +# The configuration for cache factors (caches.global_factor and +# caches.per_cache_factors) can be reloaded while the application is running, +# by sending a SIGHUP signal to the Synapse process. Changes to other parts of +# the caching config will NOT be applied after a SIGHUP is received; a restart +# is necessary. # The number of events to cache in memory. Not affected by # caches.global_factor. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 21dad0ac41e2..f292b94fb0cd 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1130,6 +1130,23 @@ caches: expire_caches: false sync_response_cache_duration: 2m ``` + +### Reloading cache factors + +The cache factors (i.e. `caches.global_factor` and `caches.per_cache_factors`) may be reloaded at any time by sending a +[`SIGHUP`](https://en.wikipedia.org/wiki/SIGHUP) signal to Synapse using e.g. + +```commandline +kill -HUP [PID_OF_SYNAPSE_PROCESS] +``` + +If you are running multiple workers, you must individually update the worker +config file and send this signal to each worker process. + +If you're using the [example systemd service](https://github.com/matrix-org/synapse/blob/develop/contrib/systemd/matrix-synapse.service) +file in Synapse's `contrib` directory, you can send a `SIGHUP` signal by using +`systemctl reload matrix-synapse`. + --- ## Database ## Config options related to database settings. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3623c1724ded..a3446ac6e874 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -49,9 +49,12 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.threadpool import ThreadPool +import synapse.util.caches from synapse.api.constants import MAX_PDU_SIZE from synapse.app import check_bind_error from synapse.app.phone_stats_home import start_phone_stats_home +from synapse.config import ConfigError +from synapse.config._base import format_config_error from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ManholeConfig from synapse.crypto import context_factory @@ -432,6 +435,10 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: signal.signal(signal.SIGHUP, run_sighup) register_sighup(refresh_certificate, hs) + register_sighup(reload_cache_config, hs.config) + + # Apply the cache config. + hs.config.caches.resize_all_caches() # Load the certificate from disk. refresh_certificate(hs) @@ -486,6 +493,43 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: atexit.register(gc.freeze) +def reload_cache_config(config: HomeServerConfig) -> None: + """Reload cache config from disk and immediately apply it.resize caches accordingly. + + If the config is invalid, a `ConfigError` is logged and no changes are made. + + Otherwise, this: + - replaces the `caches` section on the given `config` object, + - resizes all caches according to the new cache factors, and + + Note that the following cache config keys are read, but not applied: + - event_cache_size: used to set a max_size and _original_max_size on + EventsWorkerStore._get_event_cache when it is created. We'd have to update + the _original_max_size (and maybe + - sync_response_cache_duration: would have to update the timeout_sec attribute on + HomeServer -> SyncHandler -> ResponseCache. + - track_memory_usage. This affects synapse.util.caches.TRACK_MEMORY_USAGE which + influences Synapse's self-reported metrics. + + Also, the HTTPConnectionPool in SimpleHTTPClient sets its maxPersistentPerHost + parameter based on the global_factor. This won't be applied on a config reload. + """ + try: + previous_cache_config = config.reload_config_section("caches") + except ConfigError as e: + logger.warning("Failed to reload cache config") + for f in format_config_error(e): + logger.warning(f) + else: + logger.debug( + "New cache config. Was:\n %s\nNow:\n", + previous_cache_config.__dict__, + config.caches.__dict__, + ) + synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage + config.caches.resize_all_caches() + + def setup_sentry(hs: "HomeServer") -> None: """Enable sentry integration, if enabled in configuration""" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0f75e7b9d491..4c6c0658ab14 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -16,7 +16,7 @@ import logging import os import sys -from typing import Dict, Iterable, Iterator, List +from typing import Dict, Iterable, List from matrix_common.versionstring import get_distribution_version_string @@ -45,7 +45,7 @@ redirect_stdio_to_logs, register_start, ) -from synapse.config._base import ConfigError +from synapse.config._base import ConfigError, format_config_error from synapse.config.emailconfig import ThreepidBehaviour from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ListenerConfig @@ -399,38 +399,6 @@ async def start() -> None: return hs -def format_config_error(e: ConfigError) -> Iterator[str]: - """ - Formats a config error neatly - - The idea is to format the immediate error, plus the "causes" of those errors, - hopefully in a way that makes sense to the user. For example: - - Error in configuration at 'oidc_config.user_mapping_provider.config.display_name_template': - Failed to parse config for module 'JinjaOidcMappingProvider': - invalid jinja template: - unexpected end of template, expected 'end of print statement'. - - Args: - e: the error to be formatted - - Returns: An iterator which yields string fragments to be formatted - """ - yield "Error in configuration" - - if e.path: - yield " at '%s'" % (".".join(e.path),) - - yield ":\n %s" % (e.msg,) - - parent_e = e.__cause__ - indent = 1 - while parent_e: - indent += 1 - yield ":\n%s%s" % (" " * indent, str(parent_e)) - parent_e = parent_e.__cause__ - - def run(hs: HomeServer) -> None: _base.start_reactor( "synapse-homeserver", diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 179aa7ff887e..42364fc133f1 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -16,14 +16,18 @@ import argparse import errno +import logging import os from collections import OrderedDict from hashlib import sha256 from textwrap import dedent from typing import ( Any, + ClassVar, + Collection, Dict, Iterable, + Iterator, List, MutableMapping, Optional, @@ -40,6 +44,8 @@ from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter +logger = logging.getLogger(__name__) + class ConfigError(Exception): """Represents a problem parsing the configuration @@ -55,6 +61,38 @@ def __init__(self, msg: str, path: Optional[Iterable[str]] = None): self.path = path +def format_config_error(e: ConfigError) -> Iterator[str]: + """ + Formats a config error neatly + + The idea is to format the immediate error, plus the "causes" of those errors, + hopefully in a way that makes sense to the user. For example: + + Error in configuration at 'oidc_config.user_mapping_provider.config.display_name_template': + Failed to parse config for module 'JinjaOidcMappingProvider': + invalid jinja template: + unexpected end of template, expected 'end of print statement'. + + Args: + e: the error to be formatted + + Returns: An iterator which yields string fragments to be formatted + """ + yield "Error in configuration" + + if e.path: + yield " at '%s'" % (".".join(e.path),) + + yield ":\n %s" % (e.msg,) + + parent_e = e.__cause__ + indent = 1 + while parent_e: + indent += 1 + yield ":\n%s%s" % (" " * indent, str(parent_e)) + parent_e = parent_e.__cause__ + + # We split these messages out to allow packages to override with package # specific instructions. MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ @@ -119,7 +157,7 @@ class Config: defined in subclasses. """ - section: str + section: ClassVar[str] def __init__(self, root_config: "RootConfig" = None): self.root = root_config @@ -309,9 +347,12 @@ class RootConfig: class, lower-cased and with "Config" removed. """ - config_classes = [] + config_classes: List[Type[Config]] = [] + + def __init__(self, config_files: Collection[str] = ()): + # Capture absolute paths here, so we can reload config after we daemonize. + self.config_files = [os.path.abspath(path) for path in config_files] - def __init__(self): for config_class in self.config_classes: if config_class.section is None: raise ValueError("%r requires a section name" % (config_class,)) @@ -512,12 +553,10 @@ def load_config_with_parser( object from parser.parse_args(..)` """ - obj = cls() - config_args = parser.parse_args(argv) config_files = find_config_files(search_paths=config_args.config_path) - + obj = cls(config_files) if not config_files: parser.error("Must supply a config file.") @@ -627,7 +666,7 @@ def load_or_generate_config( generate_missing_configs = config_args.generate_missing_configs - obj = cls() + obj = cls(config_files) if config_args.generate_config: if config_args.report_stats is None: @@ -727,6 +766,34 @@ def generate_missing_files( ) -> None: self.invoke_all("generate_files", config_dict, config_dir_path) + def reload_config_section(self, section_name: str) -> Config: + """Reconstruct the given config section, leaving all others unchanged. + + This works in three steps: + + 1. Create a new instance of the relevant `Config` subclass. + 2. Call `read_config` on that instance to parse the new config. + 3. Replace the existing config instance with the new one. + + :raises ValueError: if the given `section` does not exist. + :raises ConfigError: for any other problems reloading config. + + :returns: the previous config object, which no longer has a reference to this + RootConfig. + """ + existing_config: Optional[Config] = getattr(self, section_name, None) + if existing_config is None: + raise ValueError(f"Unknown config section '{section_name}'") + logger.info("Reloading config section '%s'", section_name) + + new_config_data = read_config_files(self.config_files) + new_config = type(existing_config)(self) + new_config.read_config(new_config_data) + setattr(self, section_name, new_config) + + existing_config.root = None + return existing_config + def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]: """Read the config files into a dict diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index bd092f956dde..71d6655fda4e 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -1,15 +1,19 @@ import argparse from typing import ( Any, + Collection, Dict, Iterable, + Iterator, List, + Literal, MutableMapping, Optional, Tuple, Type, TypeVar, Union, + overload, ) import jinja2 @@ -64,6 +68,8 @@ class ConfigError(Exception): self.msg = msg self.path = path +def format_config_error(e: ConfigError) -> Iterator[str]: ... + MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS: str MISSING_REPORT_STATS_SPIEL: str MISSING_SERVER_NAME: str @@ -117,7 +123,8 @@ class RootConfig: background_updates: background_updates.BackgroundUpdateConfig config_classes: List[Type["Config"]] = ... - def __init__(self) -> None: ... + config_files: List[str] + def __init__(self, config_files: Collection[str] = ...) -> None: ... def invoke_all( self, func_name: str, *args: Any, **kwargs: Any ) -> MutableMapping[str, Any]: ... @@ -157,6 +164,12 @@ class RootConfig: def generate_missing_files( self, config_dict: dict, config_dir_path: str ) -> None: ... + @overload + def reload_config_section( + self, section_name: Literal["caches"] + ) -> cache.CacheConfig: ... + @overload + def reload_config_section(self, section_name: str) -> Config: ... class Config: root: RootConfig diff --git a/synapse/config/cache.py b/synapse/config/cache.py index 94d852f413d9..58b2fe55193c 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -69,11 +69,11 @@ def _canonicalise_cache_name(cache_name: str) -> str: def add_resizable_cache( cache_name: str, cache_resize_callback: Callable[[float], None] ) -> None: - """Register a cache that's size can dynamically change + """Register a cache whose size can dynamically change Args: cache_name: A reference to the cache - cache_resize_callback: A callback function that will be ran whenever + cache_resize_callback: A callback function that will run whenever the cache needs to be resized """ # Some caches have '*' in them which we strip out. @@ -96,6 +96,13 @@ class CacheConfig(Config): section = "caches" _environ = os.environ + event_cache_size: int + cache_factors: Dict[str, float] + global_factor: float + track_memory_usage: bool + expiry_time_msec: Optional[int] + sync_response_cache_duration: int + @staticmethod def reset() -> None: """Resets the caches to their defaults. Used for tests.""" @@ -115,6 +122,12 @@ def generate_config_section(self, **kwargs: Any) -> str: # A cache 'factor' is a multiplier that can be applied to each of # Synapse's caches in order to increase or decrease the maximum # number of entries that can be stored. + # + # The configuration for cache factors (caches.global_factor and + # caches.per_cache_factors) can be reloaded while the application is running, + # by sending a SIGHUP signal to the Synapse process. Changes to other parts of + # the caching config will NOT be applied after a SIGHUP is received; a restart + # is necessary. # The number of events to cache in memory. Not affected by # caches.global_factor. @@ -174,21 +187,21 @@ def generate_config_section(self, **kwargs: Any) -> str: """ def read_config(self, config: JsonDict, **kwargs: Any) -> None: + """Populate this config object with values from `config`. + + This method does NOT resize existing or future caches: use `resize_all_caches`. + We use two separate methods so that we can reject bad config before applying it. + """ self.event_cache_size = self.parse_size( config.get("event_cache_size", _DEFAULT_EVENT_CACHE_SIZE) ) - self.cache_factors: Dict[str, float] = {} + self.cache_factors = {} cache_config = config.get("caches") or {} - self.global_factor = cache_config.get( - "global_factor", properties.default_factor_size - ) + self.global_factor = cache_config.get("global_factor", _DEFAULT_FACTOR_SIZE) if not isinstance(self.global_factor, (int, float)): raise ConfigError("caches.global_factor must be a number.") - # Set the global one so that it's reflected in new caches - properties.default_factor_size = self.global_factor - # Load cache factors from the config individual_factors = cache_config.get("per_cache_factors") or {} if not isinstance(individual_factors, dict): @@ -230,7 +243,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: cache_entry_ttl = cache_config.get("cache_entry_ttl", "30m") if expire_caches: - self.expiry_time_msec: Optional[int] = self.parse_duration(cache_entry_ttl) + self.expiry_time_msec = self.parse_duration(cache_entry_ttl) else: self.expiry_time_msec = None @@ -254,19 +267,19 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: cache_config.get("sync_response_cache_duration", 0) ) - # Resize all caches (if necessary) with the new factors we've loaded - self.resize_all_caches() - - # Store this function so that it can be called from other classes without - # needing an instance of Config - properties.resize_all_caches_func = self.resize_all_caches - def resize_all_caches(self) -> None: - """Ensure all cache sizes are up to date + """Ensure all cache sizes are up-to-date. For each cache, run the mapped callback function with either a specific cache factor or the default, global one. """ + # Set the global factor size, so that new caches are appropriately sized. + properties.default_factor_size = self.global_factor + + # Store this function so that it can be called from other classes without + # needing an instance of CacheConfig + properties.resize_all_caches_func = self.resize_all_caches + # block other threads from modifying _CACHES while we iterate it. with _CACHES_LOCK: for cache_name, callback in _CACHES.items(): diff --git a/synapse/http/client.py b/synapse/http/client.py index 8310fb466ac5..b2c9a7c67090 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -348,7 +348,7 @@ def __init__( # XXX: The justification for using the cache factor here is that larger instances # will need both more cache and more connections. # Still, this should probably be a separate dial - pool.maxPersistentPerHost = max((100 * hs.config.caches.global_factor, 5)) + pool.maxPersistentPerHost = max(int(100 * hs.config.caches.global_factor), 5) pool.cachedConnectionTimeout = 2 * 60 self.agent: IAgent = ProxyAgent( diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index 4bb82e810e0c..d2b3c299e354 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -38,6 +38,7 @@ def test_individual_caches_from_environ(self): "SYNAPSE_NOT_CACHE": "BLAH", } self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() self.assertEqual(dict(self.config.cache_factors), {"something_or_other": 2.0}) @@ -52,6 +53,7 @@ def test_config_overrides_environ(self): "SYNAPSE_CACHE_FACTOR_FOO": 1, } self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() self.assertEqual( dict(self.config.cache_factors), @@ -71,6 +73,7 @@ def test_individual_instantiated_before_config_load(self): config = {"caches": {"per_cache_factors": {"foo": 3}}} self.config.read_config(config) + self.config.resize_all_caches() self.assertEqual(cache.max_size, 300) @@ -82,6 +85,7 @@ def test_individual_instantiated_after_config_load(self): """ config = {"caches": {"per_cache_factors": {"foo": 2}}} self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() cache = LruCache(100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) @@ -99,6 +103,7 @@ def test_global_instantiated_before_config_load(self): config = {"caches": {"global_factor": 4}} self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() self.assertEqual(cache.max_size, 400) @@ -110,6 +115,7 @@ def test_global_instantiated_after_config_load(self): """ config = {"caches": {"global_factor": 1.5}} self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() cache = LruCache(100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) @@ -128,6 +134,7 @@ def test_cache_with_asterisk_in_name(self): "SYNAPSE_CACHE_FACTOR_CACHE_B": 3, } self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() cache_a = LruCache(100) add_resizable_cache("*cache_a*", cache_resize_callback=cache_a.set_cache_factor) @@ -148,6 +155,7 @@ def test_apply_cache_factor_from_config(self): config = {"caches": {"event_cache_size": "10k"}} self.config.read_config(config, config_dir_path="", data_dir_path="") + self.config.resize_all_caches() cache = LruCache( max_size=self.config.event_cache_size, diff --git a/tests/server.py b/tests/server.py index aaefcfc46cd9..b9f465971fdd 100644 --- a/tests/server.py +++ b/tests/server.py @@ -749,6 +749,7 @@ def setup_test_homeserver( if config is None: config = default_config(name, parse=True) + config.caches.resize_all_caches() config.ldap_enabled = False if "clock" not in kwargs: From 6ee61b905256f87dc2b75007ed711cd59065db9a Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 11 May 2022 14:52:26 +0100 Subject: [PATCH 174/263] Complain if a federation endpoint has the `@cancellable` flag (#12705) `BaseFederationServlet` wraps its endpoints in a bunch of async code that has not been vetted for compatibility with cancellation. Fail CI if a `@cancellable` flag is applied to a federation endpoint. Signed-off-by: Sean Quah --- changelog.d/12705.misc | 1 + synapse/federation/transport/server/_base.py | 13 ++++++++++++- tests/federation/transport/server/test__base.py | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12705.misc diff --git a/changelog.d/12705.misc b/changelog.d/12705.misc new file mode 100644 index 000000000000..a913d8bb85eb --- /dev/null +++ b/changelog.d/12705.misc @@ -0,0 +1 @@ +Complain if a federation endpoint has the `@cancellable` flag, since some of the wrapper code may not handle cancellation correctly yet. diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index d629a3ecb5dd..103861644a70 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -21,7 +21,7 @@ from synapse.api.errors import Codes, FederationDeniedError, SynapseError from synapse.api.urls import FEDERATION_V1_PREFIX -from synapse.http.server import HttpServer, ServletCallback +from synapse.http.server import HttpServer, ServletCallback, is_method_cancellable from synapse.http.servlet import parse_json_object_from_request from synapse.http.site import SynapseRequest from synapse.logging.context import run_in_background @@ -373,6 +373,17 @@ def register(self, server: HttpServer) -> None: if code is None: continue + if is_method_cancellable(code): + # The wrapper added by `self._wrap` will inherit the cancellable flag, + # but the wrapper itself does not support cancellation yet. + # Once resolved, the cancellation tests in + # `tests/federation/transport/server/test__base.py` can be re-enabled. + raise Exception( + f"{self.__class__.__name__}.on_{method} has been marked as " + "cancellable, but federation servlets do not support cancellation " + "yet." + ) + server.register_paths( method, (pattern,), diff --git a/tests/federation/transport/server/test__base.py b/tests/federation/transport/server/test__base.py index 98a951f03e07..ac3695a8ccab 100644 --- a/tests/federation/transport/server/test__base.py +++ b/tests/federation/transport/server/test__base.py @@ -59,6 +59,8 @@ class BaseFederationServletCancellationTests( ): """Tests for `BaseFederationServlet` cancellation.""" + skip = "`BaseFederationServlet` does not support cancellation yet." + path = f"{CancellableFederationServlet.PREFIX}{CancellableFederationServlet.PATH}" def create_test_resource(self): From db10f2c037ff59124776a10e198ab432aec2bdc6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 11 May 2022 16:34:17 +0100 Subject: [PATCH 175/263] No longer permit empty body when sending receipts (#12709) --- changelog.d/12709.removal | 1 + synapse/rest/client/receipts.py | 13 +------------ tests/rest/client/test_sync.py | 30 ++++-------------------------- 3 files changed, 6 insertions(+), 38 deletions(-) create mode 100644 changelog.d/12709.removal diff --git a/changelog.d/12709.removal b/changelog.d/12709.removal new file mode 100644 index 000000000000..6bb03e28941f --- /dev/null +++ b/changelog.d/12709.removal @@ -0,0 +1 @@ +Require a body in POST requests to `/rooms/{roomId}/receipt/{receiptType}/{eventId}`, as required by the [Matrix specification](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidreceiptreceipttypeeventid). This breaks compatibility with Element Android 1.2.0 and earlier: users of those clients will be unable to send read receipts. diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index f9caab663523..4b03eb876b75 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -13,12 +13,10 @@ # limitations under the License. import logging -import re from typing import TYPE_CHECKING, Tuple from synapse.api.constants import ReceiptTypes from synapse.api.errors import SynapseError -from synapse.http import get_request_user_agent from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -26,8 +24,6 @@ from ._base import client_patterns -pattern = re.compile(r"(?:Element|SchildiChat)/1\.[012]\.") - if TYPE_CHECKING: from synapse.server import HomeServer @@ -69,14 +65,7 @@ async def on_POST( ): raise SynapseError(400, "Receipt type must be 'm.read'") - # Do not allow older SchildiChat and Element Android clients (prior to Element/1.[012].x) to send an empty body. - user_agent = get_request_user_agent(request) - allow_empty_body = False - if "Android" in user_agent: - if pattern.match(user_agent) or "Riot" in user_agent: - allow_empty_body = True - # This call makes sure possible empty body is handled correctly - parse_json_object_from_request(request, allow_empty_body) + parse_json_object_from_request(request, allow_empty_body=False) await self.presence_handler.bump_presence_active_time(requester.user) diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 010833764957..2722bf26e76c 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +from http import HTTPStatus from typing import List, Optional from parameterized import parameterized @@ -485,30 +486,7 @@ def test_private_receipt_cannot_override_public(self) -> None: # Test that we didn't override the public read receipt self.assertIsNone(self._get_read_receipt()) - @parameterized.expand( - [ - # Old Element version, expected to send an empty body - ( - "agent1", - "Element/1.2.2 (Linux; U; Android 9; MatrixAndroidSDK_X 0.0.1)", - 200, - ), - # Old SchildiChat version, expected to send an empty body - ("agent2", "SchildiChat/1.2.1 (Android 10)", 200), - # Expected 400: Denies empty body starting at version 1.3+ - ("agent3", "Element/1.3.6 (Android 10)", 400), - ("agent4", "SchildiChat/1.3.6 (Android 11)", 400), - # Contains "Riot": Receipts with empty bodies expected - ("agent5", "Element (Riot.im) (Android 9)", 200), - # Expected 400: Does not contain "Android" - ("agent6", "Element/1.2.1", 400), - # Expected 400: Different format, missing "/" after Element; existing build that should allow empty bodies, but minimal ongoing usage - ("agent7", "Element dbg/1.1.8-dev (Android)", 400), - ] - ) - def test_read_receipt_with_empty_body( - self, name: str, user_agent: str, expected_status_code: int - ) -> None: + def test_read_receipt_with_empty_body_is_rejected(self) -> None: # Send a message as the first user res = self.helper.send(self.room_id, body="hello", tok=self.tok) @@ -517,9 +495,9 @@ def test_read_receipt_with_empty_body( "POST", f"/rooms/{self.room_id}/receipt/m.read/{res['event_id']}", access_token=self.tok2, - custom_headers=[("User-Agent", user_agent)], ) - self.assertEqual(channel.code, expected_status_code) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) + self.assertEqual(channel.json_body["errcode"], "M_NOT_JSON", channel.json_body) def _get_read_receipt(self) -> Optional[JsonDict]: """Syncs and returns the read receipt.""" From bf7ce92bf7307ded3643d8cc5ee01aee21f23f58 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 11 May 2022 17:22:34 +0100 Subject: [PATCH 176/263] Enable cancellation of `GET /members` and `GET /state` requests (#12708) Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$state_key/*` requests. Signed-off-by: Sean Quah --- changelog.d/12708.misc | 1 + synapse/http/server.py | 4 +++- synapse/rest/client/room.py | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12708.misc diff --git a/changelog.d/12708.misc b/changelog.d/12708.misc new file mode 100644 index 000000000000..aa99e7311b97 --- /dev/null +++ b/changelog.d/12708.misc @@ -0,0 +1 @@ +Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. diff --git a/synapse/http/server.py b/synapse/http/server.py index 756c6e1aeeda..e3dcc3f3dd06 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -139,7 +139,9 @@ class SomeServlet(RestServlet): async def on_GET(self, request: SynapseRequest) -> ...: ... """ - if method.__name__ not in _cancellable_method_names: + if method.__name__ not in _cancellable_method_names and not any( + method.__name__.startswith(prefix) for prefix in _cancellable_method_names + ): raise ValueError( "@cancellable decorator can only be applied to servlet methods." ) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 906fe09e9713..4b8bfbffcb36 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -34,7 +34,7 @@ ) from synapse.api.filtering import Filter from synapse.events.utils import format_event_for_client_v2 -from synapse.http.server import HttpServer +from synapse.http.server import HttpServer, cancellable from synapse.http.servlet import ( ResolveRoomIdMixin, RestServlet, @@ -143,6 +143,7 @@ def register(self, http_server: HttpServer) -> None: self.__class__.__name__, ) + @cancellable def on_GET_no_state_key( self, request: SynapseRequest, room_id: str, event_type: str ) -> Awaitable[Tuple[int, JsonDict]]: @@ -153,6 +154,7 @@ def on_PUT_no_state_key( ) -> Awaitable[Tuple[int, JsonDict]]: return self.on_PUT(request, room_id, event_type, "") + @cancellable async def on_GET( self, request: SynapseRequest, room_id: str, event_type: str, state_key: str ) -> Tuple[int, JsonDict]: @@ -481,6 +483,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.store = hs.get_datastores().main + @cancellable async def on_GET( self, request: SynapseRequest, room_id: str ) -> Tuple[int, JsonDict]: @@ -602,6 +605,7 @@ def __init__(self, hs: "HomeServer"): self.message_handler = hs.get_message_handler() self.auth = hs.get_auth() + @cancellable async def on_GET( self, request: SynapseRequest, room_id: str ) -> Tuple[int, List[JsonDict]]: From 409573f6d0f146db2a55914cb7b65a0a95f6fde5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 12 May 2022 09:29:37 +0100 Subject: [PATCH 177/263] Fix reference to the wrong symbol in the media admin api docs (#12715) --- changelog.d/12715.doc | 1 + docs/admin_api/media_admin_api.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12715.doc diff --git a/changelog.d/12715.doc b/changelog.d/12715.doc new file mode 100644 index 000000000000..150d78c3f634 --- /dev/null +++ b/changelog.d/12715.doc @@ -0,0 +1 @@ +Fix a typo in the Media Admin API documentation. diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 96b3668f2a08..d57c5aedae4c 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -289,7 +289,7 @@ POST /_synapse/admin/v1/purge_media_cache?before_ts= URL Parameters -* `unix_timestamp_in_ms`: string representing a positive integer - Unix timestamp in milliseconds. +* `before_ts`: string representing a positive integer - Unix timestamp in milliseconds. All cached media that was last accessed before this timestamp will be removed. Response: From de1e599b9defdc9b541f14a03157f614cb688729 Mon Sep 17 00:00:00 2001 From: Andy Balaam Date: Thu, 12 May 2022 11:41:35 +0100 Subject: [PATCH 178/263] add default_power_level_content_override config option. (#12618) Co-authored-by: Matthew Hodgson --- changelog.d/12618.feature | 1 + docs/sample_config.yaml | 34 +++ .../configuration/config_documentation.md | 26 ++ synapse/config/room.py | 47 ++++ synapse/handlers/room.py | 16 +- tests/rest/client/test_rooms.py | 258 ++++++++++++++++++ 6 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12618.feature diff --git a/changelog.d/12618.feature b/changelog.d/12618.feature new file mode 100644 index 000000000000..37fa03b3cb41 --- /dev/null +++ b/changelog.d/12618.feature @@ -0,0 +1 @@ +Add a `default_power_level_content_override` config option to set default room power levels per room preset. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index e7b57f5a0bdf..03a0f6314cdd 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2468,6 +2468,40 @@ push: # #encryption_enabled_by_default_for_room_type: invite +# Override the default power levels for rooms created on this server, per +# room creation preset. +# +# The appropriate dictionary for the room preset will be applied on top +# of the existing power levels content. +# +# Useful if you know that your users need special permissions in rooms +# that they create (e.g. to send particular types of state events without +# needing an elevated power level). This takes the same shape as the +# `power_level_content_override` parameter in the /createRoom API, but +# is applied before that parameter. +# +# Valid keys are some or all of `private_chat`, `trusted_private_chat` +# and `public_chat`. Inside each of those should be any of the +# properties allowed in `power_level_content_override` in the +# /createRoom API. If any property is missing, its default value will +# continue to be used. If any property is present, it will overwrite +# the existing default completely (so if the `events` property exists, +# the default event power levels will be ignored). +# +#default_power_level_content_override: +# private_chat: +# "events": +# "com.example.myeventtype" : 0 +# "m.room.avatar": 50 +# "m.room.canonical_alias": 50 +# "m.room.encryption": 100 +# "m.room.history_visibility": 100 +# "m.room.name": 50 +# "m.room.power_levels": 100 +# "m.room.server_acl": 100 +# "m.room.tombstone": 100 +# "events_default": 1 + # Uncomment to allow non-server-admin users to create groups on this server # diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index f292b94fb0cd..2af1f284b14e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3315,6 +3315,32 @@ room_list_publication_rules: room_id: "*" action: allow ``` + +--- +Config option: `default_power_level_content_override` + +The `default_power_level_content_override` option controls the default power +levels for rooms. + +Useful if you know that your users need special permissions in rooms +that they create (e.g. to send particular types of state events without +needing an elevated power level). This takes the same shape as the +`power_level_content_override` parameter in the /createRoom API, but +is applied before that parameter. + +Note that each key provided inside a preset (for example `events` in the example +below) will overwrite all existing defaults inside that key. So in the example +below, newly-created private_chat rooms will have no rules for any event types +except `com.example.foo`. + +Example configuration: +```yaml +default_power_level_content_override: + private_chat: { "events": { "com.example.foo" : 0 } } + trusted_private_chat: null + public_chat: null +``` + --- ## Opentracing ## Configuration options related to Opentracing support. diff --git a/synapse/config/room.py b/synapse/config/room.py index e18a87ea37f6..462d85ac1d1e 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -63,6 +63,19 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "Invalid value for encryption_enabled_by_default_for_room_type" ) + self.default_power_level_content_override = config.get( + "default_power_level_content_override", + None, + ) + if self.default_power_level_content_override is not None: + for preset in self.default_power_level_content_override: + if preset not in vars(RoomCreationPreset).values(): + raise ConfigError( + "Unrecognised room preset %s in default_power_level_content_override" + % preset + ) + # We validate the actual overrides when we try to apply them. + def generate_config_section(self, **kwargs: Any) -> str: return """\ ## Rooms ## @@ -83,4 +96,38 @@ def generate_config_section(self, **kwargs: Any) -> str: # will also not affect rooms created by other servers. # #encryption_enabled_by_default_for_room_type: invite + + # Override the default power levels for rooms created on this server, per + # room creation preset. + # + # The appropriate dictionary for the room preset will be applied on top + # of the existing power levels content. + # + # Useful if you know that your users need special permissions in rooms + # that they create (e.g. to send particular types of state events without + # needing an elevated power level). This takes the same shape as the + # `power_level_content_override` parameter in the /createRoom API, but + # is applied before that parameter. + # + # Valid keys are some or all of `private_chat`, `trusted_private_chat` + # and `public_chat`. Inside each of those should be any of the + # properties allowed in `power_level_content_override` in the + # /createRoom API. If any property is missing, its default value will + # continue to be used. If any property is present, it will overwrite + # the existing default completely (so if the `events` property exists, + # the default event power levels will be ignored). + # + #default_power_level_content_override: + # private_chat: + # "events": + # "com.example.myeventtype" : 0 + # "m.room.avatar": 50 + # "m.room.canonical_alias": 50 + # "m.room.encryption": 100 + # "m.room.history_visibility": 100 + # "m.room.name": 50 + # "m.room.power_levels": 100 + # "m.room.server_acl": 100 + # "m.room.tombstone": 100 + # "events_default": 1 """ diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 604eb6ec154a..e71c78adad67 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -149,6 +149,10 @@ def __init__(self, hs: "HomeServer"): ) preset_config["encrypted"] = encrypted + self._default_power_level_content_override = ( + self.config.room.default_power_level_content_override + ) + self._replication = hs.get_replication_data_handler() # linearizer to stop two upgrades happening at once @@ -1042,9 +1046,19 @@ async def send(etype: str, content: JsonDict, **kwargs: Any) -> int: for invitee in invite_list: power_level_content["users"][invitee] = 100 - # Power levels overrides are defined per chat preset + # If the user supplied a preset name e.g. "private_chat", + # we apply that preset power_level_content.update(config["power_level_content_override"]) + # If the server config contains default_power_level_content_override, + # and that contains information for this room preset, apply it. + if self._default_power_level_content_override: + override = self._default_power_level_content_override.get(preset_config) + if override is not None: + power_level_content.update(override) + + # Finally, if the user supplied specific permissions for this room, + # apply those. if power_level_content_override: power_level_content.update(power_level_content_override) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 9443daa0560a..ad416e2fd8c0 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -1116,6 +1116,264 @@ def test_rooms_messages_sent(self) -> None: self.assertEqual(200, channel.code, msg=channel.result["body"]) +class RoomPowerLevelOverridesTestCase(RoomBase): + """Tests that the power levels can be overridden with server config.""" + + user_id = "@sid1:red" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user_id = self.register_user("admin", "pass") + self.admin_access_token = self.login("admin", "pass") + + def power_levels(self, room_id: str) -> Dict[str, Any]: + return self.helper.get_state( + room_id, "m.room.power_levels", self.admin_access_token + ) + + def test_default_power_levels_with_room_override(self) -> None: + """ + Create a room, providing power level overrides. + Confirm that the room's power levels reflect the overrides. + + See https://github.com/matrix-org/matrix-spec/issues/492 + - currently we overwrite each key of power_level_content_override + completely. + """ + + room_id = self.helper.create_room_as( + self.user_id, + extra_content={ + "power_level_content_override": {"events": {"custom.event": 0}} + }, + ) + self.assertEqual( + { + "custom.event": 0, + }, + self.power_levels(room_id)["events"], + ) + + @unittest.override_config( + { + "default_power_level_content_override": { + "public_chat": {"events": {"custom.event": 0}}, + } + }, + ) + def test_power_levels_with_server_override(self) -> None: + """ + With a server configured to modify the room-level defaults, + Create a room, without providing any extra power level overrides. + Confirm that the room's power levels reflect the server-level overrides. + + Similar to https://github.com/matrix-org/matrix-spec/issues/492, + we overwrite each key of power_level_content_override completely. + """ + + room_id = self.helper.create_room_as(self.user_id) + self.assertEqual( + { + "custom.event": 0, + }, + self.power_levels(room_id)["events"], + ) + + @unittest.override_config( + { + "default_power_level_content_override": { + "public_chat": { + "events": {"server.event": 0}, + "ban": 13, + }, + } + }, + ) + def test_power_levels_with_server_and_room_overrides(self) -> None: + """ + With a server configured to modify the room-level defaults, + create a room, providing different overrides. + Confirm that the room's power levels reflect both overrides, and + choose the room overrides where they clash. + """ + + room_id = self.helper.create_room_as( + self.user_id, + extra_content={ + "power_level_content_override": {"events": {"room.event": 0}} + }, + ) + + # Room override wins over server config + self.assertEqual( + {"room.event": 0}, + self.power_levels(room_id)["events"], + ) + + # But where there is no room override, server config wins + self.assertEqual(13, self.power_levels(room_id)["ban"]) + + +class RoomPowerLevelOverridesInPracticeTestCase(RoomBase): + """ + Tests that we can really do various otherwise-prohibited actions + based on overriding the power levels in config. + """ + + user_id = "@sid1:red" + + def test_creator_can_post_state_event(self) -> None: + # Given I am the creator of a room + room_id = self.helper.create_room_as(self.user_id) + + # When I send a state event + path = "/rooms/{room_id}/state/custom.event/my_state_key".format( + room_id=urlparse.quote(room_id), + ) + channel = self.make_request("PUT", path, "{}") + + # Then I am allowed + self.assertEqual(200, channel.code, msg=channel.result["body"]) + + def test_normal_user_can_not_post_state_event(self) -> None: + # Given I am a normal member of a room + room_id = self.helper.create_room_as("@some_other_guy:red") + self.helper.join(room=room_id, user=self.user_id) + + # When I send a state event + path = "/rooms/{room_id}/state/custom.event/my_state_key".format( + room_id=urlparse.quote(room_id), + ) + channel = self.make_request("PUT", path, "{}") + + # Then I am not allowed because state events require PL>=50 + self.assertEqual(403, channel.code, msg=channel.result["body"]) + self.assertEqual( + "You don't have permission to post that to the room. " + "user_level (0) < send_level (50)", + channel.json_body["error"], + ) + + @unittest.override_config( + { + "default_power_level_content_override": { + "public_chat": {"events": {"custom.event": 0}}, + } + }, + ) + def test_with_config_override_normal_user_can_post_state_event(self) -> None: + # Given the server has config allowing normal users to post my event type, + # and I am a normal member of a room + room_id = self.helper.create_room_as("@some_other_guy:red") + self.helper.join(room=room_id, user=self.user_id) + + # When I send a state event + path = "/rooms/{room_id}/state/custom.event/my_state_key".format( + room_id=urlparse.quote(room_id), + ) + channel = self.make_request("PUT", path, "{}") + + # Then I am allowed + self.assertEqual(200, channel.code, msg=channel.result["body"]) + + @unittest.override_config( + { + "default_power_level_content_override": { + "public_chat": {"events": {"custom.event": 0}}, + } + }, + ) + def test_any_room_override_defeats_config_override(self) -> None: + # Given the server has config allowing normal users to post my event type + # And I am a normal member of a room + # But the room was created with special permissions + extra_content: Dict[str, Any] = { + "power_level_content_override": {"events": {}}, + } + room_id = self.helper.create_room_as( + "@some_other_guy:red", extra_content=extra_content + ) + self.helper.join(room=room_id, user=self.user_id) + + # When I send a state event + path = "/rooms/{room_id}/state/custom.event/my_state_key".format( + room_id=urlparse.quote(room_id), + ) + channel = self.make_request("PUT", path, "{}") + + # Then I am not allowed + self.assertEqual(403, channel.code, msg=channel.result["body"]) + + @unittest.override_config( + { + "default_power_level_content_override": { + "public_chat": {"events": {"custom.event": 0}}, + } + }, + ) + def test_specific_room_override_defeats_config_override(self) -> None: + # Given the server has config allowing normal users to post my event type, + # and I am a normal member of a room, + # but the room was created with special permissions for this event type + extra_content = { + "power_level_content_override": {"events": {"custom.event": 1}}, + } + room_id = self.helper.create_room_as( + "@some_other_guy:red", extra_content=extra_content + ) + self.helper.join(room=room_id, user=self.user_id) + + # When I send a state event + path = "/rooms/{room_id}/state/custom.event/my_state_key".format( + room_id=urlparse.quote(room_id), + ) + channel = self.make_request("PUT", path, "{}") + + # Then I am not allowed + self.assertEqual(403, channel.code, msg=channel.result["body"]) + self.assertEqual( + "You don't have permission to post that to the room. " + + "user_level (0) < send_level (1)", + channel.json_body["error"], + ) + + @unittest.override_config( + { + "default_power_level_content_override": { + "public_chat": {"events": {"custom.event": 0}}, + "private_chat": None, + "trusted_private_chat": None, + } + }, + ) + def test_config_override_applies_only_to_specific_preset(self) -> None: + # Given the server has config for public_chats, + # and I am a normal member of a private_chat room + room_id = self.helper.create_room_as("@some_other_guy:red", is_public=False) + self.helper.invite(room=room_id, src="@some_other_guy:red", targ=self.user_id) + self.helper.join(room=room_id, user=self.user_id) + + # When I send a state event + path = "/rooms/{room_id}/state/custom.event/my_state_key".format( + room_id=urlparse.quote(room_id), + ) + channel = self.make_request("PUT", path, "{}") + + # Then I am not allowed because the public_chat config does not + # affect this room, because this room is a private_chat + self.assertEqual(403, channel.code, msg=channel.result["body"]) + self.assertEqual( + "You don't have permission to post that to the room. " + + "user_level (0) < send_level (50)", + channel.json_body["error"], + ) + + class RoomInitialSyncTestCase(RoomBase): """Tests /rooms/$room_id/initialSync.""" From 17e1eb7749adf12d43f534c50115bbe19c809ea6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 12 May 2022 15:33:50 +0100 Subject: [PATCH 179/263] Reduce the number of "untyped defs" (#12716) --- changelog.d/12716.misc | 1 + mypy.ini | 24 ++++++++++ synapse/groups/groups_server.py | 2 +- synapse/http/client.py | 16 ++++--- .../federation/matrix_federation_agent.py | 2 +- synapse/http/federation/srv_resolver.py | 4 +- .../http/federation/well_known_resolver.py | 6 +-- synapse/http/matrixfederationclient.py | 31 ++++++++----- synapse/http/request_metrics.py | 10 ++--- synapse/storage/database.py | 44 +++++++++++++------ synapse/storage/databases/main/metrics.py | 24 +++++----- synapse/storage/databases/main/stream.py | 8 ++-- synapse/storage/persist_events.py | 21 +++++---- synapse/storage/prepare_database.py | 2 +- synapse/storage/state.py | 6 ++- synapse/storage/types.py | 10 ++++- 16 files changed, 142 insertions(+), 69 deletions(-) create mode 100644 changelog.d/12716.misc diff --git a/changelog.d/12716.misc b/changelog.d/12716.misc new file mode 100644 index 000000000000..b07e1b52ee7c --- /dev/null +++ b/changelog.d/12716.misc @@ -0,0 +1 @@ +Add type annotations to increase the number of modules passing `disallow-untyped-defs`. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index ba0de419f5ea..8478dd9e510b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -119,9 +119,18 @@ disallow_untyped_defs = True [mypy-synapse.federation.transport.client] disallow_untyped_defs = False +[mypy-synapse.groups.*] +disallow_untyped_defs = True + [mypy-synapse.handlers.*] disallow_untyped_defs = True +[mypy-synapse.http.federation.*] +disallow_untyped_defs = True + +[mypy-synapse.http.request_metrics] +disallow_untyped_defs = True + [mypy-synapse.http.server] disallow_untyped_defs = True @@ -196,12 +205,27 @@ disallow_untyped_defs = True [mypy-synapse.storage.databases.main.state_deltas] disallow_untyped_defs = True +[mypy-synapse.storage.databases.main.stream] +disallow_untyped_defs = True + [mypy-synapse.storage.databases.main.transactions] disallow_untyped_defs = True [mypy-synapse.storage.databases.main.user_erasure_store] disallow_untyped_defs = True +[mypy-synapse.storage.prepare_database] +disallow_untyped_defs = True + +[mypy-synapse.storage.persist_events] +disallow_untyped_defs = True + +[mypy-synapse.storage.state] +disallow_untyped_defs = True + +[mypy-synapse.storage.types] +disallow_untyped_defs = True + [mypy-synapse.storage.util.*] disallow_untyped_defs = True diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 4c3a5a6e24d1..dfd24af695ab 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -934,7 +934,7 @@ async def delete_group(self, group_id: str, requester_user_id: str) -> None: # Before deleting the group lets kick everyone out of it users = await self.store.get_users_in_group(group_id, include_private=True) - async def _kick_user_from_group(user_id): + async def _kick_user_from_group(user_id: str) -> None: if self.hs.is_mine_id(user_id): groups_local = self.hs.get_groups_local_handler() assert isinstance( diff --git a/synapse/http/client.py b/synapse/http/client.py index b2c9a7c67090..084d0a5b84e9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -43,8 +43,10 @@ from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.interfaces import ( IAddress, + IDelayedCall, IHostResolution, IReactorPluggableNameResolver, + IReactorTime, IResolutionReceiver, ITCPTransport, ) @@ -121,13 +123,15 @@ def check_against_blacklist( _EPSILON = 0.00000001 -def _make_scheduler(reactor): +def _make_scheduler( + reactor: IReactorTime, +) -> Callable[[Callable[[], object]], IDelayedCall]: """Makes a schedular suitable for a Cooperator using the given reactor. (This is effectively just a copy from `twisted.internet.task`) """ - def _scheduler(x): + def _scheduler(x: Callable[[], object]) -> IDelayedCall: return reactor.callLater(_EPSILON, x) return _scheduler @@ -775,7 +779,7 @@ async def get_file( ) -def _timeout_to_request_timed_out_error(f: Failure): +def _timeout_to_request_timed_out_error(f: Failure) -> Failure: if f.check(twisted_error.TimeoutError, twisted_error.ConnectingCancelledError): # The TCP connection has its own timeout (set by the 'connectTimeout' param # on the Agent), which raises twisted_error.TimeoutError exception. @@ -809,7 +813,7 @@ class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol): def __init__(self, deferred: defer.Deferred): self.deferred = deferred - def _maybe_fail(self): + def _maybe_fail(self) -> None: """ Report a max size exceed error and disconnect the first time this is called. """ @@ -933,12 +937,12 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory): Do not use this since it allows an attacker to intercept your communications. """ - def __init__(self): + def __init__(self) -> None: self._context = SSL.Context(SSL.SSLv23_METHOD) self._context.set_verify(VERIFY_NONE, lambda *_: False) def getContext(self, hostname=None, port=None): return self._context - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: bytes, port: int): return self diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index a8a520f80944..2f0177f1e203 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -239,7 +239,7 @@ def __init__( self._srv_resolver = srv_resolver - def endpointForURI(self, parsed_uri: URI): + def endpointForURI(self, parsed_uri: URI) -> "MatrixHostnameEndpoint": return MatrixHostnameEndpoint( self._reactor, self._proxy_reactor, diff --git a/synapse/http/federation/srv_resolver.py b/synapse/http/federation/srv_resolver.py index f68646fd0dd4..de0e882b3312 100644 --- a/synapse/http/federation/srv_resolver.py +++ b/synapse/http/federation/srv_resolver.py @@ -16,7 +16,7 @@ import logging import random import time -from typing import Callable, Dict, List +from typing import Any, Callable, Dict, List import attr @@ -109,7 +109,7 @@ class SrvResolver: def __init__( self, - dns_client=client, + dns_client: Any = client, cache: Dict[bytes, List[Server]] = SERVER_CACHE, get_time: Callable[[], float] = time.time, ): diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 43f2140429b5..71b685fadec9 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -74,9 +74,9 @@ _had_valid_well_known_cache: TTLCache[bytes, bool] = TTLCache("had-valid-well-known") -@attr.s(slots=True, frozen=True) +@attr.s(slots=True, frozen=True, auto_attribs=True) class WellKnownLookupResult: - delegated_server = attr.ib() + delegated_server: Optional[bytes] class WellKnownResolver: @@ -336,4 +336,4 @@ def _parse_cache_control(headers: Headers) -> Dict[bytes, Optional[bytes]]: class _FetchWellKnownFailure(Exception): # True if we didn't get a non-5xx HTTP response, i.e. this may or may not be # a temporary failure. - temporary = attr.ib() + temporary: bool = attr.ib() diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c2ec3caa0ea8..725b5c33b8c5 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -23,6 +23,8 @@ from io import BytesIO, StringIO from typing import ( TYPE_CHECKING, + Any, + BinaryIO, Callable, Dict, Generic, @@ -44,7 +46,7 @@ from twisted.internet import defer from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IReactorTime -from twisted.internet.task import _EPSILON, Cooperator +from twisted.internet.task import Cooperator from twisted.web.client import ResponseFailed from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer, IResponse @@ -58,11 +60,13 @@ RequestSendFailed, SynapseError, ) +from synapse.crypto.context_factory import FederationPolicyForHTTPS from synapse.http import QuieterFileBodyProducer from synapse.http.client import ( BlacklistingAgentWrapper, BodyExceededMaxSize, ByteWriteable, + _make_scheduler, encode_query_args, read_body_with_max_size, ) @@ -181,7 +185,7 @@ class JsonParser(ByteParser[Union[JsonDict, list]]): CONTENT_TYPE = "application/json" - def __init__(self): + def __init__(self) -> None: self._buffer = StringIO() self._binary_wrapper = BinaryIOWrapper(self._buffer) @@ -299,7 +303,9 @@ async def _handle_response( class BinaryIOWrapper: """A wrapper for a TextIO which converts from bytes on the fly.""" - def __init__(self, file: typing.TextIO, encoding="utf-8", errors="strict"): + def __init__( + self, file: typing.TextIO, encoding: str = "utf-8", errors: str = "strict" + ): self.decoder = codecs.getincrementaldecoder(encoding)(errors) self.file = file @@ -317,7 +323,11 @@ class MatrixFederationHttpClient: requests. """ - def __init__(self, hs: "HomeServer", tls_client_options_factory): + def __init__( + self, + hs: "HomeServer", + tls_client_options_factory: Optional[FederationPolicyForHTTPS], + ): self.hs = hs self.signing_key = hs.signing_key self.server_name = hs.hostname @@ -348,10 +358,7 @@ def __init__(self, hs: "HomeServer", tls_client_options_factory): self.version_string_bytes = hs.version_string.encode("ascii") self.default_timeout = 60 - def schedule(x): - self.reactor.callLater(_EPSILON, x) - - self._cooperator = Cooperator(scheduler=schedule) + self._cooperator = Cooperator(scheduler=_make_scheduler(self.reactor)) self._sleeper = AwakenableSleeper(self.reactor) @@ -364,7 +371,7 @@ async def _send_request_with_optional_trailing_slash( self, request: MatrixFederationRequest, try_trailing_slash_on_400: bool = False, - **send_request_args, + **send_request_args: Any, ) -> IResponse: """Wrapper for _send_request which can optionally retry the request upon receiving a combination of a 400 HTTP response code and a @@ -1159,7 +1166,7 @@ async def get_file( self, destination: str, path: str, - output_stream, + output_stream: BinaryIO, args: Optional[QueryParams] = None, retry_on_dns_fail: bool = True, max_size: Optional[int] = None, @@ -1250,10 +1257,10 @@ async def get_file( return length, headers -def _flatten_response_never_received(e): +def _flatten_response_never_received(e: BaseException) -> str: if hasattr(e, "reasons"): reasons = ", ".join( - _flatten_response_never_received(f.value) for f in e.reasons + _flatten_response_never_received(f.value) for f in e.reasons # type: ignore[attr-defined] ) return "%s:[%s]" % (type(e).__name__, reasons) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 4886626d5074..2b6d113544ca 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -162,7 +162,7 @@ def start(self, time_sec: float, name: str, method: str) -> None: with _in_flight_requests_lock: _in_flight_requests.add(self) - def stop(self, time_sec, response_code, sent_bytes): + def stop(self, time_sec: float, response_code: int, sent_bytes: int) -> None: with _in_flight_requests_lock: _in_flight_requests.discard(self) @@ -186,13 +186,13 @@ def stop(self, time_sec, response_code, sent_bytes): ) return - response_code = str(response_code) + response_code_str = str(response_code) - outgoing_responses_counter.labels(self.method, response_code).inc() + outgoing_responses_counter.labels(self.method, response_code_str).inc() response_count.labels(self.method, self.name, tag).inc() - response_timer.labels(self.method, self.name, tag, response_code).observe( + response_timer.labels(self.method, self.name, tag, response_code_str).observe( time_sec - self.start_ts ) @@ -221,7 +221,7 @@ def stop(self, time_sec, response_code, sent_bytes): # flight. self.update_metrics() - def update_metrics(self): + def update_metrics(self) -> None: """Updates the in flight metrics with values from this request.""" if not self.start_context: logger.error( diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 41f566b6487a..5ddb58a8a2ca 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -31,6 +31,7 @@ List, Optional, Tuple, + Type, TypeVar, cast, overload, @@ -41,6 +42,7 @@ from typing_extensions import Concatenate, Literal, ParamSpec from twisted.enterprise import adbapi +from twisted.internet.interfaces import IReactorCore from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig @@ -92,7 +94,9 @@ def make_pool( - reactor, db_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine + reactor: IReactorCore, + db_config: DatabaseConnectionConfig, + engine: BaseDatabaseEngine, ) -> adbapi.ConnectionPool: """Get the connection pool for the database.""" @@ -101,7 +105,7 @@ def make_pool( db_args = dict(db_config.config.get("args", {})) db_args.setdefault("cp_reconnect", True) - def _on_new_connection(conn): + def _on_new_connection(conn: Connection) -> None: # Ensure we have a logging context so we can correctly track queries, # etc. with LoggingContext("db.on_new_connection"): @@ -157,7 +161,11 @@ class LoggingDatabaseConnection: default_txn_name: str def cursor( - self, *, txn_name=None, after_callbacks=None, exception_callbacks=None + self, + *, + txn_name: Optional[str] = None, + after_callbacks: Optional[List["_CallbackListEntry"]] = None, + exception_callbacks: Optional[List["_CallbackListEntry"]] = None, ) -> "LoggingTransaction": if not txn_name: txn_name = self.default_txn_name @@ -183,11 +191,16 @@ def __enter__(self) -> "LoggingDatabaseConnection": self.conn.__enter__() return self - def __exit__(self, exc_type, exc_value, traceback) -> Optional[bool]: + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[types.TracebackType], + ) -> Optional[bool]: return self.conn.__exit__(exc_type, exc_value, traceback) # Proxy through any unknown lookups to the DB conn class. - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: return getattr(self.conn, name) @@ -391,17 +404,22 @@ def close(self) -> None: def __enter__(self) -> "LoggingTransaction": return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[types.TracebackType], + ) -> None: self.close() class PerformanceCounters: - def __init__(self): - self.current_counters = {} - self.previous_counters = {} + def __init__(self) -> None: + self.current_counters: Dict[str, Tuple[int, float]] = {} + self.previous_counters: Dict[str, Tuple[int, float]] = {} def update(self, key: str, duration_secs: float) -> None: - count, cum_time = self.current_counters.get(key, (0, 0)) + count, cum_time = self.current_counters.get(key, (0, 0.0)) count += 1 cum_time += duration_secs self.current_counters[key] = (count, cum_time) @@ -527,7 +545,7 @@ async def _check_safe_to_upsert(self) -> None: def start_profiling(self) -> None: self._previous_loop_ts = monotonic_time() - def loop(): + def loop() -> None: curr = self._current_txn_total_time prev = self._previous_txn_total_time self._previous_txn_total_time = curr @@ -1186,7 +1204,7 @@ def simple_upsert_txn_emulated( if lock: self.engine.lock_table(txn, table) - def _getwhere(key): + def _getwhere(key: str) -> str: # If the value we're passing in is None (aka NULL), we need to use # IS, not =, as NULL = NULL equals NULL (False). if keyvalues[key] is None: @@ -2258,7 +2276,7 @@ async def simple_search_list( term: Optional[str], col: str, retcols: Collection[str], - desc="simple_search_list", + desc: str = "simple_search_list", ) -> Optional[List[Dict[str, Any]]]: """Executes a SELECT query on the named table, which may return zero or more rows, returning the result as a list of dicts. diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index 1480a0f04829..d03555a5857b 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -23,6 +23,7 @@ from synapse.storage.databases.main.event_push_actions import ( EventPushActionsWorkerStore, ) +from synapse.storage.types import Cursor if TYPE_CHECKING: from synapse.server import HomeServer @@ -71,7 +72,7 @@ def __init__( self._last_user_visit_update = self._get_start_of_day() @wrap_as_background_process("read_forward_extremities") - async def _read_forward_extremities(self): + async def _read_forward_extremities(self) -> None: def fetch(txn): txn.execute( """ @@ -95,7 +96,7 @@ def fetch(txn): (x[0] - 1) * x[1] for x in res if x[1] ) - async def count_daily_e2ee_messages(self): + async def count_daily_e2ee_messages(self) -> int: """ Returns an estimate of the number of messages sent in the last day. @@ -115,7 +116,7 @@ def _count_messages(txn): return await self.db_pool.runInteraction("count_e2ee_messages", _count_messages) - async def count_daily_sent_e2ee_messages(self): + async def count_daily_sent_e2ee_messages(self) -> int: def _count_messages(txn): # This is good enough as if you have silly characters in your own # hostname then that's your own fault. @@ -136,7 +137,7 @@ def _count_messages(txn): "count_daily_sent_e2ee_messages", _count_messages ) - async def count_daily_active_e2ee_rooms(self): + async def count_daily_active_e2ee_rooms(self) -> int: def _count(txn): sql = """ SELECT COUNT(DISTINCT room_id) FROM events @@ -151,7 +152,7 @@ def _count(txn): "count_daily_active_e2ee_rooms", _count ) - async def count_daily_messages(self): + async def count_daily_messages(self) -> int: """ Returns an estimate of the number of messages sent in the last day. @@ -171,7 +172,7 @@ def _count_messages(txn): return await self.db_pool.runInteraction("count_messages", _count_messages) - async def count_daily_sent_messages(self): + async def count_daily_sent_messages(self) -> int: def _count_messages(txn): # This is good enough as if you have silly characters in your own # hostname then that's your own fault. @@ -192,7 +193,7 @@ def _count_messages(txn): "count_daily_sent_messages", _count_messages ) - async def count_daily_active_rooms(self): + async def count_daily_active_rooms(self) -> int: def _count(txn): sql = """ SELECT COUNT(DISTINCT room_id) FROM events @@ -226,7 +227,7 @@ async def count_monthly_users(self) -> int: "count_monthly_users", self._count_users, thirty_days_ago ) - def _count_users(self, txn, time_from): + def _count_users(self, txn: Cursor, time_from: int) -> int: """ Returns number of users seen in the past time_from period """ @@ -238,7 +239,10 @@ def _count_users(self, txn, time_from): ) u """ txn.execute(sql, (time_from,)) - (count,) = txn.fetchone() + # Mypy knows that fetchone() might return None if there are no rows. + # We know better: "SELECT COUNT(...) FROM ..." without any GROUP BY always + # returns exactly one row. + (count,) = txn.fetchone() # type: ignore[misc] return count async def count_r30_users(self) -> Dict[str, int]: @@ -453,7 +457,7 @@ def _count_r30v2_users(txn): "count_r30v2_users", _count_r30v2_users ) - def _get_start_of_day(self): + def _get_start_of_day(self) -> int: """ Returns millisecond unixtime for start of UTC day. """ diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 4e1d9647b7b8..59bbca2e3207 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -798,9 +798,11 @@ def get_stream_id_for_event_txn( self, txn: LoggingTransaction, event_id: str, - allow_none=False, - ) -> int: - return self.db_pool.simple_select_one_onecol_txn( + allow_none: bool = False, + ) -> Optional[int]: + # Type ignore: we pass keyvalues a Dict[str, str]; the function wants + # Dict[str, Any]. I think mypy is unhappy because Dict is invariant? + return self.db_pool.simple_select_one_onecol_txn( # type: ignore[call-overload] txn=txn, table="events", keyvalues={"event_id": event_id}, diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py index a7f6338e058d..0fc282866bc5 100644 --- a/synapse/storage/persist_events.py +++ b/synapse/storage/persist_events.py @@ -25,6 +25,7 @@ Collection, Deque, Dict, + Generator, Generic, Iterable, List, @@ -207,7 +208,7 @@ async def add_to_queue( return res - def _handle_queue(self, room_id): + def _handle_queue(self, room_id: str) -> None: """Attempts to handle the queue for a room if not already being handled. The queue's callback will be invoked with for each item in the queue, @@ -227,7 +228,7 @@ def _handle_queue(self, room_id): self._currently_persisting_rooms.add(room_id) - async def handle_queue_loop(): + async def handle_queue_loop() -> None: try: queue = self._get_drainining_queue(room_id) for item in queue: @@ -250,15 +251,17 @@ async def handle_queue_loop(): with PreserveLoggingContext(): item.deferred.callback(ret) finally: - queue = self._event_persist_queues.pop(room_id, None) - if queue: - self._event_persist_queues[room_id] = queue + remaining_queue = self._event_persist_queues.pop(room_id, None) + if remaining_queue: + self._event_persist_queues[room_id] = remaining_queue self._currently_persisting_rooms.discard(room_id) # set handle_queue_loop off in the background run_as_background_process("persist_events", handle_queue_loop) - def _get_drainining_queue(self, room_id): + def _get_drainining_queue( + self, room_id: str + ) -> Generator[_EventPersistQueueItem, None, None]: queue = self._event_persist_queues.setdefault(room_id, deque()) try: @@ -317,7 +320,9 @@ async def persist_events( for event, ctx in events_and_contexts: partitioned.setdefault(event.room_id, []).append((event, ctx)) - async def enqueue(item): + async def enqueue( + item: Tuple[str, List[Tuple[EventBase, EventContext]]] + ) -> Dict[str, str]: room_id, evs_ctxs = item return await self._event_persist_queue.add_to_queue( room_id, evs_ctxs, backfilled=backfilled @@ -1102,7 +1107,7 @@ async def _is_server_still_joined( return False - async def _handle_potentially_left_users(self, user_ids: Set[str]): + async def _handle_potentially_left_users(self, user_ids: Set[str]) -> None: """Given a set of remote users check if the server still shares a room with them. If not then mark those users' device cache as stale. """ diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 546d6bae6e56..c33df420841d 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -85,7 +85,7 @@ def prepare_database( database_engine: BaseDatabaseEngine, config: Optional[HomeServerConfig], databases: Collection[str] = ("main", "state"), -): +) -> None: """Prepares a physical database for usage. Will either create all necessary tables or upgrade from an older schema version. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index d1d58592145e..d4a1bd4f9d7d 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -62,7 +62,7 @@ class StateFilter: types: "frozendict[str, Optional[FrozenSet[str]]]" include_others: bool = False - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: # If `include_others` is set we canonicalise the filter by removing # wildcards from the types dictionary if self.include_others: @@ -138,7 +138,9 @@ def from_lazy_load_member_list(members: Iterable[str]) -> "StateFilter": ) @staticmethod - def freeze(types: Mapping[str, Optional[Collection[str]]], include_others: bool): + def freeze( + types: Mapping[str, Optional[Collection[str]]], include_others: bool + ) -> "StateFilter": """ Returns a (frozen) StateFilter with the same contents as the parameters specified here, which can be made of mutable types. diff --git a/synapse/storage/types.py b/synapse/storage/types.py index d7d6f1d90ecb..40536c183005 100644 --- a/synapse/storage/types.py +++ b/synapse/storage/types.py @@ -11,7 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Iterator, List, Mapping, Optional, Sequence, Tuple, Union +from types import TracebackType +from typing import Any, Iterator, List, Mapping, Optional, Sequence, Tuple, Type, Union from typing_extensions import Protocol @@ -86,5 +87,10 @@ def rollback(self) -> None: def __enter__(self) -> "Connection": ... - def __exit__(self, exc_type, exc_value, traceback) -> Optional[bool]: + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> Optional[bool]: ... From 57f6c496d0e26b1b455de936bd950e1899a5ae25 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 12 May 2022 18:16:32 +0100 Subject: [PATCH 180/263] URL preview cache expiry logs: INFO -> DEBUG, text clarifications (#12720) --- changelog.d/12720.misc | 1 + synapse/rest/media/v1/preview_url_resource.py | 30 +++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 changelog.d/12720.misc diff --git a/changelog.d/12720.misc b/changelog.d/12720.misc new file mode 100644 index 000000000000..01b427f200ae --- /dev/null +++ b/changelog.d/12720.misc @@ -0,0 +1 @@ +Drop the logging level of status messages for the URL preview cache expiry job from INFO to DEBUG. \ No newline at end of file diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 50383bdbd1c5..2b2db63bf7cc 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -668,7 +668,7 @@ async def _expire_url_cache_data(self) -> None: logger.debug("Running url preview cache expiry") if not (await self.store.db_pool.updates.has_completed_background_updates()): - logger.info("Still running DB updates; skipping expiry") + logger.debug("Still running DB updates; skipping url preview cache expiry") return def try_remove_parent_dirs(dirs: Iterable[str]) -> None: @@ -688,7 +688,9 @@ def try_remove_parent_dirs(dirs: Iterable[str]) -> None: # Failed, skip deleting the rest of the parent dirs if e.errno != errno.ENOTEMPTY: logger.warning( - "Failed to remove media directory: %r: %s", dir, e + "Failed to remove media directory while clearing url preview cache: %r: %s", + dir, + e, ) break @@ -703,7 +705,11 @@ def try_remove_parent_dirs(dirs: Iterable[str]) -> None: except FileNotFoundError: pass # If the path doesn't exist, meh except OSError as e: - logger.warning("Failed to remove media: %r: %s", media_id, e) + logger.warning( + "Failed to remove media while clearing url preview cache: %r: %s", + media_id, + e, + ) continue removed_media.append(media_id) @@ -714,9 +720,11 @@ def try_remove_parent_dirs(dirs: Iterable[str]) -> None: await self.store.delete_url_cache(removed_media) if removed_media: - logger.info("Deleted %d entries from url cache", len(removed_media)) + logger.debug( + "Deleted %d entries from url preview cache", len(removed_media) + ) else: - logger.debug("No entries removed from url cache") + logger.debug("No entries removed from url preview cache") # Now we delete old images associated with the url cache. # These may be cached for a bit on the client (i.e., they @@ -733,7 +741,9 @@ def try_remove_parent_dirs(dirs: Iterable[str]) -> None: except FileNotFoundError: pass # If the path doesn't exist, meh except OSError as e: - logger.warning("Failed to remove media: %r: %s", media_id, e) + logger.warning( + "Failed to remove media from url preview cache: %r: %s", media_id, e + ) continue dirs = self.filepaths.url_cache_filepath_dirs_to_delete(media_id) @@ -745,7 +755,9 @@ def try_remove_parent_dirs(dirs: Iterable[str]) -> None: except FileNotFoundError: pass # If the path doesn't exist, meh except OSError as e: - logger.warning("Failed to remove media: %r: %s", media_id, e) + logger.warning( + "Failed to remove media from url preview cache: %r: %s", media_id, e + ) continue removed_media.append(media_id) @@ -758,9 +770,9 @@ def try_remove_parent_dirs(dirs: Iterable[str]) -> None: await self.store.delete_url_cache_media(removed_media) if removed_media: - logger.info("Deleted %d media from url cache", len(removed_media)) + logger.debug("Deleted %d media from url preview cache", len(removed_media)) else: - logger.debug("No media removed from url cache") + logger.debug("No media removed from url preview cache") def _is_media(content_type: str) -> bool: From c9fc2c0d2260b4e77eda31a7a4a15b073d539db2 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 13 May 2022 12:15:51 +0200 Subject: [PATCH 181/263] Update issuer URL in example OIDC Keycloak config (#12727) * Update openid.md Newer versions of keycloak returning a 404 when using the `/auth` prefix. Related: https://github.com/matrix-org/synapse/issues/12714 --- changelog.d/12727.doc | 1 + docs/openid.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12727.doc diff --git a/changelog.d/12727.doc b/changelog.d/12727.doc new file mode 100644 index 000000000000..c41e50c85ba0 --- /dev/null +++ b/changelog.d/12727.doc @@ -0,0 +1 @@ +Update the OpenID Connect example for Keycloak to be compatible with newer versions of Keycloak. Contributed by @nhh. diff --git a/docs/openid.md b/docs/openid.md index 19cacaafefe0..e899db63d63b 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -159,7 +159,7 @@ Follow the [Getting Started Guide](https://www.keycloak.org/getting-started) to oidc_providers: - idp_id: keycloak idp_name: "My KeyCloak server" - issuer: "https://127.0.0.1:8443/auth/realms/{realm_name}" + issuer: "https://127.0.0.1:8443/realms/{realm_name}" client_id: "synapse" client_secret: "copy secret generated from above" scopes: ["openid", "profile"] From 39bed28b2843c79438d5cb51a6bb40e31c4420e7 Mon Sep 17 00:00:00 2001 From: Jess Porter Date: Fri, 13 May 2022 12:17:38 +0100 Subject: [PATCH 182/263] SpamChecker metrics (#12513) * add Measure blocks all over SpamChecker Signed-off-by: jesopo * fix test_spam_checker_may_join_room and test_threepid_invite_spamcheck * better changelog entry --- changelog.d/12513.feature | 1 + synapse/events/spamcheck.py | 81 +++++++++++++++++++++++---------- synapse/server.py | 2 +- tests/rest/client/test_rooms.py | 6 ++- 4 files changed, 64 insertions(+), 26 deletions(-) create mode 100644 changelog.d/12513.feature diff --git a/changelog.d/12513.feature b/changelog.d/12513.feature new file mode 100644 index 000000000000..01bf1d9d2cf6 --- /dev/null +++ b/changelog.d/12513.feature @@ -0,0 +1 @@ +Measure the time taken in spam-checking callbacks and expose those measurements as metrics. diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index 3b6795d40f6b..f30207376ae2 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -32,6 +32,7 @@ from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import RoomAlias, UserProfile from synapse.util.async_helpers import delay_cancellation, maybe_awaitable +from synapse.util.metrics import Measure if TYPE_CHECKING: import synapse.events @@ -162,7 +163,10 @@ def run(*args: Any, **kwargs: Any) -> Awaitable: class SpamChecker: - def __init__(self) -> None: + def __init__(self, hs: "synapse.server.HomeServer") -> None: + self.hs = hs + self.clock = hs.get_clock() + self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = [] self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = [] @@ -255,7 +259,10 @@ async def check_event_for_spam( will be used as the error message returned to the user. """ for callback in self._check_event_for_spam_callbacks: - res: Union[bool, str] = await delay_cancellation(callback(event)) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + res: Union[bool, str] = await delay_cancellation(callback(event)) if res: return res @@ -276,9 +283,12 @@ async def user_may_join_room( Whether the user may join the room """ for callback in self._user_may_join_room_callbacks: - may_join_room = await delay_cancellation( - callback(user_id, room_id, is_invited) - ) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_join_room = await delay_cancellation( + callback(user_id, room_id, is_invited) + ) if may_join_room is False: return False @@ -300,9 +310,12 @@ async def user_may_invite( True if the user may send an invite, otherwise False """ for callback in self._user_may_invite_callbacks: - may_invite = await delay_cancellation( - callback(inviter_userid, invitee_userid, room_id) - ) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_invite = await delay_cancellation( + callback(inviter_userid, invitee_userid, room_id) + ) if may_invite is False: return False @@ -328,9 +341,12 @@ async def user_may_send_3pid_invite( True if the user may send the invite, otherwise False """ for callback in self._user_may_send_3pid_invite_callbacks: - may_send_3pid_invite = await delay_cancellation( - callback(inviter_userid, medium, address, room_id) - ) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_send_3pid_invite = await delay_cancellation( + callback(inviter_userid, medium, address, room_id) + ) if may_send_3pid_invite is False: return False @@ -348,7 +364,10 @@ async def user_may_create_room(self, userid: str) -> bool: True if the user may create a room, otherwise False """ for callback in self._user_may_create_room_callbacks: - may_create_room = await delay_cancellation(callback(userid)) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_create_room = await delay_cancellation(callback(userid)) if may_create_room is False: return False @@ -369,9 +388,12 @@ async def user_may_create_room_alias( True if the user may create a room alias, otherwise False """ for callback in self._user_may_create_room_alias_callbacks: - may_create_room_alias = await delay_cancellation( - callback(userid, room_alias) - ) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_create_room_alias = await delay_cancellation( + callback(userid, room_alias) + ) if may_create_room_alias is False: return False @@ -390,7 +412,10 @@ async def user_may_publish_room(self, userid: str, room_id: str) -> bool: True if the user may publish the room, otherwise False """ for callback in self._user_may_publish_room_callbacks: - may_publish_room = await delay_cancellation(callback(userid, room_id)) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + may_publish_room = await delay_cancellation(callback(userid, room_id)) if may_publish_room is False: return False @@ -412,9 +437,13 @@ async def check_username_for_spam(self, user_profile: UserProfile) -> bool: True if the user is spammy. """ for callback in self._check_username_for_spam_callbacks: - # Make a copy of the user profile object to ensure the spam checker cannot - # modify it. - if await delay_cancellation(callback(user_profile.copy())): + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + # Make a copy of the user profile object to ensure the spam checker cannot + # modify it. + res = await delay_cancellation(callback(user_profile.copy())) + if res: return True return False @@ -442,9 +471,12 @@ async def check_registration_for_spam( """ for callback in self._check_registration_for_spam_callbacks: - behaviour = await delay_cancellation( - callback(email_threepid, username, request_info, auth_provider_id) - ) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + behaviour = await delay_cancellation( + callback(email_threepid, username, request_info, auth_provider_id) + ) assert isinstance(behaviour, RegistrationBehaviour) if behaviour != RegistrationBehaviour.ALLOW: return behaviour @@ -486,7 +518,10 @@ async def check_media_file_for_spam( """ for callback in self._check_media_file_for_spam_callbacks: - spam = await delay_cancellation(callback(file_wrapper, file_info)) + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + spam = await delay_cancellation(callback(file_wrapper, file_info)) if spam: return True diff --git a/synapse/server.py b/synapse/server.py index 7daa7b9334c8..ee60cce8ebce 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -681,7 +681,7 @@ def get_stats_handler(self) -> StatsHandler: @cache_in_self def get_spam_checker(self) -> SpamChecker: - return SpamChecker() + return SpamChecker(self) @cache_in_self def get_third_party_event_rules(self) -> ThirdPartyEventRules: diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index ad416e2fd8c0..d0197aca94a0 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -925,7 +925,7 @@ async def user_may_join_room( ) -> bool: return return_value - callback_mock = Mock(side_effect=user_may_join_room) + callback_mock = Mock(side_effect=user_may_join_room, spec=lambda *x: None) self.hs.get_spam_checker()._user_may_join_room_callbacks.append(callback_mock) # Join a first room, without being invited to it. @@ -2856,7 +2856,9 @@ def test_threepid_invite_spamcheck(self) -> None: # Add a mock to the spamchecker callbacks for user_may_send_3pid_invite. Make it # allow everything for now. - mock = Mock(return_value=make_awaitable(True)) + # `spec` argument is needed for this function mock to have `__qualname__`, which + # is needed for `Measure` metrics buried in SpamChecker. + mock = Mock(return_value=make_awaitable(True), spec=lambda *x: None) self.hs.get_spam_checker()._user_may_send_3pid_invite_callbacks.append(mock) # Send a 3PID invite into the room and check that it succeeded. From aec69d2481e9ea1d8ea1c0ffce1706a65a7896a8 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 13 May 2022 12:35:31 +0100 Subject: [PATCH 183/263] Another batch of type annotations (#12726) --- changelog.d/12726.misc | 1 + mypy.ini | 21 ++++++++++++ synapse/handlers/e2e_keys.py | 29 ++++++---------- synapse/http/connectproxyclient.py | 39 +++++++++++++--------- synapse/http/proxyagent.py | 2 +- synapse/logging/_remote.py | 20 ++++++----- synapse/logging/formatter.py | 14 +++++--- synapse/logging/handlers.py | 4 +-- synapse/logging/scopecontextmanager.py | 28 ++++++++++++---- synapse/storage/background_updates.py | 19 ++++++++--- synapse/types.py | 46 ++++++++++++++++---------- 11 files changed, 144 insertions(+), 79 deletions(-) create mode 100644 changelog.d/12726.misc diff --git a/changelog.d/12726.misc b/changelog.d/12726.misc new file mode 100644 index 000000000000..b07e1b52ee7c --- /dev/null +++ b/changelog.d/12726.misc @@ -0,0 +1 @@ +Add type annotations to increase the number of modules passing `disallow-untyped-defs`. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 8478dd9e510b..9ae7ad211c54 100644 --- a/mypy.ini +++ b/mypy.ini @@ -128,15 +128,30 @@ disallow_untyped_defs = True [mypy-synapse.http.federation.*] disallow_untyped_defs = True +[mypy-synapse.http.connectproxyclient] +disallow_untyped_defs = True + +[mypy-synapse.http.proxyagent] +disallow_untyped_defs = True + [mypy-synapse.http.request_metrics] disallow_untyped_defs = True [mypy-synapse.http.server] disallow_untyped_defs = True +[mypy-synapse.logging._remote] +disallow_untyped_defs = True + [mypy-synapse.logging.context] disallow_untyped_defs = True +[mypy-synapse.logging.formatter] +disallow_untyped_defs = True + +[mypy-synapse.logging.handlers] +disallow_untyped_defs = True + [mypy-synapse.metrics.*] disallow_untyped_defs = True @@ -166,6 +181,9 @@ disallow_untyped_defs = True [mypy-synapse.state.*] disallow_untyped_defs = True +[mypy-synapse.storage.databases.background_updates] +disallow_untyped_defs = True + [mypy-synapse.storage.databases.main.account_data] disallow_untyped_defs = True @@ -232,6 +250,9 @@ disallow_untyped_defs = True [mypy-synapse.streams.*] disallow_untyped_defs = True +[mypy-synapse.types] +disallow_untyped_defs = True + [mypy-synapse.util.*] disallow_untyped_defs = True diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index d6714228ef41..e6c2cfb8c8e7 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -15,7 +15,7 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple import attr from canonicaljson import encode_canonical_json @@ -1105,22 +1105,19 @@ async def _get_e2e_cross_signing_verify_key( # can request over federation raise NotFoundError("No %s key found for %s" % (key_type, user_id)) - ( - key, - key_id, - verify_key, - ) = await self._retrieve_cross_signing_keys_for_remote_user(user, key_type) - - if key is None: + cross_signing_keys = await self._retrieve_cross_signing_keys_for_remote_user( + user, key_type + ) + if cross_signing_keys is None: raise NotFoundError("No %s key found for %s" % (key_type, user_id)) - return key, key_id, verify_key + return cross_signing_keys async def _retrieve_cross_signing_keys_for_remote_user( self, user: UserID, desired_key_type: str, - ) -> Tuple[Optional[dict], Optional[str], Optional[VerifyKey]]: + ) -> Optional[Tuple[Dict[str, Any], str, VerifyKey]]: """Queries cross-signing keys for a remote user and saves them to the database Only the key specified by `key_type` will be returned, while all retrieved keys @@ -1146,12 +1143,10 @@ async def _retrieve_cross_signing_keys_for_remote_user( type(e), e, ) - return None, None, None + return None # Process each of the retrieved cross-signing keys - desired_key = None - desired_key_id = None - desired_verify_key = None + desired_key_data = None retrieved_device_ids = [] for key_type in ["master", "self_signing"]: key_content = remote_result.get(key_type + "_key") @@ -1196,9 +1191,7 @@ async def _retrieve_cross_signing_keys_for_remote_user( # If this is the desired key type, save it and its ID/VerifyKey if key_type == desired_key_type: - desired_key = key_content - desired_verify_key = verify_key - desired_key_id = key_id + desired_key_data = key_content, key_id, verify_key # At the same time, store this key in the db for subsequent queries await self.store.set_e2e_cross_signing_key( @@ -1212,7 +1205,7 @@ async def _retrieve_cross_signing_keys_for_remote_user( user.to_string(), retrieved_device_ids ) - return desired_key, desired_key_id, desired_verify_key + return desired_key_data def _check_cross_signing_key( diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 203e995bb77d..23a60af17184 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -14,15 +14,22 @@ import base64 import logging -from typing import Optional +from typing import Optional, Union import attr from zope.interface import implementer from twisted.internet import defer, protocol from twisted.internet.error import ConnectError -from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint +from twisted.internet.interfaces import ( + IAddress, + IConnector, + IProtocol, + IReactorCore, + IStreamClientEndpoint, +) from twisted.internet.protocol import ClientFactory, Protocol, connectionDone +from twisted.python.failure import Failure from twisted.web import http logger = logging.getLogger(__name__) @@ -81,14 +88,14 @@ def __init__( self._port = port self._proxy_creds = proxy_creds - def __repr__(self): + def __repr__(self) -> str: return "" % (self._proxy_endpoint,) # Mypy encounters a false positive here: it complains that ClientFactory # is incompatible with IProtocolFactory. But ClientFactory inherits from # Factory, which implements IProtocolFactory. So I think this is a bug # in mypy-zope. - def connect(self, protocolFactory: ClientFactory): # type: ignore[override] + def connect(self, protocolFactory: ClientFactory) -> "defer.Deferred[IProtocol]": # type: ignore[override] f = HTTPProxiedClientFactory( self._host, self._port, protocolFactory, self._proxy_creds ) @@ -125,10 +132,10 @@ def __init__( self.proxy_creds = proxy_creds self.on_connection: "defer.Deferred[None]" = defer.Deferred() - def startedConnecting(self, connector): + def startedConnecting(self, connector: IConnector) -> None: return self.wrapped_factory.startedConnecting(connector) - def buildProtocol(self, addr): + def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": wrapped_protocol = self.wrapped_factory.buildProtocol(addr) if wrapped_protocol is None: raise TypeError("buildProtocol produced None instead of a Protocol") @@ -141,13 +148,13 @@ def buildProtocol(self, addr): self.proxy_creds, ) - def clientConnectionFailed(self, connector, reason): + def clientConnectionFailed(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy failed: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) return self.wrapped_factory.clientConnectionFailed(connector, reason) - def clientConnectionLost(self, connector, reason): + def clientConnectionLost(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy lost: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) @@ -191,10 +198,10 @@ def __init__( ) self.http_setup_client.on_connected.addCallback(self.proxyConnected) - def connectionMade(self): + def connectionMade(self) -> None: self.http_setup_client.makeConnection(self.transport) - def connectionLost(self, reason=connectionDone): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self.wrapped_protocol.connected: self.wrapped_protocol.connectionLost(reason) @@ -203,7 +210,7 @@ def connectionLost(self, reason=connectionDone): if not self.connected_deferred.called: self.connected_deferred.errback(reason) - def proxyConnected(self, _): + def proxyConnected(self, _: Union[None, "defer.Deferred[None]"]) -> None: self.wrapped_protocol.makeConnection(self.transport) self.connected_deferred.callback(self.wrapped_protocol) @@ -213,7 +220,7 @@ def proxyConnected(self, _): if buf: self.wrapped_protocol.dataReceived(buf) - def dataReceived(self, data: bytes): + def dataReceived(self, data: bytes) -> None: # if we've set up the HTTP protocol, we can send the data there if self.wrapped_protocol.connected: return self.wrapped_protocol.dataReceived(data) @@ -243,7 +250,7 @@ def __init__( self.proxy_creds = proxy_creds self.on_connected: "defer.Deferred[None]" = defer.Deferred() - def connectionMade(self): + def connectionMade(self) -> None: logger.debug("Connected to proxy, sending CONNECT") self.sendCommand(b"CONNECT", b"%s:%d" % (self.host, self.port)) @@ -257,14 +264,14 @@ def connectionMade(self): self.endHeaders() - def handleStatus(self, version: bytes, status: bytes, message: bytes): + def handleStatus(self, version: bytes, status: bytes, message: bytes) -> None: logger.debug("Got Status: %s %s %s", status, message, version) if status != b"200": raise ProxyConnectError(f"Unexpected status on CONNECT: {status!s}") - def handleEndHeaders(self): + def handleEndHeaders(self) -> None: logger.debug("End Headers") self.on_connected.callback(None) - def handleResponse(self, body): + def handleResponse(self, body: bytes) -> None: pass diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index a16dde23807f..b2a50c910507 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -245,7 +245,7 @@ def http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, tls_options_factory: Optional[IPolicyForHTTPS], - **kwargs, + **kwargs: object, ) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]: """Parses an http proxy setting and returns an endpoint for the proxy diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py index 475756f1db64..5a61b21eaf7e 100644 --- a/synapse/logging/_remote.py +++ b/synapse/logging/_remote.py @@ -31,7 +31,11 @@ TCP4ClientEndpoint, TCP6ClientEndpoint, ) -from twisted.internet.interfaces import IPushProducer, IStreamClientEndpoint +from twisted.internet.interfaces import ( + IPushProducer, + IReactorTCP, + IStreamClientEndpoint, +) from twisted.internet.protocol import Factory, Protocol from twisted.internet.tcp import Connection from twisted.python.failure import Failure @@ -59,14 +63,14 @@ class LogProducer: _buffer: Deque[logging.LogRecord] _paused: bool = attr.ib(default=False, init=False) - def pauseProducing(self): + def pauseProducing(self) -> None: self._paused = True - def stopProducing(self): + def stopProducing(self) -> None: self._paused = True self._buffer = deque() - def resumeProducing(self): + def resumeProducing(self) -> None: # If we're already producing, nothing to do. self._paused = False @@ -102,8 +106,8 @@ def __init__( host: str, port: int, maximum_buffer: int = 1000, - level=logging.NOTSET, - _reactor=None, + level: int = logging.NOTSET, + _reactor: Optional[IReactorTCP] = None, ): super().__init__(level=level) self.host = host @@ -118,7 +122,7 @@ def __init__( if _reactor is None: from twisted.internet import reactor - _reactor = reactor + _reactor = reactor # type: ignore[assignment] try: ip = ip_address(self.host) @@ -139,7 +143,7 @@ def __init__( self._stopping = False self._connect() - def close(self): + def close(self) -> None: self._stopping = True self._service.stopService() diff --git a/synapse/logging/formatter.py b/synapse/logging/formatter.py index c0f12ecd15b8..c88b8ae5450f 100644 --- a/synapse/logging/formatter.py +++ b/synapse/logging/formatter.py @@ -16,6 +16,8 @@ import logging import traceback from io import StringIO +from types import TracebackType +from typing import Optional, Tuple, Type class LogFormatter(logging.Formatter): @@ -28,10 +30,14 @@ class LogFormatter(logging.Formatter): where it was caught are logged). """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def formatException(self, ei): + def formatException( + self, + ei: Tuple[ + Optional[Type[BaseException]], + Optional[BaseException], + Optional[TracebackType], + ], + ) -> str: sio = StringIO() (typ, val, tb) = ei diff --git a/synapse/logging/handlers.py b/synapse/logging/handlers.py index 478b5274942b..dec2a2c3dd1a 100644 --- a/synapse/logging/handlers.py +++ b/synapse/logging/handlers.py @@ -49,7 +49,7 @@ def __init__( ) self._flushing_thread.start() - def on_reactor_running(): + def on_reactor_running() -> None: self._reactor_started = True reactor_to_use: IReactorCore @@ -74,7 +74,7 @@ def shouldFlush(self, record: LogRecord) -> bool: else: return True - def _flush_periodically(self): + def _flush_periodically(self) -> None: """ Whilst this handler is active, flush the handler periodically. """ diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py index d57e7c5324f8..a26a1a58e7d6 100644 --- a/synapse/logging/scopecontextmanager.py +++ b/synapse/logging/scopecontextmanager.py @@ -13,6 +13,8 @@ # limitations under the License.import logging import logging +from types import TracebackType +from typing import Optional, Type from opentracing import Scope, ScopeManager @@ -107,19 +109,26 @@ class _LogContextScope(Scope): and - if enter_logcontext was set - the logcontext is finished too. """ - def __init__(self, manager, span, logcontext, enter_logcontext, finish_on_close): + def __init__( + self, + manager: LogContextScopeManager, + span, + logcontext, + enter_logcontext: bool, + finish_on_close: bool, + ): """ Args: - manager (LogContextScopeManager): + manager: the manager that is responsible for this scope. span (Span): the opentracing span which this scope represents the local lifetime for. logcontext (LogContext): the logcontext to which this scope is attached. - enter_logcontext (Boolean): + enter_logcontext: if True the logcontext will be exited when the scope is finished - finish_on_close (Boolean): + finish_on_close: if True finish the span when the scope is closed """ super().__init__(manager, span) @@ -127,16 +136,21 @@ def __init__(self, manager, span, logcontext, enter_logcontext, finish_on_close) self._finish_on_close = finish_on_close self._enter_logcontext = enter_logcontext - def __exit__(self, exc_type, value, traceback): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: if exc_type == twisted.internet.defer._DefGen_Return: # filter out defer.returnValue() calls exc_type = value = traceback = None super().__exit__(exc_type, value, traceback) - def __str__(self): + def __str__(self) -> str: return f"Scope<{self.span}>" - def close(self): + def close(self) -> None: active_scope = self.manager.active if active_scope is not self: logger.error( diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 08c6eabc6d1a..c2bbbb574e75 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -12,20 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +from types import TracebackType from typing import ( TYPE_CHECKING, + Any, AsyncContextManager, Awaitable, Callable, Dict, Iterable, + List, Optional, + Type, ) import attr from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.storage.types import Connection +from synapse.storage.types import Connection, Cursor from synapse.types import JsonDict from synapse.util import Clock, json_encoder @@ -74,7 +78,12 @@ async def __aenter__(self) -> int: return self._update_duration_ms - async def __aexit__(self, *exc) -> None: + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: pass @@ -352,7 +361,7 @@ async def do_next_background_update(self, sleep: bool = True) -> bool: True if we have finished running all the background updates, otherwise False """ - def get_background_updates_txn(txn): + def get_background_updates_txn(txn: Cursor) -> List[Dict[str, Any]]: txn.execute( """ SELECT update_name, depends_on FROM background_updates @@ -469,7 +478,7 @@ def register_background_update_handler( self, update_name: str, update_handler: Callable[[JsonDict, int], Awaitable[int]], - ): + ) -> None: """Register a handler for doing a background update. The handler should take two arguments: @@ -603,7 +612,7 @@ def create_index_sqlite(conn: Connection) -> None: else: runner = create_index_sqlite - async def updater(progress, batch_size): + async def updater(progress: JsonDict, batch_size: int) -> int: if runner is not None: logger.info("Adding index %s to %s", index_name, table) await self.db_pool.runWithConnection(runner) diff --git a/synapse/types.py b/synapse/types.py index 9ac688b23b28..325332a6e00f 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -24,6 +24,7 @@ Mapping, Match, MutableMapping, + NoReturn, Optional, Set, Tuple, @@ -35,6 +36,7 @@ import attr from frozendict import frozendict from signedjson.key import decode_verify_key_bytes +from signedjson.types import VerifyKey from typing_extensions import TypedDict from unpaddedbase64 import decode_base64 from zope.interface import Interface @@ -55,6 +57,7 @@ if TYPE_CHECKING: from synapse.appservice.api import ApplicationService from synapse.storage.databases.main import DataStore, PurgeEventsStore + from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore # Define a state map type from type/state_key to T (usually an event ID or # event) @@ -114,7 +117,7 @@ class Requester: app_service: Optional["ApplicationService"] authenticated_entity: str - def serialize(self): + def serialize(self) -> Dict[str, Any]: """Converts self to a type that can be serialized as JSON, and then deserialized by `deserialize` @@ -132,7 +135,9 @@ def serialize(self): } @staticmethod - def deserialize(store, input): + def deserialize( + store: "ApplicationServiceWorkerStore", input: Dict[str, Any] + ) -> "Requester": """Converts a dict that was produced by `serialize` back into a Requester. @@ -236,10 +241,10 @@ class DomainSpecificString(metaclass=abc.ABCMeta): domain: str # Because this is a frozen class, it is deeply immutable. - def __copy__(self): + def __copy__(self: DS) -> DS: return self - def __deepcopy__(self, memo): + def __deepcopy__(self: DS, memo: Dict[str, object]) -> DS: return self @classmethod @@ -729,12 +734,14 @@ async def to_string(self, store: "DataStore") -> str: ) @property - def room_stream_id(self): + def room_stream_id(self) -> int: return self.room_key.stream - def copy_and_advance(self, key, new_value) -> "StreamToken": + def copy_and_advance(self, key: str, new_value: Any) -> "StreamToken": """Advance the given key in the token to a new value if and only if the new value is after the old value. + + :raises TypeError: if `key` is not the one of the keys tracked by a StreamToken. """ if key == "room_key": new_token = self.copy_and_replace( @@ -751,7 +758,7 @@ def copy_and_advance(self, key, new_value) -> "StreamToken": else: return self - def copy_and_replace(self, key, new_value) -> "StreamToken": + def copy_and_replace(self, key: str, new_value: Any) -> "StreamToken": return attr.evolve(self, **{key: new_value}) @@ -793,14 +800,14 @@ class ThirdPartyInstanceID: # Deny iteration because it will bite you if you try to create a singleton # set by: # users = set(user) - def __iter__(self): + def __iter__(self) -> NoReturn: raise ValueError("Attempted to iterate a %s" % (type(self).__name__,)) # Because this class is a frozen class, it is deeply immutable. - def __copy__(self): + def __copy__(self) -> "ThirdPartyInstanceID": return self - def __deepcopy__(self, memo): + def __deepcopy__(self, memo: Dict[str, object]) -> "ThirdPartyInstanceID": return self @classmethod @@ -852,25 +859,28 @@ def __bool__(self) -> bool: return bool(self.changed or self.left) -def get_verify_key_from_cross_signing_key(key_info): +def get_verify_key_from_cross_signing_key( + key_info: Mapping[str, Any] +) -> Tuple[str, VerifyKey]: """Get the key ID and signedjson verify key from a cross-signing key dict Args: - key_info (dict): a cross-signing key dict, which must have a "keys" + key_info: a cross-signing key dict, which must have a "keys" property that has exactly one item in it Returns: - (str, VerifyKey): the key ID and verify key for the cross-signing key + the key ID and verify key for the cross-signing key """ - # make sure that exactly one key is provided + # make sure that a `keys` field is provided if "keys" not in key_info: raise ValueError("Invalid key") keys = key_info["keys"] - if len(keys) != 1: - raise ValueError("Invalid key") - # and return that one key - for key_id, key_data in keys.items(): + # and that it contains exactly one key + if len(keys) == 1: + key_id, key_data = next(iter(keys.items())) return key_id, decode_verify_key_bytes(key_id, decode_base64(key_data)) + else: + raise ValueError("Invalid key") @attr.s(auto_attribs=True, frozen=True, slots=True) From 90131044297ac5378fb381050f4068784dc206a8 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 13 May 2022 15:30:15 +0200 Subject: [PATCH 184/263] Don't create an empty room when checking for MAU limits (#12713) --- changelog.d/12713.bugfix | 1 + .../resource_limits_server_notices.py | 40 +++-------- .../server_notices/server_notices_manager.py | 70 ++++++++++++------- .../test_resource_limits_server_notices.py | 11 ++- 4 files changed, 66 insertions(+), 56 deletions(-) create mode 100644 changelog.d/12713.bugfix diff --git a/changelog.d/12713.bugfix b/changelog.d/12713.bugfix new file mode 100644 index 000000000000..91e70f102c5d --- /dev/null +++ b/changelog.d/12713.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.30.0 where empty rooms could be automatically created if a monthly active users limit is set. diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py index 015dd08f05e4..b5f3a0c74e9e 100644 --- a/synapse/server_notices/resource_limits_server_notices.py +++ b/synapse/server_notices/resource_limits_server_notices.py @@ -21,7 +21,6 @@ ServerNoticeMsgType, ) from synapse.api.errors import AuthError, ResourceLimitError, SynapseError -from synapse.server_notices.server_notices_manager import SERVER_NOTICE_ROOM_TAG if TYPE_CHECKING: from synapse.server import HomeServer @@ -71,18 +70,19 @@ async def maybe_send_server_notice_to_user(self, user_id: str) -> None: # In practice, not sure we can ever get here return - room_id = await self._server_notices_manager.get_or_create_notice_room_for_user( + # Check if there's a server notice room for this user. + room_id = await self._server_notices_manager.maybe_get_notice_room_for_user( user_id ) - if not room_id: - logger.warning("Failed to get server notices room") - return - - await self._check_and_set_tags(user_id, room_id) - - # Determine current state of room - currently_blocked, ref_events = await self._is_room_currently_blocked(room_id) + if room_id is not None: + # Determine current state of room + currently_blocked, ref_events = await self._is_room_currently_blocked( + room_id + ) + else: + currently_blocked = False + ref_events = [] limit_msg = None limit_type = None @@ -161,26 +161,6 @@ async def _apply_limit_block_notification( user_id, content, EventTypes.Pinned, "" ) - async def _check_and_set_tags(self, user_id: str, room_id: str) -> None: - """ - Since server notices rooms were originally not with tags, - important to check that tags have been set correctly - Args: - user_id(str): the user in question - room_id(str): the server notices room for that user - """ - tags = await self._store.get_tags_for_room(user_id, room_id) - need_to_set_tag = True - if tags: - if SERVER_NOTICE_ROOM_TAG in tags: - # tag already present, nothing to do here - need_to_set_tag = False - if need_to_set_tag: - max_id = await self._account_data_handler.add_tag_to_room( - user_id, room_id, SERVER_NOTICE_ROOM_TAG, {} - ) - self._notifier.on_new_event("account_data_key", max_id, users=[user_id]) - async def _is_room_currently_blocked(self, room_id: str) -> Tuple[bool, List[str]]: """ Determines if the room is currently blocked diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 48eae5fa062a..c2c37e1015ce 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -90,6 +90,35 @@ async def send_notice( ) return event + @cached() + async def maybe_get_notice_room_for_user(self, user_id: str) -> Optional[str]: + """Try to look up the server notice room for this user if it exists. + + Does not create one if none can be found. + + Args: + user_id: the user we want a server notice room for. + + Returns: + The room's ID, or None if no room could be found. + """ + rooms = await self._store.get_rooms_for_local_user_where_membership_is( + user_id, [Membership.INVITE, Membership.JOIN] + ) + for room in rooms: + # it's worth noting that there is an asymmetry here in that we + # expect the user to be invited or joined, but the system user must + # be joined. This is kinda deliberate, in that if somebody somehow + # manages to invite the system user to a room, that doesn't make it + # the server notices room. + user_ids = await self._store.get_users_in_room(room.room_id) + if len(user_ids) <= 2 and self.server_notices_mxid in user_ids: + # we found a room which our user shares with the system notice + # user + return room.room_id + + return None + @cached() async def get_or_create_notice_room_for_user(self, user_id: str) -> str: """Get the room for notices for a given user @@ -112,31 +141,20 @@ async def get_or_create_notice_room_for_user(self, user_id: str) -> str: self.server_notices_mxid, authenticated_entity=self._server_name ) - rooms = await self._store.get_rooms_for_local_user_where_membership_is( - user_id, [Membership.INVITE, Membership.JOIN] - ) - for room in rooms: - # it's worth noting that there is an asymmetry here in that we - # expect the user to be invited or joined, but the system user must - # be joined. This is kinda deliberate, in that if somebody somehow - # manages to invite the system user to a room, that doesn't make it - # the server notices room. - user_ids = await self._store.get_users_in_room(room.room_id) - if len(user_ids) <= 2 and self.server_notices_mxid in user_ids: - # we found a room which our user shares with the system notice - # user - logger.info( - "Using existing server notices room %s for user %s", - room.room_id, - user_id, - ) - await self._update_notice_user_profile_if_changed( - requester, - room.room_id, - self._config.servernotices.server_notices_mxid_display_name, - self._config.servernotices.server_notices_mxid_avatar_url, - ) - return room.room_id + room_id = await self.maybe_get_notice_room_for_user(user_id) + if room_id is not None: + logger.info( + "Using existing server notices room %s for user %s", + room_id, + user_id, + ) + await self._update_notice_user_profile_if_changed( + requester, + room_id, + self._config.servernotices.server_notices_mxid_display_name, + self._config.servernotices.server_notices_mxid_avatar_url, + ) + return room_id # apparently no existing notice room: create a new one logger.info("Creating server notices room for %s", user_id) @@ -166,6 +184,8 @@ async def get_or_create_notice_room_for_user(self, user_id: str) -> str: ) room_id = info["room_id"] + self.maybe_get_notice_room_for_user.invalidate((user_id,)) + max_id = await self._account_data_handler.add_tag_to_room( user_id, room_id, SERVER_NOTICE_ROOM_TAG, {} ) diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index 9ee9509d3a96..07e29788e5be 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -75,6 +75,9 @@ def prepare(self, reactor, clock, hs): self._rlsn._server_notices_manager.get_or_create_notice_room_for_user = Mock( return_value=make_awaitable("!something:localhost") ) + self._rlsn._server_notices_manager.maybe_get_notice_room_for_user = Mock( + return_value=make_awaitable("!something:localhost") + ) self._rlsn._store.add_tag_to_room = Mock(return_value=make_awaitable(None)) self._rlsn._store.get_tags_for_room = Mock(return_value=make_awaitable({})) @@ -102,6 +105,7 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice(self): ) self.get_success(self._rlsn.maybe_send_server_notice_to_user(self.user_id)) # Would be better to check the content, but once == remove blocking event + self._rlsn._server_notices_manager.maybe_get_notice_room_for_user.assert_called_once() self._send_notice.assert_called_once() def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self): @@ -300,7 +304,10 @@ def test_no_invite_without_notice(self): hasn't been reached (since it's the only user and the limit is 5), so users shouldn't receive a server notice. """ - self.register_user("user", "password") + m = Mock(return_value=make_awaitable(None)) + self._rlsn._server_notices_manager.maybe_get_notice_room_for_user = m + + user_id = self.register_user("user", "password") tok = self.login("user", "password") channel = self.make_request("GET", "/sync?timeout=0", access_token=tok) @@ -309,6 +316,8 @@ def test_no_invite_without_notice(self): "rooms", channel.json_body, "Got invites without server notice" ) + m.assert_called_once_with(user_id) + def test_invite_with_notice(self): """Tests that, if the MAU limit is hit, the server notices user invites each user to a room in which it has sent a notice. From e8ae472d3b991362bfb48fb319f386163b2d5e76 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 13 May 2022 17:45:47 +0200 Subject: [PATCH 185/263] Update configs used by Complement to allow more invites (#12731) --- changelog.d/12731.misc | 1 + docker/complement/conf-workers/workers-shared.yaml | 12 ++++++++++++ docker/complement/conf/homeserver.yaml | 12 ++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 changelog.d/12731.misc diff --git a/changelog.d/12731.misc b/changelog.d/12731.misc new file mode 100644 index 000000000000..962100d516c1 --- /dev/null +++ b/changelog.d/12731.misc @@ -0,0 +1 @@ +Update configs used by Complement to allow more invites/3PID validations during tests. \ No newline at end of file diff --git a/docker/complement/conf-workers/workers-shared.yaml b/docker/complement/conf-workers/workers-shared.yaml index 8b6987037715..86ee11ecd0e5 100644 --- a/docker/complement/conf-workers/workers-shared.yaml +++ b/docker/complement/conf-workers/workers-shared.yaml @@ -53,6 +53,18 @@ rc_joins: per_second: 9999 burst_count: 9999 +rc_3pid_validation: + per_second: 1000 + burst_count: 1000 + +rc_invites: + per_room: + per_second: 1000 + burst_count: 1000 + per_user: + per_second: 1000 + burst_count: 1000 + federation_rr_transactions_per_room_per_second: 9999 ## Experimental Features ## diff --git a/docker/complement/conf/homeserver.yaml b/docker/complement/conf/homeserver.yaml index 174f87f52ee0..e2be540bbb9e 100644 --- a/docker/complement/conf/homeserver.yaml +++ b/docker/complement/conf/homeserver.yaml @@ -87,6 +87,18 @@ rc_joins: per_second: 9999 burst_count: 9999 +rc_3pid_validation: + per_second: 1000 + burst_count: 1000 + +rc_invites: + per_room: + per_second: 1000 + burst_count: 1000 + per_user: + per_second: 1000 + burst_count: 1000 + federation_rr_transactions_per_room_per_second: 9999 ## API Configuration ## From cde8af9a495cbc7f3d0207e3f17c37eddaee34e1 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 13 May 2022 12:32:39 -0700 Subject: [PATCH 186/263] Add config flags to allow for cache auto-tuning (#12701) --- changelog.d/12701.feature | 1 + docs/sample_config.yaml | 18 +++ .../configuration/config_documentation.md | 17 ++- synapse/config/cache.py | 33 +++++ synapse/metrics/jemalloc.py | 114 +++++++++++------- synapse/util/caches/lrucache.py | 79 ++++++++++-- tests/util/test_lrucache.py | 58 ++++++++- 7 files changed, 266 insertions(+), 54 deletions(-) create mode 100644 changelog.d/12701.feature diff --git a/changelog.d/12701.feature b/changelog.d/12701.feature new file mode 100644 index 000000000000..bb2264602c84 --- /dev/null +++ b/changelog.d/12701.feature @@ -0,0 +1 @@ +Add a config options to allow for auto-tuning of caches. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 03a0f6314cdd..05a3606da043 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -784,6 +784,24 @@ caches: # #cache_entry_ttl: 30m + # This flag enables cache autotuning, and is further specified by the sub-options `max_cache_memory_usage`, + # `target_cache_memory_usage`, `min_cache_ttl`. These flags work in conjunction with each other to maintain + # a balance between cache memory usage and cache entry availability. You must be using jemalloc to utilize + # this option, and all three of the options must be specified for this feature to work. + #cache_autotuning: + # This flag sets a ceiling on much memory the cache can use before caches begin to be continuously evicted. + # They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + # the flag below, or until the `min_cache_ttl` is hit. + #max_cache_memory_usage: 1024M + + # This flag sets a rough target for the desired memory usage of the caches. + #target_cache_memory_usage: 758M + + # 'min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + # caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + # from being emptied while Synapse is evicting due to memory. + #min_cache_ttl: 5m + # Controls how long the results of a /sync request are cached for after # a successful response is returned. A higher duration can help clients with # intermittent connections, at the cost of higher memory usage. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 2af1f284b14e..ca443631055f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1119,7 +1119,17 @@ Caching can be configured through the following sub-options: with intermittent connections, at the cost of higher memory usage. By default, this is zero, which means that sync responses are not cached at all. - +* `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and + `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory + usage and cache entry availability. You must be using [jemalloc](https://github.com/matrix-org/synapse#help-synapse-is-slow-and-eats-all-my-ramcpu) + to utilize this option, and all three of the options must be specified for this feature to work. + * `max_cache_memory_usage` sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. + They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + the flag below, or until the `min_cache_ttl` is hit. + * `target_memory_usage` sets a rough target for the desired memory usage of the caches. + * `min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + from being emptied while Synapse is evicting due to memory. Example configuration: ```yaml @@ -1127,8 +1137,11 @@ caches: global_factor: 1.0 per_cache_factors: get_users_who_share_room_with_user: 2.0 - expire_caches: false sync_response_cache_duration: 2m + cache_autotuning: + max_cache_memory_usage: 1024M + target_cache_memory_usage: 758M + min_cache_ttl: 5m ``` ### Reloading cache factors diff --git a/synapse/config/cache.py b/synapse/config/cache.py index 58b2fe55193c..d2f55534d7d1 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -176,6 +176,24 @@ def generate_config_section(self, **kwargs: Any) -> str: # #cache_entry_ttl: 30m + # This flag enables cache autotuning, and is further specified by the sub-options `max_cache_memory_usage`, + # `target_cache_memory_usage`, `min_cache_ttl`. These flags work in conjunction with each other to maintain + # a balance between cache memory usage and cache entry availability. You must be using jemalloc to utilize + # this option, and all three of the options must be specified for this feature to work. + #cache_autotuning: + # This flag sets a ceiling on much memory the cache can use before caches begin to be continuously evicted. + # They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + # the flag below, or until the `min_cache_ttl` is hit. + #max_cache_memory_usage: 1024M + + # This flag sets a rough target for the desired memory usage of the caches. + #target_cache_memory_usage: 758M + + # 'min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + # caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + # from being emptied while Synapse is evicting due to memory. + #min_cache_ttl: 5m + # Controls how long the results of a /sync request are cached for after # a successful response is returned. A higher duration can help clients with # intermittent connections, at the cost of higher memory usage. @@ -263,6 +281,21 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) self.expiry_time_msec = self.parse_duration(expiry_time) + self.cache_autotuning = cache_config.get("cache_autotuning") + if self.cache_autotuning: + max_memory_usage = self.cache_autotuning.get("max_cache_memory_usage") + self.cache_autotuning["max_cache_memory_usage"] = self.parse_size( + max_memory_usage + ) + + target_mem_size = self.cache_autotuning.get("target_cache_memory_usage") + self.cache_autotuning["target_cache_memory_usage"] = self.parse_size( + target_mem_size + ) + + min_cache_ttl = self.cache_autotuning.get("min_cache_ttl") + self.cache_autotuning["min_cache_ttl"] = self.parse_duration(min_cache_ttl) + self.sync_response_cache_duration = self.parse_duration( cache_config.get("sync_response_cache_duration", 0) ) diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py index 6bc329f04a9c..1fc8a0e888a1 100644 --- a/synapse/metrics/jemalloc.py +++ b/synapse/metrics/jemalloc.py @@ -18,6 +18,7 @@ import re from typing import Iterable, Optional, overload +import attr from prometheus_client import REGISTRY, Metric from typing_extensions import Literal @@ -27,52 +28,24 @@ logger = logging.getLogger(__name__) -def _setup_jemalloc_stats() -> None: - """Checks to see if jemalloc is loaded, and hooks up a collector to record - statistics exposed by jemalloc. - """ - - # Try to find the loaded jemalloc shared library, if any. We need to - # introspect into what is loaded, rather than loading whatever is on the - # path, as if we load a *different* jemalloc version things will seg fault. - - # We look in `/proc/self/maps`, which only exists on linux. - if not os.path.exists("/proc/self/maps"): - logger.debug("Not looking for jemalloc as no /proc/self/maps exist") - return - - # We're looking for a path at the end of the line that includes - # "libjemalloc". - regex = re.compile(r"/\S+/libjemalloc.*$") - - jemalloc_path = None - with open("/proc/self/maps") as f: - for line in f: - match = regex.search(line.strip()) - if match: - jemalloc_path = match.group() - - if not jemalloc_path: - # No loaded jemalloc was found. - logger.debug("jemalloc not found") - return - - logger.debug("Found jemalloc at %s", jemalloc_path) - - jemalloc = ctypes.CDLL(jemalloc_path) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class JemallocStats: + jemalloc: ctypes.CDLL @overload def _mallctl( - name: str, read: Literal[True] = True, write: Optional[int] = None + self, name: str, read: Literal[True] = True, write: Optional[int] = None ) -> int: ... @overload - def _mallctl(name: str, read: Literal[False], write: Optional[int] = None) -> None: + def _mallctl( + self, name: str, read: Literal[False], write: Optional[int] = None + ) -> None: ... def _mallctl( - name: str, read: bool = True, write: Optional[int] = None + self, name: str, read: bool = True, write: Optional[int] = None ) -> Optional[int]: """Wrapper around `mallctl` for reading and writing integers to jemalloc. @@ -120,7 +93,7 @@ def _mallctl( # Where oldp/oldlenp is a buffer where the old value will be written to # (if not null), and newp/newlen is the buffer with the new value to set # (if not null). Note that they're all references *except* newlen. - result = jemalloc.mallctl( + result = self.jemalloc.mallctl( name.encode("ascii"), input_var_ref, input_len_ref, @@ -136,21 +109,80 @@ def _mallctl( return input_var.value - def _jemalloc_refresh_stats() -> None: + def refresh_stats(self) -> None: """Request that jemalloc updates its internal statistics. This needs to be called before querying for stats, otherwise it will return stale values. """ try: - _mallctl("epoch", read=False, write=1) + self._mallctl("epoch", read=False, write=1) except Exception as e: logger.warning("Failed to reload jemalloc stats: %s", e) + def get_stat(self, name: str) -> int: + """Request the stat of the given name at the time of the last + `refresh_stats` call. This may throw if we fail to read + the stat. + """ + return self._mallctl(f"stats.{name}") + + +_JEMALLOC_STATS: Optional[JemallocStats] = None + + +def get_jemalloc_stats() -> Optional[JemallocStats]: + """Returns an interface to jemalloc, if it is being used. + + Note that this will always return None until `setup_jemalloc_stats` has been + called. + """ + return _JEMALLOC_STATS + + +def _setup_jemalloc_stats() -> None: + """Checks to see if jemalloc is loaded, and hooks up a collector to record + statistics exposed by jemalloc. + """ + + global _JEMALLOC_STATS + + # Try to find the loaded jemalloc shared library, if any. We need to + # introspect into what is loaded, rather than loading whatever is on the + # path, as if we load a *different* jemalloc version things will seg fault. + + # We look in `/proc/self/maps`, which only exists on linux. + if not os.path.exists("/proc/self/maps"): + logger.debug("Not looking for jemalloc as no /proc/self/maps exist") + return + + # We're looking for a path at the end of the line that includes + # "libjemalloc". + regex = re.compile(r"/\S+/libjemalloc.*$") + + jemalloc_path = None + with open("/proc/self/maps") as f: + for line in f: + match = regex.search(line.strip()) + if match: + jemalloc_path = match.group() + + if not jemalloc_path: + # No loaded jemalloc was found. + logger.debug("jemalloc not found") + return + + logger.debug("Found jemalloc at %s", jemalloc_path) + + jemalloc_dll = ctypes.CDLL(jemalloc_path) + + stats = JemallocStats(jemalloc_dll) + _JEMALLOC_STATS = stats + class JemallocCollector(Collector): """Metrics for internal jemalloc stats.""" def collect(self) -> Iterable[Metric]: - _jemalloc_refresh_stats() + stats.refresh_stats() g = GaugeMetricFamily( "jemalloc_stats_app_memory_bytes", @@ -184,7 +216,7 @@ def collect(self) -> Iterable[Metric]: "metadata", ): try: - value = _mallctl(f"stats.{t}") + value = stats.get_stat(t) except Exception as e: # There was an error fetching the value, skip. logger.warning("Failed to read jemalloc stats.%s: %s", t, e) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 45ff0de638a4..a3b60578e3c3 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import math import threading import weakref from enum import Enum @@ -40,6 +41,7 @@ from synapse.config import cache as cache_config from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.metrics.jemalloc import get_jemalloc_stats from synapse.util import Clock, caches from synapse.util.caches import CacheMetric, EvictionReason, register_cache from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry @@ -106,10 +108,16 @@ def update_last_access(self, clock: Clock) -> None: @wrap_as_background_process("LruCache._expire_old_entries") -async def _expire_old_entries(clock: Clock, expiry_seconds: int) -> None: +async def _expire_old_entries( + clock: Clock, expiry_seconds: int, autotune_config: Optional[dict] +) -> None: """Walks the global cache list to find cache entries that haven't been - accessed in the given number of seconds. + accessed in the given number of seconds, or if a given memory threshold has been breached. """ + if autotune_config: + max_cache_memory_usage = autotune_config["max_cache_memory_usage"] + target_cache_memory_usage = autotune_config["target_cache_memory_usage"] + min_cache_ttl = autotune_config["min_cache_ttl"] / 1000 now = int(clock.time()) node = GLOBAL_ROOT.prev_node @@ -119,11 +127,36 @@ async def _expire_old_entries(clock: Clock, expiry_seconds: int) -> None: logger.debug("Searching for stale caches") + evicting_due_to_memory = False + + # determine if we're evicting due to memory + jemalloc_interface = get_jemalloc_stats() + if jemalloc_interface and autotune_config: + try: + jemalloc_interface.refresh_stats() + mem_usage = jemalloc_interface.get_stat("allocated") + if mem_usage > max_cache_memory_usage: + logger.info("Begin memory-based cache eviction.") + evicting_due_to_memory = True + except Exception: + logger.warning( + "Unable to read allocated memory, skipping memory-based cache eviction." + ) + while node is not GLOBAL_ROOT: # Only the root node isn't a `_TimedListNode`. assert isinstance(node, _TimedListNode) - if node.last_access_ts_secs > now - expiry_seconds: + # if node has not aged past expiry_seconds and we are not evicting due to memory usage, there's + # nothing to do here + if ( + node.last_access_ts_secs > now - expiry_seconds + and not evicting_due_to_memory + ): + break + + # if entry is newer than min_cache_entry_ttl then do not evict and don't evict anything newer + if evicting_due_to_memory and now - node.last_access_ts_secs < min_cache_ttl: break cache_entry = node.get_cache_entry() @@ -136,10 +169,29 @@ async def _expire_old_entries(clock: Clock, expiry_seconds: int) -> None: assert cache_entry is not None cache_entry.drop_from_cache() + # Check mem allocation periodically if we are evicting a bunch of caches + if jemalloc_interface and evicting_due_to_memory and (i + 1) % 100 == 0: + try: + jemalloc_interface.refresh_stats() + mem_usage = jemalloc_interface.get_stat("allocated") + if mem_usage < target_cache_memory_usage: + evicting_due_to_memory = False + logger.info("Stop memory-based cache eviction.") + except Exception: + logger.warning( + "Unable to read allocated memory, this may affect memory-based cache eviction." + ) + # If we've failed to read the current memory usage then we + # should stop trying to evict based on memory usage + evicting_due_to_memory = False + # If we do lots of work at once we yield to allow other stuff to happen. if (i + 1) % 10000 == 0: logger.debug("Waiting during drop") - await clock.sleep(0) + if node.last_access_ts_secs > now - expiry_seconds: + await clock.sleep(0.5) + else: + await clock.sleep(0) logger.debug("Waking during drop") node = next_node @@ -156,21 +208,28 @@ async def _expire_old_entries(clock: Clock, expiry_seconds: int) -> None: def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: """Start a background job that expires all cache entries if they have not - been accessed for the given number of seconds. + been accessed for the given number of seconds, or if a given memory usage threshold has been + breached. """ - if not hs.config.caches.expiry_time_msec: + if not hs.config.caches.expiry_time_msec and not hs.config.caches.cache_autotuning: return - logger.info( - "Expiring LRU caches after %d seconds", hs.config.caches.expiry_time_msec / 1000 - ) + if hs.config.caches.expiry_time_msec: + expiry_time = hs.config.caches.expiry_time_msec / 1000 + logger.info("Expiring LRU caches after %d seconds", expiry_time) + else: + expiry_time = math.inf global USE_GLOBAL_LIST USE_GLOBAL_LIST = True clock = hs.get_clock() clock.looping_call( - _expire_old_entries, 30 * 1000, clock, hs.config.caches.expiry_time_msec / 1000 + _expire_old_entries, + 30 * 1000, + clock, + expiry_time, + hs.config.caches.cache_autotuning, ) diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index 321fc1776f8c..67173a4f5b3a 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -14,8 +14,9 @@ from typing import List -from unittest.mock import Mock +from unittest.mock import Mock, patch +from synapse.metrics.jemalloc import JemallocStats from synapse.util.caches.lrucache import LruCache, setup_expire_lru_cache_entries from synapse.util.caches.treecache import TreeCache @@ -316,3 +317,58 @@ def test_evict(self): self.assertEqual(cache.get("key1"), None) self.assertEqual(cache.get("key2"), 3) + + +class MemoryEvictionTestCase(unittest.HomeserverTestCase): + @override_config( + { + "caches": { + "cache_autotuning": { + "max_cache_memory_usage": "700M", + "target_cache_memory_usage": "500M", + "min_cache_ttl": "5m", + } + } + } + ) + @patch("synapse.util.caches.lrucache.get_jemalloc_stats") + def test_evict_memory(self, jemalloc_interface) -> None: + mock_jemalloc_class = Mock(spec=JemallocStats) + jemalloc_interface.return_value = mock_jemalloc_class + + # set the return value of get_stat() to be greater than max_cache_memory_usage + mock_jemalloc_class.get_stat.return_value = 924288000 + + setup_expire_lru_cache_entries(self.hs) + cache = LruCache(4, clock=self.hs.get_clock()) + + cache["key1"] = 1 + cache["key2"] = 2 + + # advance the reactor less than the min_cache_ttl + self.reactor.advance(60 * 2) + + # our items should still be in the cache + self.assertEqual(cache.get("key1"), 1) + self.assertEqual(cache.get("key2"), 2) + + # advance the reactor past the min_cache_ttl + self.reactor.advance(60 * 6) + + # the items should be cleared from cache + self.assertEqual(cache.get("key1"), None) + self.assertEqual(cache.get("key2"), None) + + # add more stuff to caches + cache["key1"] = 1 + cache["key2"] = 2 + + # set the return value of get_stat() to be lower than target_cache_memory_usage + mock_jemalloc_class.get_stat.return_value = 10000 + + # advance the reactor past the min_cache_ttl + self.reactor.advance(60 * 6) + + # the items should still be in the cache + self.assertEqual(cache.get("key1"), 1) + self.assertEqual(cache.get("key2"), 2) From 8689230a55cf1bb8e959e993fd4b28dcee0716da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 May 2022 12:06:56 +0100 Subject: [PATCH 187/263] Fix bug /sync returning 404 (#12729) * Fix bug /sync returning 404 Fixes #12571 --- changelog.d/12729.bugfix | 1 + synapse/storage/databases/main/stream.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 changelog.d/12729.bugfix diff --git a/changelog.d/12729.bugfix b/changelog.d/12729.bugfix new file mode 100644 index 000000000000..07aae23ac037 --- /dev/null +++ b/changelog.d/12729.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was rejected. diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 793e906630e8..3c3137fe64c5 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -743,14 +743,17 @@ async def get_room_event_before_stream_ordering( """ def _f(txn: LoggingTransaction) -> Optional[Tuple[int, int, str]]: - sql = ( - "SELECT stream_ordering, topological_ordering, event_id" - " FROM events" - " WHERE room_id = ? AND stream_ordering <= ?" - " AND NOT outlier" - " ORDER BY stream_ordering DESC" - " LIMIT 1" - ) + sql = """ + SELECT stream_ordering, topological_ordering, event_id + FROM events + LEFT JOIN rejections USING (event_id) + WHERE room_id = ? + AND stream_ordering <= ? + AND NOT outlier + AND rejections.reason IS NULL + ORDER BY stream_ordering DESC + LIMIT 1 + """ txn.execute(sql, (room_id, stream_ordering)) return cast(Optional[Tuple[int, int, str]], txn.fetchone()) From c3b232cb399c5098d27e6eb50e0d350774b0afd3 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 16 May 2022 12:52:29 +0100 Subject: [PATCH 188/263] 1.59.0rc2 --- CHANGES.md | 9 +++++++++ changelog.d/12729.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/12729.bugfix diff --git a/CHANGES.md b/CHANGES.md index c625e4d56119..ccb5d143261c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.59.0rc2 (2022-05-16) +============================== + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was rejected. ([\#12729](https://github.com/matrix-org/synapse/issues/12729)) + + Synapse 1.59.0rc1 (2022-05-10) ============================== diff --git a/changelog.d/12729.bugfix b/changelog.d/12729.bugfix deleted file mode 100644 index 07aae23ac037..000000000000 --- a/changelog.d/12729.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.58.0 where `/sync` would fail if the most recent event in a room was rejected. diff --git a/debian/changelog b/debian/changelog index fabc690bae93..7f9bedd7ac0e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.59.0~rc2) stable; urgency=medium + + * New Synapse release 1.59.0rc2. + + -- Synapse Packaging team Mon, 16 May 2022 12:52:15 +0100 + matrix-synapse-py3 (1.59.0~rc1) stable; urgency=medium * Adjust how the `exported-requirements.txt` file is generated as part of diff --git a/pyproject.toml b/pyproject.toml index e3d81ae5f1e2..1a2ddddb1010 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.59.0rc1" +version = "1.59.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 6f04ae70331c21bd5026662e7281eaa01933af27 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 16 May 2022 12:53:10 +0100 Subject: [PATCH 189/263] Move 1.59 warning to the top --- CHANGES.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ccb5d143261c..347aae8f67d6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,15 @@ Synapse 1.59.0rc2 (2022-05-16) ============================== +Synapse 1.59 makes several changes that server administrators should be aware of: + +- Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) +- The `synapse.app.appservice` and `synapse.app.user_dir` worker application types are now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452), [\#12654](https://github.com/matrix-org/synapse/issues/12654)) + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. + +Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) + Bugfixes -------- @@ -10,15 +19,6 @@ Bugfixes Synapse 1.59.0rc1 (2022-05-10) ============================== -This release makes several changes that server administrators should be aware of: - -- Device name lookup over federation is now disabled by default. ([\#12616](https://github.com/matrix-org/synapse/issues/12616)) -- The `synapse.app.appservice` and `synapse.app.user_dir` worker application types are now deprecated. ([\#12452](https://github.com/matrix-org/synapse/issues/12452), [\#12654](https://github.com/matrix-org/synapse/issues/12654)) - -See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1590) for more details. - -Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) - Features -------- From 86a515ccbf359ecd65a42a3f409b8f97c8f22284 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 16 May 2022 08:42:45 -0400 Subject: [PATCH 190/263] Consolidate logic for parsing relations. (#12693) Parse the `m.relates_to` event content field (which describes relations) in a single place, this is used during: * Event persistence. * Validation of the Client-Server API. * Fetching bundled aggregations. * Processing of push rules. Each of these separately implement the logic and each made slightly different assumptions about what was valid. Some had minor / potential bugs. --- changelog.d/12693.misc | 1 + synapse/events/__init__.py | 45 ++++++++++++++++++++++ synapse/handlers/message.py | 30 ++++++--------- synapse/handlers/relations.py | 20 +++++----- synapse/push/bulk_push_rule_evaluator.py | 6 +-- synapse/storage/databases/main/events.py | 49 ++++++++++-------------- tests/rest/client/test_sync.py | 8 +++- 7 files changed, 98 insertions(+), 61 deletions(-) create mode 100644 changelog.d/12693.misc diff --git a/changelog.d/12693.misc b/changelog.d/12693.misc new file mode 100644 index 000000000000..8bd1e1cb0cd5 --- /dev/null +++ b/changelog.d/12693.misc @@ -0,0 +1 @@ +Consolidate parsing of relation information from events. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index c238376caf62..39ad2793d98d 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -15,6 +15,7 @@ # limitations under the License. import abc +import collections.abc import os from typing import ( TYPE_CHECKING, @@ -32,9 +33,11 @@ overload, ) +import attr from typing_extensions import Literal from unpaddedbase64 import encode_base64 +from synapse.api.constants import RelationTypes from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions from synapse.types import JsonDict, RoomStreamToken from synapse.util.caches import intern_dict @@ -615,3 +618,45 @@ def make_event_from_dict( return event_type( event_dict, room_version, internal_metadata_dict or {}, rejected_reason ) + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class _EventRelation: + # The target event of the relation. + parent_id: str + # The relation type. + rel_type: str + # The aggregation key. Will be None if the rel_type is not m.annotation or is + # not a string. + aggregation_key: Optional[str] + + +def relation_from_event(event: EventBase) -> Optional[_EventRelation]: + """ + Attempt to parse relation information an event. + + Returns: + The event relation information, if it is valid. None, otherwise. + """ + relation = event.content.get("m.relates_to") + if not relation or not isinstance(relation, collections.abc.Mapping): + # No relation information. + return None + + # Relations must have a type and parent event ID. + rel_type = relation.get("rel_type") + if not isinstance(rel_type, str): + return None + + parent_id = relation.get("event_id") + if not isinstance(parent_id, str): + return None + + # Annotations have a key field. + aggregation_key = None + if rel_type == RelationTypes.ANNOTATION: + aggregation_key = relation.get("key") + if not isinstance(aggregation_key, str): + aggregation_key = None + + return _EventRelation(parent_id, rel_type, aggregation_key) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4a4b535bae6a..0951b9c71f75 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -44,7 +44,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase +from synapse.events import EventBase, relation_from_event from synapse.events.builder import EventBuilder from synapse.events.snapshot import EventContext from synapse.events.validator import EventValidator @@ -1060,20 +1060,11 @@ async def _validate_event_relation(self, event: EventBase) -> None: SynapseError if the event is invalid. """ - relation = event.content.get("m.relates_to") + relation = relation_from_event(event) if not relation: return - relation_type = relation.get("rel_type") - if not relation_type: - return - - # Ensure the parent is real. - relates_to = relation.get("event_id") - if not relates_to: - return - - parent_event = await self.store.get_event(relates_to, allow_none=True) + parent_event = await self.store.get_event(relation.parent_id, allow_none=True) if parent_event: # And in the same room. if parent_event.room_id != event.room_id: @@ -1082,28 +1073,31 @@ async def _validate_event_relation(self, event: EventBase) -> None: else: # There must be some reason that the client knows the event exists, # see if there are existing relations. If so, assume everything is fine. - if not await self.store.event_is_target_of_relation(relates_to): + if not await self.store.event_is_target_of_relation(relation.parent_id): # Otherwise, the client can't know about the parent event! raise SynapseError(400, "Can't send relation to unknown event") # If this event is an annotation then we check that that the sender # can't annotate the same way twice (e.g. stops users from liking an # event multiple times). - if relation_type == RelationTypes.ANNOTATION: - aggregation_key = relation["key"] + if relation.rel_type == RelationTypes.ANNOTATION: + aggregation_key = relation.aggregation_key + + if aggregation_key is None: + raise SynapseError(400, "Missing aggregation key") if len(aggregation_key) > 500: raise SynapseError(400, "Aggregation key is too long") already_exists = await self.store.has_user_annotated_event( - relates_to, event.type, aggregation_key, event.sender + relation.parent_id, event.type, aggregation_key, event.sender ) if already_exists: raise SynapseError(400, "Can't send same reaction twice") # Don't attempt to start a thread if the parent event is a relation. - elif relation_type == RelationTypes.THREAD: - if await self.store.event_includes_relation(relates_to): + elif relation.rel_type == RelationTypes.THREAD: + if await self.store.event_includes_relation(relation.parent_id): raise SynapseError( 400, "Cannot start threads from an event with a relation" ) diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index c2754ec918de..ab7e54857d56 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import collections.abc import logging from typing import ( TYPE_CHECKING, @@ -28,7 +27,7 @@ from synapse.api.constants import RelationTypes from synapse.api.errors import SynapseError -from synapse.events import EventBase +from synapse.events import EventBase, relation_from_event from synapse.storage.databases.main.relations import _RelatedEvent from synapse.types import JsonDict, Requester, StreamToken, UserID from synapse.visibility import filter_events_for_client @@ -373,20 +372,21 @@ async def get_bundled_aggregations( if event.is_state(): continue - relates_to = event.content.get("m.relates_to") - relation_type = None - if isinstance(relates_to, collections.abc.Mapping): - relation_type = relates_to.get("rel_type") + relates_to = relation_from_event(event) + if relates_to: # An event which is a replacement (ie edit) or annotation (ie, # reaction) may not have any other event related to it. - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): + if relates_to.rel_type in ( + RelationTypes.ANNOTATION, + RelationTypes.REPLACE, + ): continue + # Track the event's relation information for later. + relations_by_id[event.event_id] = relates_to.rel_type + # The event should get bundled aggregations. events_by_id[event.event_id] = event - # Track the event's relation information for later. - if isinstance(relation_type, str): - relations_by_id[event.event_id] = relation_type # event ID -> bundled aggregation in non-serialized form. results: Dict[str, BundledAggregations] = {} diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 0ffafc882b65..4ac2c546bf2a 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, RelationTypes from synapse.event_auth import get_user_power_level -from synapse.events import EventBase +from synapse.events import EventBase, relation_from_event from synapse.events.snapshot import EventContext from synapse.state import POWER_KEY from synapse.storage.databases.main.roommember import EventIdMembership @@ -78,8 +78,8 @@ def _should_count_as_unread(event: EventBase, context: EventContext) -> bool: return False # Exclude edits. - relates_to = event.content.get("m.relates_to", {}) - if relates_to.get("rel_type") == RelationTypes.REPLACE: + relates_to = relation_from_event(event) + if relates_to and relates_to.rel_type == RelationTypes.REPLACE: return False # Mark events that have a non-empty string body as unread. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index f544bcfff07f..42d484dc98d9 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -36,8 +36,8 @@ import synapse.metrics from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase # noqa: F401 -from synapse.events.snapshot import EventContext # noqa: F401 +from synapse.events import EventBase, relation_from_event +from synapse.events.snapshot import EventContext from synapse.storage._base import db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -1807,52 +1807,45 @@ def _handle_event_relations( txn: The current database transaction. event: The event which might have relations. """ - relation = event.content.get("m.relates_to") + relation = relation_from_event(event) if not relation: - # No relations + # No relation, nothing to do. return - # Relations must have a type and parent event ID. - rel_type = relation.get("rel_type") - if not isinstance(rel_type, str): - return - - parent_id = relation.get("event_id") - if not isinstance(parent_id, str): - return - - # Annotations have a key field. - aggregation_key = None - if rel_type == RelationTypes.ANNOTATION: - aggregation_key = relation.get("key") - self.db_pool.simple_insert_txn( txn, table="event_relations", values={ "event_id": event.event_id, - "relates_to_id": parent_id, - "relation_type": rel_type, - "aggregation_key": aggregation_key, + "relates_to_id": relation.parent_id, + "relation_type": relation.rel_type, + "aggregation_key": relation.aggregation_key, }, ) - txn.call_after(self.store.get_relations_for_event.invalidate, (parent_id,)) txn.call_after( - self.store.get_aggregation_groups_for_event.invalidate, (parent_id,) + self.store.get_relations_for_event.invalidate, (relation.parent_id,) + ) + txn.call_after( + self.store.get_aggregation_groups_for_event.invalidate, + (relation.parent_id,), ) - if rel_type == RelationTypes.REPLACE: - txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,)) + if relation.rel_type == RelationTypes.REPLACE: + txn.call_after( + self.store.get_applicable_edit.invalidate, (relation.parent_id,) + ) - if rel_type == RelationTypes.THREAD: - txn.call_after(self.store.get_thread_summary.invalidate, (parent_id,)) + if relation.rel_type == RelationTypes.THREAD: + txn.call_after( + self.store.get_thread_summary.invalidate, (relation.parent_id,) + ) # It should be safe to only invalidate the cache if the user has not # previously participated in the thread, but that's difficult (and # potentially error-prone) so it is always invalidated. txn.call_after( self.store.get_thread_participated.invalidate, - (parent_id, event.sender), + (relation.parent_id, event.sender), ) def _handle_insertion_event( diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 2722bf26e76c..74b6560cbcf5 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -656,12 +656,13 @@ def test_unread_counts(self) -> None: self._check_unread_count(3) # Check that custom events with a body increase the unread counter. - self.helper.send_event( + result = self.helper.send_event( self.room_id, "org.matrix.custom_type", {"body": "hello"}, tok=self.tok2, ) + event_id = result["event_id"] self._check_unread_count(4) # Check that edits don't increase the unread counter. @@ -671,7 +672,10 @@ def test_unread_counts(self) -> None: content={ "body": "hello", "msgtype": "m.text", - "m.relates_to": {"rel_type": RelationTypes.REPLACE}, + "m.relates_to": { + "rel_type": RelationTypes.REPLACE, + "event_id": event_id, + }, }, tok=self.tok2, ) From a5c26750b50563f2edda8b5d37c70b1d49e5f34c Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 16 May 2022 14:06:04 +0100 Subject: [PATCH 191/263] Fix room upgrades creating an empty room when auth fails (#12696) Signed-off-by: Sean Quah --- changelog.d/12696.bugfix | 1 + synapse/handlers/room.py | 125 ++++++++++++------ .../test_sharded_event_persister.py | 14 +- 3 files changed, 84 insertions(+), 56 deletions(-) create mode 100644 changelog.d/12696.bugfix diff --git a/changelog.d/12696.bugfix b/changelog.d/12696.bugfix new file mode 100644 index 000000000000..e410184a22af --- /dev/null +++ b/changelog.d/12696.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where an empty room would be created when a user with an insufficient power level tried to upgrade a room. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e71c78adad67..23baa50d0375 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -33,6 +33,7 @@ import attr from typing_extensions import TypedDict +import synapse.events.snapshot from synapse.api.constants import ( EventContentFields, EventTypes, @@ -77,7 +78,6 @@ create_requester, ) from synapse.util import stringutils -from synapse.util.async_helpers import Linearizer from synapse.util.caches.response_cache import ResponseCache from synapse.util.stringutils import parse_and_validate_server_name from synapse.visibility import filter_events_for_client @@ -155,9 +155,6 @@ def __init__(self, hs: "HomeServer"): self._replication = hs.get_replication_data_handler() - # linearizer to stop two upgrades happening at once - self._upgrade_linearizer = Linearizer("room_upgrade_linearizer") - # If a user tries to update the same room multiple times in quick # succession, only process the first attempt and return its result to # subsequent requests @@ -200,6 +197,39 @@ async def upgrade_room( 400, "An upgrade for this room is currently in progress" ) + # Check whether the room exists and 404 if it doesn't. + # We could go straight for the auth check, but that will raise a 403 instead. + old_room = await self.store.get_room(old_room_id) + if old_room is None: + raise NotFoundError("Unknown room id %s" % (old_room_id,)) + + new_room_id = self._generate_room_id() + + # Check whether the user has the power level to carry out the upgrade. + # `check_auth_rules_from_context` will check that they are in the room and have + # the required power level to send the tombstone event. + ( + tombstone_event, + tombstone_context, + ) = await self.event_creation_handler.create_event( + requester, + { + "type": EventTypes.Tombstone, + "state_key": "", + "room_id": old_room_id, + "sender": user_id, + "content": { + "body": "This room has been replaced", + "replacement_room": new_room_id, + }, + }, + ) + old_room_version = await self.store.get_room_version(old_room_id) + validate_event_for_room_version(old_room_version, tombstone_event) + await self._event_auth_handler.check_auth_rules_from_context( + old_room_version, tombstone_event, tombstone_context + ) + # Upgrade the room # # If this user has sent multiple upgrade requests for the same room @@ -210,19 +240,35 @@ async def upgrade_room( self._upgrade_room, requester, old_room_id, - new_version, # args for _upgrade_room + old_room, # args for _upgrade_room + new_room_id, + new_version, + tombstone_event, + tombstone_context, ) return ret async def _upgrade_room( - self, requester: Requester, old_room_id: str, new_version: RoomVersion + self, + requester: Requester, + old_room_id: str, + old_room: Dict[str, Any], + new_room_id: str, + new_version: RoomVersion, + tombstone_event: EventBase, + tombstone_context: synapse.events.snapshot.EventContext, ) -> str: """ Args: requester: the user requesting the upgrade old_room_id: the id of the room to be replaced - new_versions: the version to upgrade the room to + old_room: a dict containing room information for the room to be replaced, + as returned by `RoomWorkerStore.get_room`. + new_room_id: the id of the replacement room + new_version: the version to upgrade the room to + tombstone_event: the tombstone event to send to the old room + tombstone_context: the context for the tombstone event Raises: ShadowBanError if the requester is shadow-banned. @@ -230,40 +276,15 @@ async def _upgrade_room( user_id = requester.user.to_string() assert self.hs.is_mine_id(user_id), "User must be our own: %s" % (user_id,) - # start by allocating a new room id - r = await self.store.get_room(old_room_id) - if r is None: - raise NotFoundError("Unknown room id %s" % (old_room_id,)) - new_room_id = await self._generate_room_id( - creator_id=user_id, - is_public=r["is_public"], - room_version=new_version, - ) - logger.info("Creating new room %s to replace %s", new_room_id, old_room_id) - # we create and auth the tombstone event before properly creating the new - # room, to check our user has perms in the old room. - ( - tombstone_event, - tombstone_context, - ) = await self.event_creation_handler.create_event( - requester, - { - "type": EventTypes.Tombstone, - "state_key": "", - "room_id": old_room_id, - "sender": user_id, - "content": { - "body": "This room has been replaced", - "replacement_room": new_room_id, - }, - }, - ) - old_room_version = await self.store.get_room_version(old_room_id) - validate_event_for_room_version(old_room_version, tombstone_event) - await self._event_auth_handler.check_auth_rules_from_context( - old_room_version, tombstone_event, tombstone_context + # create the new room. may raise a `StoreError` in the exceedingly unlikely + # event of a room ID collision. + await self.store.store_room( + room_id=new_room_id, + room_creator_user_id=user_id, + is_public=old_room["is_public"], + room_version=new_version, ) await self.clone_existing_room( @@ -782,7 +803,7 @@ async def create_room( visibility = config.get("visibility", "private") is_public = visibility == "public" - room_id = await self._generate_room_id( + room_id = await self._generate_and_create_room_id( creator_id=user_id, is_public=is_public, room_version=room_version, @@ -1104,7 +1125,26 @@ async def send(etype: str, content: JsonDict, **kwargs: Any) -> int: return last_sent_stream_id - async def _generate_room_id( + def _generate_room_id(self) -> str: + """Generates a random room ID. + + Room IDs look like "!opaque_id:domain" and are case-sensitive as per the spec + at https://spec.matrix.org/v1.2/appendices/#room-ids-and-event-ids. + + Does not check for collisions with existing rooms or prevent future calls from + returning the same room ID. To ensure the uniqueness of a new room ID, use + `_generate_and_create_room_id` instead. + + Synapse's room IDs are 18 [a-zA-Z] characters long, which comes out to around + 102 bits. + + Returns: + A random room ID of the form "!opaque_id:domain". + """ + random_string = stringutils.random_string(18) + return RoomID(random_string, self.hs.hostname).to_string() + + async def _generate_and_create_room_id( self, creator_id: str, is_public: bool, @@ -1115,8 +1155,7 @@ async def _generate_room_id( attempts = 0 while attempts < 5: try: - random_string = stringutils.random_string(18) - gen_room_id = RoomID(random_string, self.hs.hostname).to_string() + gen_room_id = self._generate_room_id() await self.store.store_room( room_id=gen_room_id, room_creator_user_id=creator_id, diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index 5f142e84c359..a7ca68069e86 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -14,7 +14,6 @@ import logging from unittest.mock import patch -from synapse.api.room_versions import RoomVersion from synapse.rest import admin from synapse.rest.client import login, room, sync from synapse.storage.util.id_generators import MultiWriterIdGenerator @@ -64,21 +63,10 @@ def _create_room(self, room_id: str, user_id: str, tok: str): # We control the room ID generation by patching out the # `_generate_room_id` method - async def generate_room( - creator_id: str, is_public: bool, room_version: RoomVersion - ): - await self.store.store_room( - room_id=room_id, - room_creator_user_id=creator_id, - is_public=is_public, - room_version=room_version, - ) - return room_id - with patch( "synapse.handlers.room.RoomCreationHandler._generate_room_id" ) as mock: - mock.side_effect = generate_room + mock.side_effect = lambda: room_id self.helper.create_room_as(user_id, tok=tok) def test_basic(self): From 806003461226dc8db19bf9d631260b78c48d544e Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 16 May 2022 15:50:07 +0200 Subject: [PATCH 192/263] Fix typo in listener config (#12742) --- changelog.d/12742.doc | 1 + docs/sample_config.yaml | 2 +- docs/usage/configuration/config_documentation.md | 4 ++-- synapse/config/server.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12742.doc diff --git a/changelog.d/12742.doc b/changelog.d/12742.doc new file mode 100644 index 000000000000..0084e27a7d03 --- /dev/null +++ b/changelog.d/12742.doc @@ -0,0 +1 @@ +Fix typo in server listener documentation. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 05a3606da043..ee98d193cbb9 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -289,7 +289,7 @@ presence: # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # -# keys: the key discovery API (/_matrix/keys). +# keys: the key discovery API (/_matrix/key). # # media: the media API (/_matrix/media). # diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ca443631055f..f0bf13976d2c 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -467,13 +467,13 @@ Sub-options for each listener include: Valid resource names are: -* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies 'media' and 'static'. +* `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`. * `consent`: user consent forms (/_matrix/consent). See [here](../../consent_tracking.md) for more. * `federation`: the server-server API (/_matrix/federation). Also implies `media`, `keys`, `openid` -* `keys`: the key discovery API (/_matrix/keys). +* `keys`: the key discovery API (/_matrix/key). * `media`: the media API (/_matrix/media). diff --git a/synapse/config/server.py b/synapse/config/server.py index 005a3ee48ce4..f73d5e1f6666 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -996,7 +996,7 @@ def generate_config_section( # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # - # keys: the key discovery API (/_matrix/keys). + # keys: the key discovery API (/_matrix/key). # # media: the media API (/_matrix/media). # From 3ce15cc7be02da139e0b274418b2c137d737035a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 16 May 2022 17:06:23 +0200 Subject: [PATCH 193/263] Avoid unnecessary copies when filtering private read receipts. (#12711) A minor optimization to avoid unnecessary copying/building identical dictionaries when filtering private read receipts. Also clarifies comments and cleans-up some tests. --- changelog.d/12711.misc | 1 + synapse/handlers/initial_sync.py | 6 +- synapse/handlers/receipts.py | 94 +++++++++++++++++++++----------- tests/handlers/test_receipts.py | 64 +++++++++------------- 4 files changed, 92 insertions(+), 73 deletions(-) create mode 100644 changelog.d/12711.misc diff --git a/changelog.d/12711.misc b/changelog.d/12711.misc new file mode 100644 index 000000000000..0831ce045268 --- /dev/null +++ b/changelog.d/12711.misc @@ -0,0 +1 @@ +Optimize private read receipt filtering. diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 7b94770f9722..de09aed3a356 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -143,7 +143,7 @@ async def _snapshot_all_rooms( to_key=int(now_token.receipt_key), ) if self.hs.config.experimental.msc2285_enabled: - receipt = ReceiptEventSource.filter_out_private(receipt, user_id) + receipt = ReceiptEventSource.filter_out_private_receipts(receipt, user_id) tags_by_room = await self.store.get_tags_for_user(user_id) @@ -449,7 +449,9 @@ async def get_receipts() -> List[JsonDict]: if not receipts: return [] if self.hs.config.experimental.msc2285_enabled: - receipts = ReceiptEventSource.filter_out_private(receipts, user_id) + receipts = ReceiptEventSource.filter_out_private_receipts( + receipts, user_id + ) return receipts presence, receipts, (messages, token) = await make_deferred_yieldable( diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 43d615357b3c..550d58b0e1c9 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -165,43 +165,69 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config @staticmethod - def filter_out_private(events: List[JsonDict], user_id: str) -> List[JsonDict]: + def filter_out_private_receipts( + rooms: List[JsonDict], user_id: str + ) -> List[JsonDict]: """ - This method takes in what is returned by - get_linearized_receipts_for_rooms() and goes through read receipts - filtering out m.read.private receipts if they were not sent by the - current user. - """ - - visible_events = [] + Filters a list of serialized receipts (as returned by /sync and /initialSync) + and removes private read receipts of other users. - # filter out private receipts the user shouldn't see - for event in events: - content = event.get("content", {}) - new_event = event.copy() - new_event["content"] = {} + This operates on the return value of get_linearized_receipts_for_rooms(), + which is wrapped in a cache. Care must be taken to ensure that the input + values are not modified. - for event_id, event_content in content.items(): - receipt_event = {} - for receipt_type, receipt_content in event_content.items(): - if receipt_type == ReceiptTypes.READ_PRIVATE: - user_rr = receipt_content.get(user_id, None) - if user_rr: - receipt_event[ReceiptTypes.READ_PRIVATE] = { - user_id: user_rr.copy() - } - else: - receipt_event[receipt_type] = receipt_content.copy() - - # Only include the receipt event if it is non-empty. - if receipt_event: - new_event["content"][event_id] = receipt_event + Args: + rooms: A list of mappings, each mapping has a `content` field, which + is a map of event ID -> receipt type -> user ID -> receipt information. - # Append new_event to visible_events unless empty - if len(new_event["content"].keys()) > 0: - visible_events.append(new_event) + Returns: + The same as rooms, but filtered. + """ - return visible_events + result = [] + + # Iterate through each room's receipt content. + for room in rooms: + # The receipt content with other user's private read receipts removed. + content = {} + + # Iterate over each event ID / receipts for that event. + for event_id, orig_event_content in room.get("content", {}).items(): + event_content = orig_event_content + # If there are private read receipts, additional logic is necessary. + if ReceiptTypes.READ_PRIVATE in event_content: + # Make a copy without private read receipts to avoid leaking + # other user's private read receipts.. + event_content = { + receipt_type: receipt_value + for receipt_type, receipt_value in event_content.items() + if receipt_type != ReceiptTypes.READ_PRIVATE + } + + # Copy the current user's private read receipt from the + # original content, if it exists. + user_private_read_receipt = orig_event_content[ + ReceiptTypes.READ_PRIVATE + ].get(user_id, None) + if user_private_read_receipt: + event_content[ReceiptTypes.READ_PRIVATE] = { + user_id: user_private_read_receipt + } + + # Include the event if there is at least one non-private read + # receipt or the current user has a private read receipt. + if event_content: + content[event_id] = event_content + + # Include the event if there is at least one non-private read receipt + # or the current user has a private read receipt. + if content: + # Build a new event to avoid mutating the cache. + new_room = {k: v for k, v in room.items() if k != "content"} + new_room["content"] = content + result.append(new_room) + + return result async def get_new_events( self, @@ -223,7 +249,9 @@ async def get_new_events( ) if self.config.experimental.msc2285_enabled: - events = ReceiptEventSource.filter_out_private(events, user.to_string()) + events = ReceiptEventSource.filter_out_private_receipts( + events, user.to_string() + ) return events, to_key diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index 0482a1ea34fb..78807cdcfcdc 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from copy import deepcopy from typing import List from synapse.api.constants import ReceiptTypes @@ -125,42 +125,6 @@ def test_filters_out_event_with_only_private_receipts_and_ignores_the_rest(self) ], ) - def test_handles_missing_content_of_m_read(self): - self._test_filters_private( - [ - { - "content": { - "$14356419ggffg114394fHBLK:matrix.org": {ReceiptTypes.READ: {}}, - "$1435641916114394fHBLK:matrix.org": { - ReceiptTypes.READ: { - "@user:jki.re": { - "ts": 1436451550453, - } - } - }, - }, - "room_id": "!jEsUZKDJdhlrceRyVU:example.org", - "type": "m.receipt", - } - ], - [ - { - "content": { - "$14356419ggffg114394fHBLK:matrix.org": {ReceiptTypes.READ: {}}, - "$1435641916114394fHBLK:matrix.org": { - ReceiptTypes.READ: { - "@user:jki.re": { - "ts": 1436451550453, - } - } - }, - }, - "room_id": "!jEsUZKDJdhlrceRyVU:example.org", - "type": "m.receipt", - } - ], - ) - def test_handles_empty_event(self): self._test_filters_private( [ @@ -332,9 +296,33 @@ def test_leaves_our_private_and_their_public(self): ], ) + def test_we_do_not_mutate(self): + """Ensure the input values are not modified.""" + events = [ + { + "content": { + "$1435641916114394fHBLK:matrix.org": { + ReceiptTypes.READ_PRIVATE: { + "@rikj:jki.re": { + "ts": 1436451550453, + } + } + } + }, + "room_id": "!jEsUZKDJdhlrceRyVU:example.org", + "type": "m.receipt", + } + ] + original_events = deepcopy(events) + self._test_filters_private(events, []) + # Since the events are fed in from a cache they should not be modified. + self.assertEqual(events, original_events) + def _test_filters_private( self, events: List[JsonDict], expected_output: List[JsonDict] ): """Tests that the _filter_out_private returns the expected output""" - filtered_events = self.event_source.filter_out_private(events, "@me:server.org") + filtered_events = self.event_source.filter_out_private_receipts( + events, "@me:server.org" + ) self.assertEqual(filtered_events, expected_output) From 4ea546067dcdd5a3e67adc56a02645a2724636c8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 May 2022 16:30:35 +0100 Subject: [PATCH 194/263] Fix query performance for /sync (#12745) --- changelog.d/12745.bugfix | 1 + synapse/storage/databases/main/stream.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12745.bugfix diff --git a/changelog.d/12745.bugfix b/changelog.d/12745.bugfix new file mode 100644 index 000000000000..d1dc8d03aaec --- /dev/null +++ b/changelog.d/12745.bugfix @@ -0,0 +1 @@ +Fix DB performance regression introduced in v1.59.0rc2. diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 3c3137fe64c5..0373af86c82b 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -750,7 +750,7 @@ def _f(txn: LoggingTransaction) -> Optional[Tuple[int, int, str]]: WHERE room_id = ? AND stream_ordering <= ? AND NOT outlier - AND rejections.reason IS NULL + AND rejections.event_id IS NULL ORDER BY stream_ordering DESC LIMIT 1 """ From 83be72d76ca171ceb0fc381aa4548c1d9fea0dc7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 16 May 2022 16:35:31 +0100 Subject: [PATCH 195/263] Add `StreamKeyType` class and replace string literals with constants (#12567) --- changelog.d/12567.misc | 1 + synapse/handlers/account_data.py | 10 ++--- synapse/handlers/appservice.py | 39 ++++++++++--------- synapse/handlers/device.py | 7 +++- synapse/handlers/devicemessage.py | 6 +-- synapse/handlers/initial_sync.py | 13 ++++--- synapse/handlers/pagination.py | 6 +-- synapse/handlers/presence.py | 6 +-- synapse/handlers/receipts.py | 12 +++++- synapse/handlers/room.py | 9 +++-- synapse/handlers/search.py | 10 ++--- synapse/handlers/sync.py | 23 ++++++----- synapse/handlers/typing.py | 4 +- synapse/notifier.py | 5 ++- synapse/replication/tcp/client.py | 18 +++++---- .../server_notices/server_notices_manager.py | 4 +- .../storage/databases/main/e2e_room_keys.py | 4 +- synapse/storage/databases/main/relations.py | 6 ++- synapse/types.py | 22 +++++++++-- 19 files changed, 125 insertions(+), 80 deletions(-) create mode 100644 changelog.d/12567.misc diff --git a/changelog.d/12567.misc b/changelog.d/12567.misc new file mode 100644 index 000000000000..35f08569bada --- /dev/null +++ b/changelog.d/12567.misc @@ -0,0 +1 @@ +Replace string literal instances of stream key types with typed constants. \ No newline at end of file diff --git a/synapse/handlers/account_data.py b/synapse/handlers/account_data.py index 4af9fbc5d10a..0478448b47ea 100644 --- a/synapse/handlers/account_data.py +++ b/synapse/handlers/account_data.py @@ -23,7 +23,7 @@ ReplicationUserAccountDataRestServlet, ) from synapse.streams import EventSource -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, StreamKeyType, UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -105,7 +105,7 @@ async def add_account_data_to_room( ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) await self._notify_modules(user_id, room_id, account_data_type, content) @@ -141,7 +141,7 @@ async def add_account_data_for_user( ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) await self._notify_modules(user_id, None, account_data_type, content) @@ -176,7 +176,7 @@ async def add_tag_to_room( ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) return max_stream_id else: @@ -201,7 +201,7 @@ async def remove_tag_from_room(self, user_id: str, room_id: str, tag: str) -> in ) self._notifier.on_new_event( - "account_data_key", max_stream_id, users=[user_id] + StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id] ) return max_stream_id else: diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 85bd5e47682b..1da7bcc85b5c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -38,6 +38,7 @@ JsonDict, RoomAlias, RoomStreamToken, + StreamKeyType, UserID, ) from synapse.util.async_helpers import Linearizer @@ -213,8 +214,8 @@ def notify_interested_services_ephemeral( Args: stream_key: The stream the event came from. - `stream_key` can be "typing_key", "receipt_key", "presence_key", - "to_device_key" or "device_list_key". Any other value for `stream_key` + `stream_key` can be StreamKeyType.TYPING, StreamKeyType.RECEIPT, StreamKeyType.PRESENCE, + StreamKeyType.TO_DEVICE or StreamKeyType.DEVICE_LIST. Any other value for `stream_key` will cause this function to return early. Ephemeral events will only be pushed to appservices that have opted into @@ -235,11 +236,11 @@ def notify_interested_services_ephemeral( # Only the following streams are currently supported. # FIXME: We should use constants for these values. if stream_key not in ( - "typing_key", - "receipt_key", - "presence_key", - "to_device_key", - "device_list_key", + StreamKeyType.TYPING, + StreamKeyType.RECEIPT, + StreamKeyType.PRESENCE, + StreamKeyType.TO_DEVICE, + StreamKeyType.DEVICE_LIST, ): return @@ -258,14 +259,14 @@ def notify_interested_services_ephemeral( # Ignore to-device messages if the feature flag is not enabled if ( - stream_key == "to_device_key" + stream_key == StreamKeyType.TO_DEVICE and not self._msc2409_to_device_messages_enabled ): return # Ignore device lists if the feature flag is not enabled if ( - stream_key == "device_list_key" + stream_key == StreamKeyType.DEVICE_LIST and not self._msc3202_transaction_extensions_enabled ): return @@ -283,15 +284,15 @@ def notify_interested_services_ephemeral( if ( stream_key in ( - "typing_key", - "receipt_key", - "presence_key", - "to_device_key", + StreamKeyType.TYPING, + StreamKeyType.RECEIPT, + StreamKeyType.PRESENCE, + StreamKeyType.TO_DEVICE, ) and service.supports_ephemeral ) or ( - stream_key == "device_list_key" + stream_key == StreamKeyType.DEVICE_LIST and service.msc3202_transaction_extensions ) ] @@ -317,7 +318,7 @@ async def _notify_interested_services_ephemeral( logger.debug("Checking interested services for %s", stream_key) with Measure(self.clock, "notify_interested_services_ephemeral"): for service in services: - if stream_key == "typing_key": + if stream_key == StreamKeyType.TYPING: # Note that we don't persist the token (via set_appservice_stream_type_pos) # for typing_key due to performance reasons and due to their highly # ephemeral nature. @@ -333,7 +334,7 @@ async def _notify_interested_services_ephemeral( async with self._ephemeral_events_linearizer.queue( (service.id, stream_key) ): - if stream_key == "receipt_key": + if stream_key == StreamKeyType.RECEIPT: events = await self._handle_receipts(service, new_token) self.scheduler.enqueue_for_appservice(service, ephemeral=events) @@ -342,7 +343,7 @@ async def _notify_interested_services_ephemeral( service, "read_receipt", new_token ) - elif stream_key == "presence_key": + elif stream_key == StreamKeyType.PRESENCE: events = await self._handle_presence(service, users, new_token) self.scheduler.enqueue_for_appservice(service, ephemeral=events) @@ -351,7 +352,7 @@ async def _notify_interested_services_ephemeral( service, "presence", new_token ) - elif stream_key == "to_device_key": + elif stream_key == StreamKeyType.TO_DEVICE: # Retrieve a list of to-device message events, as well as the # maximum stream token of the messages we were able to retrieve. to_device_messages = await self._get_to_device_messages( @@ -366,7 +367,7 @@ async def _notify_interested_services_ephemeral( service, "to_device", new_token ) - elif stream_key == "device_list_key": + elif stream_key == StreamKeyType.DEVICE_LIST: device_list_summary = await self._get_device_list_summary( service, new_token ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index a91b1ee4d5f4..1d6d1f8a9248 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -43,6 +43,7 @@ ) from synapse.types import ( JsonDict, + StreamKeyType, StreamToken, UserID, get_domain_from_id, @@ -502,7 +503,7 @@ async def notify_device_update( # specify the user ID too since the user should always get their own device list # updates, even if they aren't in any rooms. self.notifier.on_new_event( - "device_list_key", position, users={user_id}, rooms=room_ids + StreamKeyType.DEVICE_LIST, position, users={user_id}, rooms=room_ids ) # We may need to do some processing asynchronously for local user IDs. @@ -523,7 +524,9 @@ async def notify_user_signature_update( from_user_id, user_ids ) - self.notifier.on_new_event("device_list_key", position, users=[from_user_id]) + self.notifier.on_new_event( + StreamKeyType.DEVICE_LIST, position, users=[from_user_id] + ) async def user_left_room(self, user: UserID, room_id: str) -> None: user_id = user.to_string() diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 4cb725d027c7..53668cce3bb4 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -26,7 +26,7 @@ set_tag, ) from synapse.replication.http.devices import ReplicationUserDevicesResyncRestServlet -from synapse.types import JsonDict, Requester, UserID, get_domain_from_id +from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util import json_encoder from synapse.util.stringutils import random_string @@ -151,7 +151,7 @@ async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None: # Notify listeners that there are new to-device messages to process, # handing them the latest stream id. self.notifier.on_new_event( - "to_device_key", last_stream_id, users=local_messages.keys() + StreamKeyType.TO_DEVICE, last_stream_id, users=local_messages.keys() ) async def _check_for_unknown_devices( @@ -285,7 +285,7 @@ async def send_device_message( # Notify listeners that there are new to-device messages to process, # handing them the latest stream id. self.notifier.on_new_event( - "to_device_key", last_stream_id, users=local_messages.keys() + StreamKeyType.TO_DEVICE, last_stream_id, users=local_messages.keys() ) if self.federation_sender: diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index de09aed3a356..d79248ad905b 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -30,6 +30,7 @@ Requester, RoomStreamToken, StateMap, + StreamKeyType, StreamToken, UserID, ) @@ -220,8 +221,10 @@ async def handle_room(event: RoomsForUser) -> None: self.storage, user_id, messages ) - start_token = now_token.copy_and_replace("room_key", token) - end_token = now_token.copy_and_replace("room_key", room_end_token) + start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token) + end_token = now_token.copy_and_replace( + StreamKeyType.ROOM, room_end_token + ) time_now = self.clock.time_msec() d["messages"] = { @@ -369,8 +372,8 @@ async def _room_initial_sync_parted( self.storage, user_id, messages, is_peeking=is_peeking ) - start_token = StreamToken.START.copy_and_replace("room_key", token) - end_token = StreamToken.START.copy_and_replace("room_key", stream_token) + start_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, token) + end_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, stream_token) time_now = self.clock.time_msec() @@ -474,7 +477,7 @@ async def get_receipts() -> List[JsonDict]: self.storage, user_id, messages, is_peeking=is_peeking ) - start_token = now_token.copy_and_replace("room_key", token) + start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token) end_token = now_token time_now = self.clock.time_msec() diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 2e30180094d2..6ae88add9526 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -27,7 +27,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.state import StateFilter from synapse.streams.config import PaginationConfig -from synapse.types import JsonDict, Requester +from synapse.types import JsonDict, Requester, StreamKeyType from synapse.util.async_helpers import ReadWriteLock from synapse.util.stringutils import random_string from synapse.visibility import filter_events_for_client @@ -491,7 +491,7 @@ async def get_messages( if leave_token.topological < curr_topo: from_token = from_token.copy_and_replace( - "room_key", leave_token + StreamKeyType.ROOM, leave_token ) await self.hs.get_federation_handler().maybe_backfill( @@ -513,7 +513,7 @@ async def get_messages( event_filter=event_filter, ) - next_token = from_token.copy_and_replace("room_key", next_key) + next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) if events: if event_filter: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 268481ec1963..dd84e6c88b6c 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -66,7 +66,7 @@ from synapse.replication.tcp.streams import PresenceFederationStream, PresenceStream from synapse.storage.databases.main import DataStore from synapse.streams import EventSource -from synapse.types import JsonDict, UserID, get_domain_from_id +from synapse.types import JsonDict, StreamKeyType, UserID, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.caches.descriptors import _CacheContext, cached from synapse.util.metrics import Measure @@ -522,7 +522,7 @@ async def notify_from_replication( room_ids_to_states, users_to_states = parties self.notifier.on_new_event( - "presence_key", + StreamKeyType.PRESENCE, stream_id, rooms=room_ids_to_states.keys(), users=users_to_states.keys(), @@ -1145,7 +1145,7 @@ async def _persist_and_notify(self, states: List[UserPresenceState]) -> None: room_ids_to_states, users_to_states = parties self.notifier.on_new_event( - "presence_key", + StreamKeyType.PRESENCE, stream_id, rooms=room_ids_to_states.keys(), users=[UserID.from_string(u) for u in users_to_states], diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 550d58b0e1c9..e6a35f1d093c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -17,7 +17,13 @@ from synapse.api.constants import ReceiptTypes from synapse.appservice import ApplicationService from synapse.streams import EventSource -from synapse.types import JsonDict, ReadReceipt, UserID, get_domain_from_id +from synapse.types import ( + JsonDict, + ReadReceipt, + StreamKeyType, + UserID, + get_domain_from_id, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -129,7 +135,9 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool: affected_room_ids = list({r.room_id for r in receipts}) - self.notifier.on_new_event("receipt_key", max_batch_id, rooms=affected_room_ids) + self.notifier.on_new_event( + StreamKeyType.RECEIPT, max_batch_id, rooms=affected_room_ids + ) # Note that the min here shouldn't be relied upon to be accurate. await self.hs.get_pusherpool().on_new_receipts( min_batch_id, max_batch_id, affected_room_ids diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 23baa50d0375..a2973109adc4 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -73,6 +73,7 @@ RoomID, RoomStreamToken, StateMap, + StreamKeyType, StreamToken, UserID, create_requester, @@ -1292,10 +1293,10 @@ async def filter_evts(events: List[EventBase]) -> List[EventBase]: events_after=events_after, state=await filter_evts(state_events), aggregations=aggregations, - start=await token.copy_and_replace("room_key", results.start).to_string( - self.store - ), - end=await token.copy_and_replace("room_key", results.end).to_string( + start=await token.copy_and_replace( + StreamKeyType.ROOM, results.start + ).to_string(self.store), + end=await token.copy_and_replace(StreamKeyType.ROOM, results.end).to_string( self.store ), ) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 5619f8f50e03..cd1c47dae8b1 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -24,7 +24,7 @@ from synapse.api.filtering import Filter from synapse.events import EventBase from synapse.storage.state import StateFilter -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, StreamKeyType, UserID from synapse.visibility import filter_events_for_client if TYPE_CHECKING: @@ -655,11 +655,11 @@ async def _calculate_event_contexts( "events_before": events_before, "events_after": events_after, "start": await now_token.copy_and_replace( - "room_key", res.start + StreamKeyType.ROOM, res.start + ).to_string(self.store), + "end": await now_token.copy_and_replace( + StreamKeyType.ROOM, res.end ).to_string(self.store), - "end": await now_token.copy_and_replace("room_key", res.end).to_string( - self.store - ), } if include_profile: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 2c555a66d066..4be08fe7cbc6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,6 +37,7 @@ Requester, RoomStreamToken, StateMap, + StreamKeyType, StreamToken, UserID, ) @@ -449,7 +450,7 @@ async def ephemeral_by_room( room_ids=room_ids, is_guest=sync_config.is_guest, ) - now_token = now_token.copy_and_replace("typing_key", typing_key) + now_token = now_token.copy_and_replace(StreamKeyType.TYPING, typing_key) ephemeral_by_room: JsonDict = {} @@ -471,7 +472,7 @@ async def ephemeral_by_room( room_ids=room_ids, is_guest=sync_config.is_guest, ) - now_token = now_token.copy_and_replace("receipt_key", receipt_key) + now_token = now_token.copy_and_replace(StreamKeyType.RECEIPT, receipt_key) for event in receipts: room_id = event["room_id"] @@ -537,7 +538,9 @@ async def _load_filtered_recents( prev_batch_token = now_token if recents: room_key = recents[0].internal_metadata.before - prev_batch_token = now_token.copy_and_replace("room_key", room_key) + prev_batch_token = now_token.copy_and_replace( + StreamKeyType.ROOM, room_key + ) return TimelineBatch( events=recents, prev_batch=prev_batch_token, limited=False @@ -611,7 +614,7 @@ async def _load_filtered_recents( recents = recents[-timeline_limit:] room_key = recents[0].internal_metadata.before - prev_batch_token = now_token.copy_and_replace("room_key", room_key) + prev_batch_token = now_token.copy_and_replace(StreamKeyType.ROOM, room_key) # Don't bother to bundle aggregations if the timeline is unlimited, # as clients will have all the necessary information. @@ -1398,7 +1401,7 @@ async def _generate_sync_entry_for_to_device( now_token.to_device_key, ) sync_result_builder.now_token = now_token.copy_and_replace( - "to_device_key", stream_id + StreamKeyType.TO_DEVICE, stream_id ) sync_result_builder.to_device = messages else: @@ -1503,7 +1506,7 @@ async def _generate_sync_entry_for_presence( ) assert presence_key sync_result_builder.now_token = now_token.copy_and_replace( - "presence_key", presence_key + StreamKeyType.PRESENCE, presence_key ) extra_users_ids = set(newly_joined_or_invited_users) @@ -1826,7 +1829,7 @@ async def _get_rooms_changed( # stream token as it'll only be used in the context of this # room. (c.f. the docstring of `to_room_stream_token`). leave_token = since_token.copy_and_replace( - "room_key", leave_position.to_room_stream_token() + StreamKeyType.ROOM, leave_position.to_room_stream_token() ) # If this is an out of band message, like a remote invite @@ -1875,7 +1878,9 @@ async def _get_rooms_changed( if room_entry: events, start_key = room_entry - prev_batch_token = now_token.copy_and_replace("room_key", start_key) + prev_batch_token = now_token.copy_and_replace( + StreamKeyType.ROOM, start_key + ) entry = RoomSyncResultBuilder( room_id=room_id, @@ -1972,7 +1977,7 @@ async def _get_all_rooms( continue leave_token = now_token.copy_and_replace( - "room_key", RoomStreamToken(None, event.stream_ordering) + StreamKeyType.ROOM, RoomStreamToken(None, event.stream_ordering) ) room_entries.append( RoomSyncResultBuilder( diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 6854428b7ca5..bb00750bfd47 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -25,7 +25,7 @@ ) from synapse.replication.tcp.streams import TypingStream from synapse.streams import EventSource -from synapse.types import JsonDict, Requester, UserID, get_domain_from_id +from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -382,7 +382,7 @@ def _push_update_local(self, member: RoomMember, typing: bool) -> None: ) self.notifier.on_new_event( - "typing_key", self._latest_room_serial, rooms=[member.room_id] + StreamKeyType.TYPING, self._latest_room_serial, rooms=[member.room_id] ) async def get_all_typing_updates( diff --git a/synapse/notifier.py b/synapse/notifier.py index 01a50b9d6226..ba23257f5498 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -46,6 +46,7 @@ JsonDict, PersistedEventPosition, RoomStreamToken, + StreamKeyType, StreamToken, UserID, ) @@ -370,7 +371,7 @@ def _notify_pending_new_room_events( if users or rooms: self.on_new_event( - "room_key", + StreamKeyType.ROOM, max_room_stream_token, users=users, rooms=rooms, @@ -440,7 +441,7 @@ def on_new_event( for room in rooms: user_streams |= self.room_to_user_streams.get(room, set()) - if stream_key == "to_device_key": + if stream_key == StreamKeyType.TO_DEVICE: issue9533_logger.debug( "to-device messages stream id %s, awaking streams for %s", new_token, diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 350762f49447..a52e25c1af3f 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -43,7 +43,7 @@ EventsStreamEventRow, EventsStreamRow, ) -from synapse.types import PersistedEventPosition, ReadReceipt, UserID +from synapse.types import PersistedEventPosition, ReadReceipt, StreamKeyType, UserID from synapse.util.async_helpers import Linearizer, timeout_deferred from synapse.util.metrics import Measure @@ -153,19 +153,19 @@ async def on_rdata( if stream_name == TypingStream.NAME: self._typing_handler.process_replication_rows(token, rows) self.notifier.on_new_event( - "typing_key", token, rooms=[row.room_id for row in rows] + StreamKeyType.TYPING, token, rooms=[row.room_id for row in rows] ) elif stream_name == PushRulesStream.NAME: self.notifier.on_new_event( - "push_rules_key", token, users=[row.user_id for row in rows] + StreamKeyType.PUSH_RULES, token, users=[row.user_id for row in rows] ) elif stream_name in (AccountDataStream.NAME, TagAccountDataStream.NAME): self.notifier.on_new_event( - "account_data_key", token, users=[row.user_id for row in rows] + StreamKeyType.ACCOUNT_DATA, token, users=[row.user_id for row in rows] ) elif stream_name == ReceiptsStream.NAME: self.notifier.on_new_event( - "receipt_key", token, rooms=[row.room_id for row in rows] + StreamKeyType.RECEIPT, token, rooms=[row.room_id for row in rows] ) await self._pusher_pool.on_new_receipts( token, token, {row.room_id for row in rows} @@ -173,14 +173,18 @@ async def on_rdata( elif stream_name == ToDeviceStream.NAME: entities = [row.entity for row in rows if row.entity.startswith("@")] if entities: - self.notifier.on_new_event("to_device_key", token, users=entities) + self.notifier.on_new_event( + StreamKeyType.TO_DEVICE, token, users=entities + ) elif stream_name == DeviceListsStream.NAME: all_room_ids: Set[str] = set() for row in rows: if row.entity.startswith("@"): room_ids = await self.store.get_rooms_for_user(row.entity) all_room_ids.update(room_ids) - self.notifier.on_new_event("device_list_key", token, rooms=all_room_ids) + self.notifier.on_new_event( + StreamKeyType.DEVICE_LIST, token, rooms=all_room_ids + ) elif stream_name == GroupServerStream.NAME: self.notifier.on_new_event( "groups_key", token, users=[row.user_id for row in rows] diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index c2c37e1015ce..8ecab86ec7d3 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -16,7 +16,7 @@ from synapse.api.constants import EventTypes, Membership, RoomCreationPreset from synapse.events import EventBase -from synapse.types import Requester, UserID, create_requester +from synapse.types import Requester, StreamKeyType, UserID, create_requester from synapse.util.caches.descriptors import cached if TYPE_CHECKING: @@ -189,7 +189,7 @@ async def get_or_create_notice_room_for_user(self, user_id: str) -> str: max_id = await self._account_data_handler.add_tag_to_room( user_id, room_id, SERVER_NOTICE_ROOM_TAG, {} ) - self._notifier.on_new_event("account_data_key", max_id, users=[user_id]) + self._notifier.on_new_event(StreamKeyType.ACCOUNT_DATA, max_id, users=[user_id]) logger.info("Created server notices room %s for %s", room_id, user_id) return room_id diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index b789a588a54b..af59be6b4854 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -21,7 +21,7 @@ from synapse.logging.opentracing import log_kv, trace from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import LoggingTransaction -from synapse.types import JsonDict, JsonSerializable +from synapse.types import JsonDict, JsonSerializable, StreamKeyType from synapse.util import json_encoder @@ -126,7 +126,7 @@ async def add_e2e_room_keys( "message": "Set room key", "room_id": room_id, "session_id": session_id, - "room_key": room_key, + StreamKeyType.ROOM: room_key, } ) diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 484976ca6b0b..fe8fded88b87 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -34,7 +34,7 @@ from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause from synapse.storage.databases.main.stream import generate_pagination_where_clause from synapse.storage.engines import PostgresEngine -from synapse.types import JsonDict, RoomStreamToken, StreamToken +from synapse.types import JsonDict, RoomStreamToken, StreamKeyType, StreamToken from synapse.util.caches.descriptors import cached, cachedList logger = logging.getLogger(__name__) @@ -161,7 +161,9 @@ def _get_recent_references_for_event_txn( if len(events) > limit and last_topo_id and last_stream_id: next_key = RoomStreamToken(last_topo_id, last_stream_id) if from_token: - next_token = from_token.copy_and_replace("room_key", next_key) + next_token = from_token.copy_and_replace( + StreamKeyType.ROOM, next_key + ) else: next_token = StreamToken( room_key=next_key, diff --git a/synapse/types.py b/synapse/types.py index 325332a6e00f..bd8071d51d78 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -37,7 +37,7 @@ from frozendict import frozendict from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey -from typing_extensions import TypedDict +from typing_extensions import Final, TypedDict from unpaddedbase64 import decode_base64 from zope.interface import Interface @@ -630,6 +630,22 @@ async def to_string(self, store: "DataStore") -> str: return "s%d" % (self.stream,) +class StreamKeyType: + """Known stream types. + + A stream is a list of entities ordered by an incrementing "stream token". + """ + + ROOM: Final = "room_key" + PRESENCE: Final = "presence_key" + TYPING: Final = "typing_key" + RECEIPT: Final = "receipt_key" + ACCOUNT_DATA: Final = "account_data_key" + PUSH_RULES: Final = "push_rules_key" + TO_DEVICE: Final = "to_device_key" + DEVICE_LIST: Final = "device_list_key" + + @attr.s(slots=True, frozen=True, auto_attribs=True) class StreamToken: """A collection of keys joined together by underscores in the following @@ -743,9 +759,9 @@ def copy_and_advance(self, key: str, new_value: Any) -> "StreamToken": :raises TypeError: if `key` is not the one of the keys tracked by a StreamToken. """ - if key == "room_key": + if key == StreamKeyType.ROOM: new_token = self.copy_and_replace( - "room_key", self.room_key.copy_and_advance(new_value) + StreamKeyType.ROOM, self.room_key.copy_and_advance(new_value) ) return new_token From e24c11afd6bb63de2dd7e029a5839d84f592df3c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 16 May 2022 17:51:43 +0100 Subject: [PATCH 196/263] changelog --- changelog.d/12748.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12748.doc diff --git a/changelog.d/12748.doc b/changelog.d/12748.doc new file mode 100644 index 000000000000..996ad3a1b926 --- /dev/null +++ b/changelog.d/12748.doc @@ -0,0 +1 @@ +Link to the configuration manual from the welcome page of the documentation. From 3eafee629d39eadbf4a3df3cb97801405d232e08 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 16 May 2022 17:52:17 +0100 Subject: [PATCH 197/263] Revert "changelog" This reverts commit e24c11afd6bb63de2dd7e029a5839d84f592df3c. whoops... --- changelog.d/12748.doc | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/12748.doc diff --git a/changelog.d/12748.doc b/changelog.d/12748.doc deleted file mode 100644 index 996ad3a1b926..000000000000 --- a/changelog.d/12748.doc +++ /dev/null @@ -1 +0,0 @@ -Link to the configuration manual from the welcome page of the documentation. From 6d8d1218dde7c265dadf723cf9955d2c2fc77df9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 16 May 2022 18:31:12 +0100 Subject: [PATCH 198/263] Fix typo in name of 'run_background_tasks_on' option in config manual (#12749) --- changelog.d/12749.doc | 1 + docs/usage/configuration/config_documentation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12749.doc diff --git a/changelog.d/12749.doc b/changelog.d/12749.doc new file mode 100644 index 000000000000..4560319ee43b --- /dev/null +++ b/changelog.d/12749.doc @@ -0,0 +1 @@ +Fix typo in 'run_background_tasks_on' option name in configuration manual documentation. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index f0bf13976d2c..3e2031f08aa6 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3454,7 +3454,7 @@ stream_writers: typing: worker1 ``` --- -Config option: `run_background_task_on` +Config option: `run_background_tasks_on` The worker that is used to run background tasks (e.g. cleaning up expired data). If not provided this defaults to the main process. From 1fe202a1a3343fad77da270ffe0923a46f1944dd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 17 May 2022 00:34:38 +0100 Subject: [PATCH 199/263] Tidy up and type-hint the database engine modules (#12734) Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com> --- changelog.d/12734.misc | 1 + mypy.ini | 3 + synapse/storage/engines/__init__.py | 12 ++-- synapse/storage/engines/_base.py | 26 ++++---- synapse/storage/engines/postgres.py | 92 ++++++++++++++++------------- synapse/storage/engines/sqlite.py | 72 +++++++++++----------- synapse/storage/types.py | 70 ++++++++++++++++++++++ 7 files changed, 182 insertions(+), 94 deletions(-) create mode 100644 changelog.d/12734.misc diff --git a/changelog.d/12734.misc b/changelog.d/12734.misc new file mode 100644 index 000000000000..ffbfb0d63233 --- /dev/null +++ b/changelog.d/12734.misc @@ -0,0 +1 @@ +Tidy up and type-hint the database engine modules. diff --git a/mypy.ini b/mypy.ini index 9ae7ad211c54..b5b907973ffc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -232,6 +232,9 @@ disallow_untyped_defs = True [mypy-synapse.storage.databases.main.user_erasure_store] disallow_untyped_defs = True +[mypy-synapse.storage.engines.*] +disallow_untyped_defs = True + [mypy-synapse.storage.prepare_database] disallow_untyped_defs = True diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index afb7d5054db8..f51b3d228ee7 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -11,25 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Mapping from ._base import BaseDatabaseEngine, IncorrectDatabaseSetup from .postgres import PostgresEngine from .sqlite import Sqlite3Engine -def create_engine(database_config) -> BaseDatabaseEngine: +def create_engine(database_config: Mapping[str, Any]) -> BaseDatabaseEngine: name = database_config["name"] if name == "sqlite3": - import sqlite3 - - return Sqlite3Engine(sqlite3, database_config) + return Sqlite3Engine(database_config) if name == "psycopg2": - # Note that psycopg2cffi-compat provides the psycopg2 module on pypy. - import psycopg2 - - return PostgresEngine(psycopg2, database_config) + return PostgresEngine(database_config) raise RuntimeError("Unsupported database engine '%s'" % (name,)) diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 143cd98ca292..971ff8269323 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -13,9 +13,12 @@ # limitations under the License. import abc from enum import IntEnum -from typing import Generic, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Generic, Mapping, Optional, TypeVar -from synapse.storage.types import Connection +from synapse.storage.types import Connection, Cursor, DBAPI2Module + +if TYPE_CHECKING: + from synapse.storage.database import LoggingDatabaseConnection class IsolationLevel(IntEnum): @@ -32,7 +35,7 @@ class IncorrectDatabaseSetup(RuntimeError): class BaseDatabaseEngine(Generic[ConnectionType], metaclass=abc.ABCMeta): - def __init__(self, module, database_config: dict): + def __init__(self, module: DBAPI2Module, config: Mapping[str, Any]): self.module = module @property @@ -69,7 +72,7 @@ def check_database( ... @abc.abstractmethod - def check_new_database(self, txn) -> None: + def check_new_database(self, txn: Cursor) -> None: """Gets called when setting up a brand new database. This allows us to apply stricter checks on new databases versus existing database. """ @@ -79,8 +82,11 @@ def check_new_database(self, txn) -> None: def convert_param_style(self, sql: str) -> str: ... + # This method would ideally take a plain ConnectionType, but it seems that + # the Sqlite engine expects to use LoggingDatabaseConnection.cursor + # instead of sqlite3.Connection.cursor: only the former takes a txn_name. @abc.abstractmethod - def on_new_connection(self, db_conn: ConnectionType) -> None: + def on_new_connection(self, db_conn: "LoggingDatabaseConnection") -> None: ... @abc.abstractmethod @@ -92,7 +98,7 @@ def is_connection_closed(self, conn: ConnectionType) -> bool: ... @abc.abstractmethod - def lock_table(self, txn, table: str) -> None: + def lock_table(self, txn: Cursor, table: str) -> None: ... @property @@ -102,12 +108,12 @@ def server_version(self) -> str: ... @abc.abstractmethod - def in_transaction(self, conn: Connection) -> bool: + def in_transaction(self, conn: ConnectionType) -> bool: """Whether the connection is currently in a transaction.""" ... @abc.abstractmethod - def attempt_to_set_autocommit(self, conn: Connection, autocommit: bool): + def attempt_to_set_autocommit(self, conn: ConnectionType, autocommit: bool) -> None: """Attempt to set the connections autocommit mode. When True queries are run outside of transactions. @@ -119,8 +125,8 @@ def attempt_to_set_autocommit(self, conn: Connection, autocommit: bool): @abc.abstractmethod def attempt_to_set_isolation_level( - self, conn: Connection, isolation_level: Optional[int] - ): + self, conn: ConnectionType, isolation_level: Optional[int] + ) -> None: """Attempt to set the connections isolation level. Note: This has no effect on SQLite3, as transactions are SERIALIZABLE by default. diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index e8d29e287004..391f8ed24a3d 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -13,39 +13,47 @@ # limitations under the License. import logging -from typing import Mapping, Optional +from typing import TYPE_CHECKING, Any, Mapping, NoReturn, Optional, Tuple, cast from synapse.storage.engines._base import ( BaseDatabaseEngine, IncorrectDatabaseSetup, IsolationLevel, ) -from synapse.storage.types import Connection +from synapse.storage.types import Cursor + +if TYPE_CHECKING: + import psycopg2 # noqa: F401 + + from synapse.storage.database import LoggingDatabaseConnection + logger = logging.getLogger(__name__) -class PostgresEngine(BaseDatabaseEngine): - def __init__(self, database_module, database_config): - super().__init__(database_module, database_config) - self.module.extensions.register_type(self.module.extensions.UNICODE) +class PostgresEngine(BaseDatabaseEngine["psycopg2.connection"]): + def __init__(self, database_config: Mapping[str, Any]): + import psycopg2.extensions + + super().__init__(psycopg2, database_config) + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) # Disables passing `bytes` to txn.execute, c.f. #6186. If you do # actually want to use bytes than wrap it in `bytearray`. - def _disable_bytes_adapter(_): + def _disable_bytes_adapter(_: bytes) -> NoReturn: raise Exception("Passing bytes to DB is disabled.") - self.module.extensions.register_adapter(bytes, _disable_bytes_adapter) - self.synchronous_commit = database_config.get("synchronous_commit", True) - self._version = None # unknown as yet + psycopg2.extensions.register_adapter(bytes, _disable_bytes_adapter) + self.synchronous_commit: bool = database_config.get("synchronous_commit", True) + self._version: Optional[int] = None # unknown as yet self.isolation_level_map: Mapping[int, int] = { - IsolationLevel.READ_COMMITTED: self.module.extensions.ISOLATION_LEVEL_READ_COMMITTED, - IsolationLevel.REPEATABLE_READ: self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ, - IsolationLevel.SERIALIZABLE: self.module.extensions.ISOLATION_LEVEL_SERIALIZABLE, + IsolationLevel.READ_COMMITTED: psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED, + IsolationLevel.REPEATABLE_READ: psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ, + IsolationLevel.SERIALIZABLE: psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE, } self.default_isolation_level = ( - self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ + psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ ) self.config = database_config @@ -53,19 +61,21 @@ def _disable_bytes_adapter(_): def single_threaded(self) -> bool: return False - def get_db_locale(self, txn): + def get_db_locale(self, txn: Cursor) -> Tuple[str, str]: txn.execute( "SELECT datcollate, datctype FROM pg_database WHERE datname = current_database()" ) - collation, ctype = txn.fetchone() + collation, ctype = cast(Tuple[str, str], txn.fetchone()) return collation, ctype - def check_database(self, db_conn, allow_outdated_version: bool = False): + def check_database( + self, db_conn: "psycopg2.connection", allow_outdated_version: bool = False + ) -> None: # Get the version of PostgreSQL that we're using. As per the psycopg2 # docs: The number is formed by converting the major, minor, and # revision numbers into two-decimal-digit numbers and appending them # together. For example, version 8.1.5 will be returned as 80105 - self._version = db_conn.server_version + self._version = cast(int, db_conn.server_version) allow_unsafe_locale = self.config.get("allow_unsafe_locale", False) # Are we on a supported PostgreSQL version? @@ -108,7 +118,7 @@ def check_database(self, db_conn, allow_outdated_version: bool = False): ctype, ) - def check_new_database(self, txn): + def check_new_database(self, txn: Cursor) -> None: """Gets called when setting up a brand new database. This allows us to apply stricter checks on new databases versus existing database. """ @@ -129,10 +139,10 @@ def check_new_database(self, txn): "See docs/postgres.md for more information." % ("\n".join(errors)) ) - def convert_param_style(self, sql): + def convert_param_style(self, sql: str) -> str: return sql.replace("?", "%s") - def on_new_connection(self, db_conn): + def on_new_connection(self, db_conn: "LoggingDatabaseConnection") -> None: db_conn.set_isolation_level(self.default_isolation_level) # Set the bytea output to escape, vs the default of hex @@ -149,14 +159,14 @@ def on_new_connection(self, db_conn): db_conn.commit() @property - def can_native_upsert(self): + def can_native_upsert(self) -> bool: """ Can we use native UPSERTs? """ return True @property - def supports_using_any_list(self): + def supports_using_any_list(self) -> bool: """Do we support using `a = ANY(?)` and passing a list""" return True @@ -165,27 +175,25 @@ def supports_returning(self) -> bool: """Do we support the `RETURNING` clause in insert/update/delete?""" return True - def is_deadlock(self, error): - if isinstance(error, self.module.DatabaseError): + def is_deadlock(self, error: Exception) -> bool: + import psycopg2.extensions + + if isinstance(error, psycopg2.DatabaseError): # https://www.postgresql.org/docs/current/static/errcodes-appendix.html # "40001" serialization_failure # "40P01" deadlock_detected return error.pgcode in ["40001", "40P01"] return False - def is_connection_closed(self, conn): + def is_connection_closed(self, conn: "psycopg2.connection") -> bool: return bool(conn.closed) - def lock_table(self, txn, table): + def lock_table(self, txn: Cursor, table: str) -> None: txn.execute("LOCK TABLE %s in EXCLUSIVE MODE" % (table,)) @property - def server_version(self): - """Returns a string giving the server version. For example: '8.1.5' - - Returns: - string - """ + def server_version(self) -> str: + """Returns a string giving the server version. For example: '8.1.5'.""" # note that this is a bit of a hack because it relies on check_database # having been called. Still, that should be a safe bet here. numver = self._version @@ -197,17 +205,21 @@ def server_version(self): else: return "%i.%i.%i" % (numver / 10000, (numver % 10000) / 100, numver % 100) - def in_transaction(self, conn: Connection) -> bool: - return conn.status != self.module.extensions.STATUS_READY # type: ignore + def in_transaction(self, conn: "psycopg2.connection") -> bool: + import psycopg2.extensions + + return conn.status != psycopg2.extensions.STATUS_READY - def attempt_to_set_autocommit(self, conn: Connection, autocommit: bool): - return conn.set_session(autocommit=autocommit) # type: ignore + def attempt_to_set_autocommit( + self, conn: "psycopg2.connection", autocommit: bool + ) -> None: + return conn.set_session(autocommit=autocommit) def attempt_to_set_isolation_level( - self, conn: Connection, isolation_level: Optional[int] - ): + self, conn: "psycopg2.connection", isolation_level: Optional[int] + ) -> None: if isolation_level is None: isolation_level = self.default_isolation_level else: isolation_level = self.isolation_level_map[isolation_level] - return conn.set_isolation_level(isolation_level) # type: ignore + return conn.set_isolation_level(isolation_level) diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 6c19e55999bd..621f2c5efe28 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -12,21 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. import platform +import sqlite3 import struct import threading -import typing -from typing import Optional +from typing import TYPE_CHECKING, Any, List, Mapping, Optional from synapse.storage.engines import BaseDatabaseEngine -from synapse.storage.types import Connection +from synapse.storage.types import Cursor -if typing.TYPE_CHECKING: - import sqlite3 # noqa: F401 +if TYPE_CHECKING: + from synapse.storage.database import LoggingDatabaseConnection -class Sqlite3Engine(BaseDatabaseEngine["sqlite3.Connection"]): - def __init__(self, database_module, database_config): - super().__init__(database_module, database_config) +class Sqlite3Engine(BaseDatabaseEngine[sqlite3.Connection]): + def __init__(self, database_config: Mapping[str, Any]): + super().__init__(sqlite3, database_config) database = database_config.get("args", {}).get("database") self._is_in_memory = database in ( @@ -37,7 +37,7 @@ def __init__(self, database_module, database_config): if platform.python_implementation() == "PyPy": # pypy's sqlite3 module doesn't handle bytearrays, convert them # back to bytes. - database_module.register_adapter(bytearray, lambda array: bytes(array)) + sqlite3.register_adapter(bytearray, lambda array: bytes(array)) # The current max state_group, or None if we haven't looked # in the DB yet. @@ -49,41 +49,43 @@ def single_threaded(self) -> bool: return True @property - def can_native_upsert(self): + def can_native_upsert(self) -> bool: """ Do we support native UPSERTs? This requires SQLite3 3.24+, plus some more work we haven't done yet to tell what was inserted vs updated. """ - return self.module.sqlite_version_info >= (3, 24, 0) + return sqlite3.sqlite_version_info >= (3, 24, 0) @property - def supports_using_any_list(self): + def supports_using_any_list(self) -> bool: """Do we support using `a = ANY(?)` and passing a list""" return False @property def supports_returning(self) -> bool: """Do we support the `RETURNING` clause in insert/update/delete?""" - return self.module.sqlite_version_info >= (3, 35, 0) + return sqlite3.sqlite_version_info >= (3, 35, 0) - def check_database(self, db_conn, allow_outdated_version: bool = False): + def check_database( + self, db_conn: sqlite3.Connection, allow_outdated_version: bool = False + ) -> None: if not allow_outdated_version: - version = self.module.sqlite_version_info + version = sqlite3.sqlite_version_info # Synapse is untested against older SQLite versions, and we don't want # to let users upgrade to a version of Synapse with broken support for their # sqlite version, because it risks leaving them with a half-upgraded db. if version < (3, 22, 0): raise RuntimeError("Synapse requires sqlite 3.22 or above.") - def check_new_database(self, txn): + def check_new_database(self, txn: Cursor) -> None: """Gets called when setting up a brand new database. This allows us to apply stricter checks on new databases versus existing database. """ - def convert_param_style(self, sql): + def convert_param_style(self, sql: str) -> str: return sql - def on_new_connection(self, db_conn): + def on_new_connection(self, db_conn: "LoggingDatabaseConnection") -> None: # We need to import here to avoid an import loop. from synapse.storage.prepare_database import prepare_database @@ -97,48 +99,46 @@ def on_new_connection(self, db_conn): db_conn.execute("PRAGMA foreign_keys = ON;") db_conn.commit() - def is_deadlock(self, error): + def is_deadlock(self, error: Exception) -> bool: return False - def is_connection_closed(self, conn): + def is_connection_closed(self, conn: sqlite3.Connection) -> bool: return False - def lock_table(self, txn, table): + def lock_table(self, txn: Cursor, table: str) -> None: return @property - def server_version(self): - """Gets a string giving the server version. For example: '3.22.0' + def server_version(self) -> str: + """Gets a string giving the server version. For example: '3.22.0'.""" + return "%i.%i.%i" % sqlite3.sqlite_version_info - Returns: - string - """ - return "%i.%i.%i" % self.module.sqlite_version_info - - def in_transaction(self, conn: Connection) -> bool: - return conn.in_transaction # type: ignore + def in_transaction(self, conn: sqlite3.Connection) -> bool: + return conn.in_transaction - def attempt_to_set_autocommit(self, conn: Connection, autocommit: bool): + def attempt_to_set_autocommit( + self, conn: sqlite3.Connection, autocommit: bool + ) -> None: # Twisted doesn't let us set attributes on the connections, so we can't # set the connection to autocommit mode. pass def attempt_to_set_isolation_level( - self, conn: Connection, isolation_level: Optional[int] - ): - # All transactions are SERIALIZABLE by default in sqllite + self, conn: sqlite3.Connection, isolation_level: Optional[int] + ) -> None: + # All transactions are SERIALIZABLE by default in sqlite pass # Following functions taken from: https://github.com/coleifer/peewee -def _parse_match_info(buf): +def _parse_match_info(buf: bytes) -> List[int]: bufsize = len(buf) return [struct.unpack("@I", buf[i : i + 4])[0] for i in range(0, bufsize, 4)] -def _rank(raw_match_info): +def _rank(raw_match_info: bytes) -> float: """Handle match_info called w/default args 'pcx' - based on the example rank function http://sqlite.org/fts3.html#appendix_a """ diff --git a/synapse/storage/types.py b/synapse/storage/types.py index 40536c183005..0031df1e0649 100644 --- a/synapse/storage/types.py +++ b/synapse/storage/types.py @@ -94,3 +94,73 @@ def __exit__( traceback: Optional[TracebackType], ) -> Optional[bool]: ... + + +class DBAPI2Module(Protocol): + """The module-level attributes that we use from PEP 249. + + This is NOT a comprehensive stub for the entire DBAPI2.""" + + __name__: str + + # Exceptions. See https://peps.python.org/pep-0249/#exceptions + + # For our specific drivers: + # - Python's sqlite3 module doesn't contains the same descriptions as the + # DBAPI2 spec, see https://docs.python.org/3/library/sqlite3.html#exceptions + # - Psycopg2 maps every Postgres error code onto a unique exception class which + # extends from this hierarchy. See + # https://docs.python.org/3/library/sqlite3.html?highlight=sqlite3#exceptions + # https://www.postgresql.org/docs/current/errcodes-appendix.html#ERRCODES-TABLE + Warning: Type[Exception] + Error: Type[Exception] + + # Errors are divided into `InterfaceError`s (something went wrong in the database + # driver) and `DatabaseError`s (something went wrong in the database). These are + # both subclasses of `Error`, but we can't currently express this in type + # annotations due to https://github.com/python/mypy/issues/8397 + InterfaceError: Type[Exception] + DatabaseError: Type[Exception] + + # Everything below is a subclass of `DatabaseError`. + + # Roughly: the database rejected a nonsensical value. Examples: + # - An integer was too big for its data type. + # - An invalid date time was provided. + # - A string contained a null code point. + DataError: Type[Exception] + + # Roughly: something went wrong in the database, but it's not within the application + # programmer's control. Examples: + # - We failed to establish a connection to the database. + # - The connection to the database was lost. + # - A deadlock was detected. + # - A serialisation failure occurred. + # - The database ran out of resources, such as storage, memory, connections, etc. + # - The database encountered an error from the operating system. + OperationalError: Type[Exception] + + # Roughly: we've given the database data which breaks a rule we asked it to enforce. + # Examples: + # - Stop, criminal scum! You violated the foreign key constraint + # - Also check constraints, non-null constraints, etc. + IntegrityError: Type[Exception] + + # Roughly: something went wrong within the database server itself. + InternalError: Type[Exception] + + # Roughly: the application did something silly that needs to be fixed. Examples: + # - We don't have permissions to do something. + # - We tried to create a table with duplicate column names. + # - We tried to use a reserved name. + # - We referred to a column that doesn't exist. + ProgrammingError: Type[Exception] + + # Roughly: we've tried to do something that this database doesn't support. + NotSupportedError: Type[Exception] + + def connect(self, **parameters: object) -> Connection: + ... + + +__all__ = ["Cursor", "Connection", "DBAPI2Module"] From 5c3d525cada69c2df3be65111d62ddcb52623a16 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 17 May 2022 10:27:51 +0100 Subject: [PATCH 200/263] 1.59.0 --- CHANGES.md | 16 ++++++++++++++-- changelog.d/12745.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/12745.bugfix diff --git a/CHANGES.md b/CHANGES.md index 347aae8f67d6..81feeb9273c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -Synapse 1.59.0rc2 (2022-05-16) -============================== +Synapse 1.59.0 (2022-05-17) +=========================== Synapse 1.59 makes several changes that server administrators should be aware of: @@ -10,6 +10,18 @@ See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/ Additionally, this release removes the non-standard `m.login.jwt` login type from Synapse. It can be replaced with `org.matrix.login.jwt` for identical behaviour. This is only used if `jwt_config.enabled` is set to `true` in the configuration. ([\#12597](https://github.com/matrix-org/synapse/issues/12597)) + +Bugfixes +-------- + +- Fix DB performance regression introduced in v1.59.0rc2. ([\#12745](https://github.com/matrix-org/synapse/issues/12745)) + + +Synapse 1.59.0rc2 (2022-05-16) +============================== + +Note: this release candidate includes a performance regression which can cause database disruption. Other release candidates in the v1.59.0 series are not affected, and a fix will be included in the v1.59.0 final release. + Bugfixes -------- diff --git a/changelog.d/12745.bugfix b/changelog.d/12745.bugfix deleted file mode 100644 index d1dc8d03aaec..000000000000 --- a/changelog.d/12745.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix DB performance regression introduced in v1.59.0rc2. diff --git a/debian/changelog b/debian/changelog index 7f9bedd7ac0e..cc6152d5b314 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.59.0) stable; urgency=medium + + * New Synapse release 1.59.0. + + -- Synapse Packaging team Tue, 17 May 2022 10:26:50 +0100 + matrix-synapse-py3 (1.59.0~rc2) stable; urgency=medium * New Synapse release 1.59.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 1a2ddddb1010..e600a1d52e4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.59.0rc2" +version = "1.59.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 44d7bb13c357fb8cb6ff7c75f5abc189fa530529 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 17 May 2022 10:30:31 +0100 Subject: [PATCH 201/263] version tweak in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 81feeb9273c7..56d1a5a7d7bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ Additionally, this release removes the non-standard `m.login.jwt` login type fro Bugfixes -------- -- Fix DB performance regression introduced in v1.59.0rc2. ([\#12745](https://github.com/matrix-org/synapse/issues/12745)) +- Fix DB performance regression introduced in Synapse 1.59.0rc2. ([\#12745](https://github.com/matrix-org/synapse/issues/12745)) Synapse 1.59.0rc2 (2022-05-16) From fcf951d5dc7ca8c4cb18aa9c1f5ccb005df3610a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 May 2022 10:34:27 +0100 Subject: [PATCH 202/263] Track in memory events using weakrefs (#10533) --- changelog.d/10533.misc | 1 + .../storage/databases/main/events_worker.py | 35 +++++++++++++++++-- tests/handlers/test_sync.py | 1 + .../databases/main/test_events_worker.py | 25 +++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 changelog.d/10533.misc diff --git a/changelog.d/10533.misc b/changelog.d/10533.misc new file mode 100644 index 000000000000..f70dc6496fcf --- /dev/null +++ b/changelog.d/10533.misc @@ -0,0 +1 @@ +Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index a4a604a49915..5b22d6b45211 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -14,6 +14,7 @@ import logging import threading +import weakref from enum import Enum, auto from typing import ( TYPE_CHECKING, @@ -23,6 +24,7 @@ Dict, Iterable, List, + MutableMapping, Optional, Set, Tuple, @@ -248,6 +250,12 @@ def __init__( str, ObservableDeferred[Dict[str, EventCacheEntry]] ] = {} + # We keep track of the events we have currently loaded in memory so that + # we can reuse them even if they've been evicted from the cache. We only + # track events that don't need redacting in here (as then we don't need + # to track redaction status). + self._event_ref: MutableMapping[str, EventBase] = weakref.WeakValueDictionary() + self._event_fetch_lock = threading.Condition() self._event_fetch_list: List[ Tuple[Iterable[str], "defer.Deferred[Dict[str, _EventRow]]"] @@ -723,6 +731,8 @@ async def get_missing_events_from_db() -> Dict[str, EventCacheEntry]: def _invalidate_get_event_cache(self, event_id: str) -> None: self._get_event_cache.invalidate((event_id,)) + self._event_ref.pop(event_id, None) + self._current_event_fetches.pop(event_id, None) def _get_events_from_cache( self, events: Iterable[str], update_metrics: bool = True @@ -738,13 +748,30 @@ def _get_events_from_cache( event_map = {} for event_id in events: + # First check if it's in the event cache ret = self._get_event_cache.get( (event_id,), None, update_metrics=update_metrics ) - if not ret: + if ret: + event_map[event_id] = ret continue - event_map[event_id] = ret + # Otherwise check if we still have the event in memory. + event = self._event_ref.get(event_id) + if event: + # Reconstruct an event cache entry + + cache_entry = EventCacheEntry( + event=event, + # We don't cache weakrefs to redacted events, so we know + # this is None. + redacted_event=None, + ) + event_map[event_id] = cache_entry + + # We add the entry back into the cache as we want to keep + # recently queried events in the cache. + self._get_event_cache.set((event_id,), cache_entry) return event_map @@ -1124,6 +1151,10 @@ async def _get_events_from_db( self._get_event_cache.set((event_id,), cache_entry) result_map[event_id] = cache_entry + if not redacted_event: + # We only cache references to unredacted events. + self._event_ref[event_id] = original_ev + return result_map async def _enqueue_events(self, events: Collection[str]) -> Dict[str, _EventRow]: diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 865b8b7e47ef..db3302a4c78d 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -160,6 +160,7 @@ def test_unknown_room_version(self): # Blow away caches (supported room versions can only change due to a restart). self.store.get_rooms_for_user_with_stream_ordering.invalidate_all() self.store._get_event_cache.clear() + self.store._event_ref.clear() # The rooms should be excluded from the sync response. # Get a new request key. diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index c237a8c7e228..38963ce4a74c 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -154,6 +154,31 @@ def test_simple(self): # We should have fetched the event from the DB self.assertEqual(ctx.get_resource_usage().evt_db_fetch_count, 1) + def test_event_ref(self): + """Test that we reuse events that are still in memory but have fallen + out of the cache, rather than requesting them from the DB. + """ + + # Reset the event cache + self.store._get_event_cache.clear() + + with LoggingContext("test") as ctx: + # We keep hold of the event event though we never use it. + event = self.get_success(self.store.get_event(self.event_id)) # noqa: F841 + + # We should have fetched the event from the DB + self.assertEqual(ctx.get_resource_usage().evt_db_fetch_count, 1) + + # Reset the event cache + self.store._get_event_cache.clear() + + with LoggingContext("test") as ctx: + self.get_success(self.store.get_event(self.event_id)) + + # Since the event is still in memory we shouldn't have fetched it + # from the DB + self.assertEqual(ctx.get_resource_usage().evt_db_fetch_count, 0) + def test_dedupe(self): """Test that if we request the same event multiple times we only pull it out once. From 32ef24fbd74b8822c3e57c8ce74b979506aea7be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 May 2022 10:34:59 +0100 Subject: [PATCH 203/263] Add index to cache invalidations (#12747) For workers that rarely write to the cache the `get_all_updated_caches` query can become expensive if the worker falls behind when reading the cache. --- changelog.d/12747.bugfix | 1 + synapse/storage/databases/main/cache.py | 8 ++++++++ .../delta/69/02cache_invalidation_index.sql | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 changelog.d/12747.bugfix create mode 100644 synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql diff --git a/changelog.d/12747.bugfix b/changelog.d/12747.bugfix new file mode 100644 index 000000000000..0fb0059237cc --- /dev/null +++ b/changelog.d/12747.bugfix @@ -0,0 +1 @@ +Fix poor database performance when reading the cache invalidation stream for large servers with lots of workers. diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index dd4e83a2ad19..1653a6a9b694 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -57,6 +57,14 @@ def __init__( self._instance_name = hs.get_instance_name() + self.db_pool.updates.register_background_index_update( + update_name="cache_invalidation_index_by_instance", + index_name="cache_invalidation_stream_by_instance_instance_index", + table="cache_invalidation_stream_by_instance", + columns=("instance_name", "stream_id"), + psql_only=True, # The table is only on postgres DBs. + ) + async def get_all_updated_caches( self, instance_name: str, last_id: int, current_id: int, limit: int ) -> Tuple[List[Tuple[int, tuple]], int, bool]: diff --git a/synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql b/synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql new file mode 100644 index 000000000000..22ae3b8c0005 --- /dev/null +++ b/synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql @@ -0,0 +1,18 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Background update to clear the inboxes of hidden and deleted devices. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (6902, 'cache_invalidation_index_by_instance', '{}'); From a34a41f1354147565da248bf3222449c6e976035 Mon Sep 17 00:00:00 2001 From: SpiritCroc Date: Tue, 17 May 2022 12:03:07 +0200 Subject: [PATCH 204/263] Fix push for m.read events (#12721) badge_count_last_call was always zero when the response for push notifications included a "rejected" key which mapped to an empty list. --- changelog.d/12721.bugfix | 1 + synapse/push/httppusher.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12721.bugfix diff --git a/changelog.d/12721.bugfix b/changelog.d/12721.bugfix new file mode 100644 index 000000000000..6987f7ab15e1 --- /dev/null +++ b/changelog.d/12721.bugfix @@ -0,0 +1 @@ +Fix push to dismiss notifications when read on another client. Contributed by @SpiritCroc @ Beeper. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5818344520f5..d5603596c004 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -405,7 +405,7 @@ async def dispatch_push( rejected = [] if "rejected" in resp: rejected = resp["rejected"] - else: + if not rejected: self.badge_count_last_call = badge return rejected From 24b590de32154eb3965220bd62715e52b37b4074 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 17 May 2022 12:07:18 +0200 Subject: [PATCH 205/263] Remove code which updates `application_services_state.last_txn` (#12680) This column is unused as of #12209, so let's stop writing to it. --- changelog.d/12680.misc | 1 + synapse/storage/databases/main/appservice.py | 47 ++++++++++---------- synapse/storage/schema/__init__.py | 5 ++- tests/handlers/test_appservice.py | 10 ----- tests/storage/test_appservice.py | 27 +++-------- 5 files changed, 35 insertions(+), 55 deletions(-) create mode 100644 changelog.d/12680.misc diff --git a/changelog.d/12680.misc b/changelog.d/12680.misc new file mode 100644 index 000000000000..dfd1f0a6c658 --- /dev/null +++ b/changelog.d/12680.misc @@ -0,0 +1 @@ +Remove code which updates unused database column `application_services_state.last_txn`. diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 945707b0ecd5..e284454b660f 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -203,19 +203,29 @@ async def get_appservice_state( """Get the application service state. Args: - service: The service whose state to set. + service: The service whose state to get. Returns: - An ApplicationServiceState or none. + An ApplicationServiceState, or None if we have yet to attempt any + transactions to the AS. """ - result = await self.db_pool.simple_select_one( + # if we have created transactions for this AS but not yet attempted to send + # them, we will have a row in the table with state=NULL (recording the stream + # positions we have processed up to). + # + # On the other hand, if we have yet to create any transactions for this AS at + # all, then there will be no row for the AS. + # + # In either case, we return None to indicate "we don't yet know the state of + # this AS". + result = await self.db_pool.simple_select_one_onecol( "application_services_state", {"as_id": service.id}, - ["state"], + retcol="state", allow_none=True, desc="get_appservice_state", ) if result: - return ApplicationServiceState(result.get("state")) + return ApplicationServiceState(result) return None async def set_appservice_state( @@ -296,14 +306,6 @@ async def complete_appservice_txn( """ def _complete_appservice_txn(txn: LoggingTransaction) -> None: - # Set current txn_id for AS to 'txn_id' - self.db_pool.simple_upsert_txn( - txn, - "application_services_state", - {"as_id": service.id}, - {"last_txn": txn_id}, - ) - # Delete txn self.db_pool.simple_delete_txn( txn, @@ -452,16 +454,15 @@ async def set_appservice_stream_type_pos( % (stream_type,) ) - def set_appservice_stream_type_pos_txn(txn: LoggingTransaction) -> None: - stream_id_type = "%s_stream_id" % stream_type - txn.execute( - "UPDATE application_services_state SET %s = ? WHERE as_id=?" - % stream_id_type, - (pos, service.id), - ) - - await self.db_pool.runInteraction( - "set_appservice_stream_type_pos", set_appservice_stream_type_pos_txn + # this may be the first time that we're recording any state for this AS, so + # we don't yet know if a row for it exists; hence we have to upsert here. + await self.db_pool.simple_upsert( + table="application_services_state", + keyvalues={"as_id": service.id}, + values={f"{stream_type}_stream_id": pos}, + # no need to lock when emulating upsert: as_id is a unique key + lock=False, + desc="set_appservice_stream_type_pos", ) diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 20c344faeab3..da98f05e0348 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -61,7 +61,9 @@ Changes in SCHEMA_VERSION = 69: - We now write to `device_lists_changes_in_room` table. - - Use sequence to generate future `application_services_txns.txn_id`s + - We now use a PostgreSQL sequence to generate future txn_ids for + `application_services_txns`. `application_services_state.last_txn` is no longer + updated. Changes in SCHEMA_VERSION = 70: - event_reference_hashes is no longer written to. @@ -71,6 +73,7 @@ SCHEMA_COMPAT_VERSION = ( # We now assume that `device_lists_changes_in_room` has been filled out for # recent device_list_updates. + # ... and that `application_services_state.last_txn` is not used. 69 ) """Limit on how far the synapse codebase can be rolled back without breaking db compat diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 5b0cd1ab8608..53e7a5d81b7d 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -434,16 +434,6 @@ def test_sending_read_receipt_batches_to_application_services(self): }, ) - # "Complete" a transaction. - # All this really does for us is make an entry in the application_services_state - # database table, which tracks the current stream_token per stream ID per AS. - self.get_success( - self.hs.get_datastores().main.complete_appservice_txn( - 0, - interested_appservice, - ) - ) - # Now, pretend that we receive a large burst of read receipts (300 total) that # all come in at once. for i in range(300): diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 1bf93e79a7fa..1047ed09c85d 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -14,7 +14,7 @@ import json import os import tempfile -from typing import List, Optional, cast +from typing import List, cast from unittest.mock import Mock import yaml @@ -149,15 +149,12 @@ def _add_service(self, url, as_token, id) -> None: outfile.write(yaml.dump(as_yaml)) self.as_yaml_files.append(as_token) - def _set_state( - self, id: str, state: ApplicationServiceState, txn: Optional[int] = None - ): + def _set_state(self, id: str, state: ApplicationServiceState): return self.db_pool.runOperation( self.engine.convert_param_style( - "INSERT INTO application_services_state(as_id, state, last_txn) " - "VALUES(?,?,?)" + "INSERT INTO application_services_state(as_id, state) VALUES(?,?)" ), - (id, state.value, txn), + (id, state.value), ) def _insert_txn(self, as_id, txn_id, events): @@ -280,17 +277,6 @@ def test_complete_appservice_txn_first_txn( self.store.complete_appservice_txn(txn_id=txn_id, service=service) ) - res = self.get_success( - self.db_pool.runQuery( - self.engine.convert_param_style( - "SELECT last_txn FROM application_services_state WHERE as_id=?" - ), - (service.id,), - ) - ) - self.assertEqual(1, len(res)) - self.assertEqual(txn_id, res[0][0]) - res = self.get_success( self.db_pool.runQuery( self.engine.convert_param_style( @@ -316,14 +302,13 @@ def test_complete_appservice_txn_updates_last_txn_state( res = self.get_success( self.db_pool.runQuery( self.engine.convert_param_style( - "SELECT last_txn, state FROM application_services_state WHERE as_id=?" + "SELECT state FROM application_services_state WHERE as_id=?" ), (service.id,), ) ) self.assertEqual(1, len(res)) - self.assertEqual(txn_id, res[0][0]) - self.assertEqual(ApplicationServiceState.UP.value, res[0][1]) + self.assertEqual(ApplicationServiceState.UP.value, res[0][0]) res = self.get_success( self.db_pool.runQuery( From 942c30b16b86cb05d2109b13bc2c1dc9ac2fea70 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 May 2022 04:41:39 -0600 Subject: [PATCH 206/263] Add a new room version for MSC3787's knock+restricted join rule (#12623) --- changelog.d/12623.feature | 1 + synapse/api/constants.py | 2 ++ synapse/api/room_versions.py | 32 ++++++++++++++++++++++++++++++++ synapse/event_auth.py | 21 +++++++++++++++++---- synapse/handlers/event_auth.py | 10 +++++++++- synapse/handlers/room_summary.py | 9 +++++++-- 6 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 changelog.d/12623.feature diff --git a/changelog.d/12623.feature b/changelog.d/12623.feature new file mode 100644 index 000000000000..cdee19fafa36 --- /dev/null +++ b/changelog.d/12623.feature @@ -0,0 +1 @@ +Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787). \ No newline at end of file diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0ccd4c95581e..330de21f6b80 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -65,6 +65,8 @@ class JoinRules: PRIVATE: Final = "private" # As defined for MSC3083. RESTRICTED: Final = "restricted" + # As defined for MSC3787. + KNOCK_RESTRICTED: Final = "knock_restricted" class RestrictedJoinRuleTypes: diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index a747a4081497..3f85d61b4633 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -81,6 +81,9 @@ class RoomVersion: msc2716_historical: bool # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events msc2716_redactions: bool + # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of + # knocks and restricted join rules into the same join condition. + msc3787_knock_restricted_join_rule: bool class RoomVersions: @@ -99,6 +102,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V2 = RoomVersion( "2", @@ -115,6 +119,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V3 = RoomVersion( "3", @@ -131,6 +136,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V4 = RoomVersion( "4", @@ -147,6 +153,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V5 = RoomVersion( "5", @@ -163,6 +170,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V6 = RoomVersion( "6", @@ -179,6 +187,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -195,6 +204,7 @@ class RoomVersions: msc2403_knocking=False, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V7 = RoomVersion( "7", @@ -211,6 +221,7 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V8 = RoomVersion( "8", @@ -227,6 +238,7 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) V9 = RoomVersion( "9", @@ -243,6 +255,7 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=False, msc2716_redactions=False, + msc3787_knock_restricted_join_rule=False, ) MSC2716v3 = RoomVersion( "org.matrix.msc2716v3", @@ -259,6 +272,24 @@ class RoomVersions: msc2403_knocking=True, msc2716_historical=True, msc2716_redactions=True, + msc3787_knock_restricted_join_rule=False, + ) + MSC3787 = RoomVersion( + "org.matrix.msc3787", + RoomDisposition.UNSTABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + msc3083_join_rules=True, + msc3375_redaction_rules=True, + msc2403_knocking=True, + msc2716_historical=False, + msc2716_redactions=False, + msc3787_knock_restricted_join_rule=True, ) @@ -276,6 +307,7 @@ class RoomVersions: RoomVersions.V8, RoomVersions.V9, RoomVersions.MSC2716v3, + RoomVersions.MSC3787, ) } diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 621a3efcccec..4c0b587a7643 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -414,7 +414,12 @@ def _is_membership_change_allowed( raise AuthError(403, "You are banned from this room") elif join_rule == JoinRules.PUBLIC: pass - elif room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED: + elif ( + room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED + ) or ( + room_version.msc3787_knock_restricted_join_rule + and join_rule == JoinRules.KNOCK_RESTRICTED + ): # This is the same as public, but the event must contain a reference # to the server who authorised the join. If the event does not contain # the proper content it is rejected. @@ -440,8 +445,13 @@ def _is_membership_change_allowed( if authorising_user_level < invite_level: raise AuthError(403, "Join event authorised by invalid server.") - elif join_rule == JoinRules.INVITE or ( - room_version.msc2403_knocking and join_rule == JoinRules.KNOCK + elif ( + join_rule == JoinRules.INVITE + or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK) + or ( + room_version.msc3787_knock_restricted_join_rule + and join_rule == JoinRules.KNOCK_RESTRICTED + ) ): if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") @@ -462,7 +472,10 @@ def _is_membership_change_allowed( if user_level < ban_level or user_level <= target_level: raise AuthError(403, "You don't have permission to ban") elif room_version.msc2403_knocking and Membership.KNOCK == membership: - if join_rule != JoinRules.KNOCK: + if join_rule != JoinRules.KNOCK and ( + not room_version.msc3787_knock_restricted_join_rule + or join_rule != JoinRules.KNOCK_RESTRICTED + ): raise AuthError(403, "You don't have permission to knock") elif target_user_id != event.user_id: raise AuthError(403, "You cannot knock for other users") diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index d441ebb0ab3d..6bed46435135 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -241,7 +241,15 @@ async def has_restricted_join_rules( # If the join rule is not restricted, this doesn't apply. join_rules_event = await self._store.get_event(join_rules_event_id) - return join_rules_event.content.get("join_rule") == JoinRules.RESTRICTED + content_join_rule = join_rules_event.content.get("join_rule") + if content_join_rule == JoinRules.RESTRICTED: + return True + + # also check for MSC3787 behaviour + if room_version.msc3787_knock_restricted_join_rule: + return content_join_rule == JoinRules.KNOCK_RESTRICTED + + return False async def get_rooms_that_allow_join( self, state_ids: StateMap[str] diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index ff24ec806357..af83de319348 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -562,8 +562,13 @@ async def _is_local_room_accessible( if join_rules_event_id: join_rules_event = await self._store.get_event(join_rules_event_id) join_rule = join_rules_event.content.get("join_rule") - if join_rule == JoinRules.PUBLIC or ( - room_version.msc2403_knocking and join_rule == JoinRules.KNOCK + if ( + join_rule == JoinRules.PUBLIC + or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK) + or ( + room_version.msc3787_knock_restricted_join_rule + and join_rule == JoinRules.KNOCK_RESTRICTED + ) ): return True From 6edefef60289cc54e17fd6af838eb66c4973f5f5 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 17 May 2022 16:29:06 +0200 Subject: [PATCH 207/263] Add some type hints to datastore (#12717) --- changelog.d/12717.misc | 1 + mypy.ini | 2 - synapse/federation/sender/__init__.py | 24 ++- synapse/handlers/sync.py | 6 +- synapse/rest/client/push_rule.py | 4 +- synapse/state/__init__.py | 4 +- synapse/storage/databases/main/__init__.py | 8 +- synapse/storage/databases/main/metrics.py | 56 +++--- synapse/storage/databases/main/push_rule.py | 184 ++++++++++++------- synapse/storage/databases/main/roommember.py | 126 ++++++++----- 10 files changed, 254 insertions(+), 161 deletions(-) create mode 100644 changelog.d/12717.misc diff --git a/changelog.d/12717.misc b/changelog.d/12717.misc new file mode 100644 index 000000000000..e793d08e5e3f --- /dev/null +++ b/changelog.d/12717.misc @@ -0,0 +1 @@ +Add some type hints to datastore. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index b5b907973ffc..45668974b363 100644 --- a/mypy.ini +++ b/mypy.ini @@ -28,8 +28,6 @@ exclude = (?x) |synapse/storage/databases/main/cache.py |synapse/storage/databases/main/devices.py |synapse/storage/databases/main/event_federation.py - |synapse/storage/databases/main/push_rule.py - |synapse/storage/databases/main/roommember.py |synapse/storage/schema/ |tests/api/test_auth.py diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 6d2f46318bea..dbe303ed9be8 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -15,7 +15,17 @@ import abc import logging from collections import OrderedDict -from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Optional, Set, Tuple +from typing import ( + TYPE_CHECKING, + Collection, + Dict, + Hashable, + Iterable, + List, + Optional, + Set, + Tuple, +) import attr from prometheus_client import Counter @@ -409,7 +419,7 @@ async def handle_event(event: EventBase) -> None: ) return - destinations: Optional[Set[str]] = None + destinations: Optional[Collection[str]] = None if not event.prev_event_ids(): # If there are no prev event IDs then the state is empty # and so no remote servers in the room @@ -444,7 +454,7 @@ async def handle_event(event: EventBase) -> None: ) return - destinations = { + sharded_destinations = { d for d in destinations if self._federation_shard_config.should_handle( @@ -456,12 +466,12 @@ async def handle_event(event: EventBase) -> None: # If we are sending the event on behalf of another server # then it already has the event and there is no reason to # send the event to it. - destinations.discard(send_on_behalf_of) + sharded_destinations.discard(send_on_behalf_of) - logger.debug("Sending %s to %r", event, destinations) + logger.debug("Sending %s to %r", event, sharded_destinations) - if destinations: - await self._send_pdu(event, destinations) + if sharded_destinations: + await self._send_pdu(event, sharded_destinations) now = self.clock.time_msec() ts = await self.store.get_received_ts(event.event_id) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 4be08fe7cbc6..59b5d497be68 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -411,10 +411,10 @@ async def current_sync_for_user( set_tag(SynapseTags.SYNC_RESULT, bool(sync_result)) return sync_result - async def push_rules_for_user(self, user: UserID) -> JsonDict: + async def push_rules_for_user(self, user: UserID) -> Dict[str, Dict[str, list]]: user_id = user.to_string() - rules = await self.store.get_push_rules_for_user(user_id) - rules = format_push_rules_for_user(user, rules) + rules_raw = await self.store.get_push_rules_for_user(user_id) + rules = format_push_rules_for_user(user, rules_raw) return rules async def ephemeral_by_room( diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index b98640b14ac5..8191b4e32c34 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -148,9 +148,9 @@ async def on_GET(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDic # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rules = await self.store.get_push_rules_for_user(user_id) + rules_raw = await self.store.get_push_rules_for_user(user_id) - rules = format_push_rules_for_user(requester.user, rules) + rules = format_push_rules_for_user(requester.user, rules_raw) path_parts = path.split("/")[1:] diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 54e41d537584..0219091c4e8b 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -239,13 +239,13 @@ async def get_current_users_in_room( entry = await self.resolve_state_groups_for_events(room_id, latest_event_ids) return await self.store.get_joined_users_from_state(room_id, entry) - async def get_current_hosts_in_room(self, room_id: str) -> Set[str]: + async def get_current_hosts_in_room(self, room_id: str) -> FrozenSet[str]: event_ids = await self.store.get_latest_event_ids_in_room(room_id) return await self.get_hosts_in_room_at_events(room_id, event_ids) async def get_hosts_in_room_at_events( self, room_id: str, event_ids: Collection[str] - ) -> Set[str]: + ) -> FrozenSet[str]: """Get the hosts that were in a room at the given event ids Args: diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 5895b892024c..d545a1c002c9 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -26,11 +26,7 @@ from synapse.storage.databases.main.stats import UserSortOrder from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine from synapse.storage.types import Cursor -from synapse.storage.util.id_generators import ( - IdGenerator, - MultiWriterIdGenerator, - StreamIdGenerator, -) +from synapse.storage.util.id_generators import MultiWriterIdGenerator, StreamIdGenerator from synapse.types import JsonDict, get_domain_from_id from synapse.util.caches.stream_change_cache import StreamChangeCache @@ -155,8 +151,6 @@ def __init__( ], ) - self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") - self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") self._group_updates_id_gen = StreamIdGenerator( db_conn, "local_group_updates", "stream_id" ) diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index d03555a5857b..14294a0bb85a 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -14,16 +14,19 @@ import calendar import logging import time -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, List, Tuple, cast from synapse.metrics import GaugeBucketCollector from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore -from synapse.storage.database import DatabasePool, LoggingDatabaseConnection +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.storage.databases.main.event_push_actions import ( EventPushActionsWorkerStore, ) -from synapse.storage.types import Cursor if TYPE_CHECKING: from synapse.server import HomeServer @@ -73,7 +76,7 @@ def __init__( @wrap_as_background_process("read_forward_extremities") async def _read_forward_extremities(self) -> None: - def fetch(txn): + def fetch(txn: LoggingTransaction) -> List[Tuple[int, int]]: txn.execute( """ SELECT t1.c, t2.c @@ -86,7 +89,7 @@ def fetch(txn): ) t2 ON t1.room_id = t2.room_id """ ) - return txn.fetchall() + return cast(List[Tuple[int, int]], txn.fetchall()) res = await self.db_pool.runInteraction("read_forward_extremities", fetch) @@ -104,20 +107,20 @@ async def count_daily_e2ee_messages(self) -> int: call to this function, it will return None. """ - def _count_messages(txn): + def _count_messages(txn: LoggingTransaction) -> int: sql = """ SELECT COUNT(*) FROM events WHERE type = 'm.room.encrypted' AND stream_ordering > ? """ txn.execute(sql, (self.stream_ordering_day_ago,)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction("count_e2ee_messages", _count_messages) async def count_daily_sent_e2ee_messages(self) -> int: - def _count_messages(txn): + def _count_messages(txn: LoggingTransaction) -> int: # This is good enough as if you have silly characters in your own # hostname then that's your own fault. like_clause = "%:" + self.hs.hostname @@ -130,7 +133,7 @@ def _count_messages(txn): """ txn.execute(sql, (like_clause, self.stream_ordering_day_ago)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction( @@ -138,14 +141,14 @@ def _count_messages(txn): ) async def count_daily_active_e2ee_rooms(self) -> int: - def _count(txn): + def _count(txn: LoggingTransaction) -> int: sql = """ SELECT COUNT(DISTINCT room_id) FROM events WHERE type = 'm.room.encrypted' AND stream_ordering > ? """ txn.execute(sql, (self.stream_ordering_day_ago,)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction( @@ -160,20 +163,20 @@ async def count_daily_messages(self) -> int: call to this function, it will return None. """ - def _count_messages(txn): + def _count_messages(txn: LoggingTransaction) -> int: sql = """ SELECT COUNT(*) FROM events WHERE type = 'm.room.message' AND stream_ordering > ? """ txn.execute(sql, (self.stream_ordering_day_ago,)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction("count_messages", _count_messages) async def count_daily_sent_messages(self) -> int: - def _count_messages(txn): + def _count_messages(txn: LoggingTransaction) -> int: # This is good enough as if you have silly characters in your own # hostname then that's your own fault. like_clause = "%:" + self.hs.hostname @@ -186,7 +189,7 @@ def _count_messages(txn): """ txn.execute(sql, (like_clause, self.stream_ordering_day_ago)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction( @@ -194,14 +197,14 @@ def _count_messages(txn): ) async def count_daily_active_rooms(self) -> int: - def _count(txn): + def _count(txn: LoggingTransaction) -> int: sql = """ SELECT COUNT(DISTINCT room_id) FROM events WHERE type = 'm.room.message' AND stream_ordering > ? """ txn.execute(sql, (self.stream_ordering_day_ago,)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return count return await self.db_pool.runInteraction("count_daily_active_rooms", _count) @@ -227,7 +230,7 @@ async def count_monthly_users(self) -> int: "count_monthly_users", self._count_users, thirty_days_ago ) - def _count_users(self, txn: Cursor, time_from: int) -> int: + def _count_users(self, txn: LoggingTransaction, time_from: int) -> int: """ Returns number of users seen in the past time_from period """ @@ -242,7 +245,7 @@ def _count_users(self, txn: Cursor, time_from: int) -> int: # Mypy knows that fetchone() might return None if there are no rows. # We know better: "SELECT COUNT(...) FROM ..." without any GROUP BY always # returns exactly one row. - (count,) = txn.fetchone() # type: ignore[misc] + (count,) = cast(Tuple[int], txn.fetchone()) return count async def count_r30_users(self) -> Dict[str, int]: @@ -256,7 +259,7 @@ async def count_r30_users(self) -> Dict[str, int]: A mapping of counts globally as well as broken out by platform. """ - def _count_r30_users(txn): + def _count_r30_users(txn: LoggingTransaction) -> Dict[str, int]: thirty_days_in_secs = 86400 * 30 now = int(self._clock.time()) thirty_days_ago_in_secs = now - thirty_days_in_secs @@ -321,7 +324,7 @@ def _count_r30_users(txn): txn.execute(sql, (thirty_days_ago_in_secs, thirty_days_ago_in_secs)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) results["all"] = count return results @@ -348,7 +351,7 @@ async def count_r30v2_users(self) -> Dict[str, int]: - "web" (any web application -- it's not possible to distinguish Element Web here) """ - def _count_r30v2_users(txn): + def _count_r30v2_users(txn: LoggingTransaction) -> Dict[str, int]: thirty_days_in_secs = 86400 * 30 now = int(self._clock.time()) sixty_days_ago_in_secs = now - 2 * thirty_days_in_secs @@ -445,11 +448,8 @@ def _count_r30v2_users(txn): thirty_days_in_secs * 1000, ), ) - row = txn.fetchone() - if row is None: - results["all"] = 0 - else: - results["all"] = row[0] + (count,) = cast(Tuple[int], txn.fetchone()) + results["all"] = count return results @@ -471,7 +471,7 @@ async def generate_user_daily_visits(self) -> None: Generates daily visit data for use in cohort/ retention analysis """ - def _generate_user_daily_visits(txn): + def _generate_user_daily_visits(txn: LoggingTransaction) -> None: logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() a_day_in_milliseconds = 24 * 60 * 60 * 1000 diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 4ed913e24879..0e2855fb446c 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -14,14 +14,18 @@ # limitations under the License. import abc import logging -from typing import TYPE_CHECKING, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Tuple, Union, cast from synapse.api.errors import StoreError from synapse.config.homeserver import ExperimentalConfig from synapse.push.baserules import list_with_base_rules from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker from synapse.storage._base import SQLBaseStore, db_to_json -from synapse.storage.database import DatabasePool, LoggingDatabaseConnection +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.pusher import PusherWorkerStore @@ -30,9 +34,12 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException from synapse.storage.util.id_generators import ( + AbstractStreamIdGenerator, AbstractStreamIdTracker, + IdGenerator, StreamIdGenerator, ) +from synapse.types import JsonDict from synapse.util import json_encoder from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches.stream_change_cache import StreamChangeCache @@ -57,7 +64,11 @@ def _is_experimental_rule_enabled( return True -def _load_rules(rawrules, enabled_map, experimental_config: ExperimentalConfig): +def _load_rules( + rawrules: List[JsonDict], + enabled_map: Dict[str, bool], + experimental_config: ExperimentalConfig, +) -> List[JsonDict]: ruleslist = [] for rawrule in rawrules: rule = dict(rawrule) @@ -137,7 +148,7 @@ def __init__( ) @abc.abstractmethod - def get_max_push_rules_stream_id(self): + def get_max_push_rules_stream_id(self) -> int: """Get the position of the push rules stream. Returns: @@ -146,7 +157,7 @@ def get_max_push_rules_stream_id(self): raise NotImplementedError() @cached(max_entries=5000) - async def get_push_rules_for_user(self, user_id): + async def get_push_rules_for_user(self, user_id: str) -> List[JsonDict]: rows = await self.db_pool.simple_select_list( table="push_rules", keyvalues={"user_name": user_id}, @@ -168,7 +179,7 @@ async def get_push_rules_for_user(self, user_id): return _load_rules(rows, enabled_map, self.hs.config.experimental) @cached(max_entries=5000) - async def get_push_rules_enabled_for_user(self, user_id) -> Dict[str, bool]: + async def get_push_rules_enabled_for_user(self, user_id: str) -> Dict[str, bool]: results = await self.db_pool.simple_select_list( table="push_rules_enable", keyvalues={"user_name": user_id}, @@ -184,13 +195,13 @@ async def have_push_rules_changed_for_user( return False else: - def have_push_rules_changed_txn(txn): + def have_push_rules_changed_txn(txn: LoggingTransaction) -> bool: sql = ( "SELECT COUNT(stream_id) FROM push_rules_stream" " WHERE user_id = ? AND ? < stream_id" ) txn.execute(sql, (user_id, last_id)) - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) return bool(count) return await self.db_pool.runInteraction( @@ -202,11 +213,13 @@ def have_push_rules_changed_txn(txn): list_name="user_ids", num_args=1, ) - async def bulk_get_push_rules(self, user_ids): + async def bulk_get_push_rules( + self, user_ids: Collection[str] + ) -> Dict[str, List[JsonDict]]: if not user_ids: return {} - results = {user_id: [] for user_id in user_ids} + results: Dict[str, List[JsonDict]] = {user_id: [] for user_id in user_ids} rows = await self.db_pool.simple_select_many_batch( table="push_rules", @@ -250,7 +263,7 @@ async def copy_push_rule_from_room_to_room( condition["pattern"] = new_room_id # Add the rule for the new room - await self.add_push_rule( + await self.add_push_rule( # type: ignore[attr-defined] user_id=user_id, rule_id=new_rule_id, priority_class=rule["priority_class"], @@ -286,11 +299,13 @@ async def copy_push_rules_from_room_to_room_for_user( list_name="user_ids", num_args=1, ) - async def bulk_get_push_rules_enabled(self, user_ids): + async def bulk_get_push_rules_enabled( + self, user_ids: Collection[str] + ) -> Dict[str, Dict[str, bool]]: if not user_ids: return {} - results = {user_id: {} for user_id in user_ids} + results: Dict[str, Dict[str, bool]] = {user_id: {} for user_id in user_ids} rows = await self.db_pool.simple_select_many_batch( table="push_rules_enable", @@ -306,7 +321,7 @@ async def bulk_get_push_rules_enabled(self, user_ids): async def get_all_push_rule_updates( self, instance_name: str, last_id: int, current_id: int, limit: int - ) -> Tuple[List[Tuple[int, tuple]], int, bool]: + ) -> Tuple[List[Tuple[int, Tuple[str]]], int, bool]: """Get updates for push_rules replication stream. Args: @@ -331,7 +346,9 @@ async def get_all_push_rule_updates( if last_id == current_id: return [], current_id, False - def get_all_push_rule_updates_txn(txn): + def get_all_push_rule_updates_txn( + txn: LoggingTransaction, + ) -> Tuple[List[Tuple[int, Tuple[str]]], int, bool]: sql = """ SELECT stream_id, user_id FROM push_rules_stream @@ -340,7 +357,10 @@ def get_all_push_rule_updates_txn(txn): LIMIT ? """ txn.execute(sql, (last_id, current_id, limit)) - updates = [(stream_id, (user_id,)) for stream_id, user_id in txn] + updates = cast( + List[Tuple[int, Tuple[str]]], + [(stream_id, (user_id,)) for stream_id, user_id in txn], + ) limited = False upper_bound = current_id @@ -356,15 +376,30 @@ def get_all_push_rule_updates_txn(txn): class PushRuleStore(PushRulesWorkerStore): + # Because we have write access, this will be a StreamIdGenerator + # (see PushRulesWorkerStore.__init__) + _push_rules_stream_id_gen: AbstractStreamIdGenerator + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") + self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") + async def add_push_rule( self, - user_id, - rule_id, - priority_class, - conditions, - actions, - before=None, - after=None, + user_id: str, + rule_id: str, + priority_class: int, + conditions: List[Dict[str, str]], + actions: List[Union[JsonDict, str]], + before: Optional[str] = None, + after: Optional[str] = None, ) -> None: conditions_json = json_encoder.encode(conditions) actions_json = json_encoder.encode(actions) @@ -400,17 +435,17 @@ async def add_push_rule( def _add_push_rule_relative_txn( self, - txn, - stream_id, - event_stream_ordering, - user_id, - rule_id, - priority_class, - conditions_json, - actions_json, - before, - after, - ): + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + user_id: str, + rule_id: str, + priority_class: int, + conditions_json: str, + actions_json: str, + before: str, + after: str, + ) -> None: # Lock the table since otherwise we'll have annoying races between the # SELECT here and the UPSERT below. self.database_engine.lock_table(txn, "push_rules") @@ -470,15 +505,15 @@ def _add_push_rule_relative_txn( def _add_push_rule_highest_priority_txn( self, - txn, - stream_id, - event_stream_ordering, - user_id, - rule_id, - priority_class, - conditions_json, - actions_json, - ): + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + user_id: str, + rule_id: str, + priority_class: int, + conditions_json: str, + actions_json: str, + ) -> None: # Lock the table since otherwise we'll have annoying races between the # SELECT here and the UPSERT below. self.database_engine.lock_table(txn, "push_rules") @@ -510,17 +545,17 @@ def _add_push_rule_highest_priority_txn( def _upsert_push_rule_txn( self, - txn, - stream_id, - event_stream_ordering, - user_id, - rule_id, - priority_class, - priority, - conditions_json, - actions_json, - update_stream=True, - ): + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + user_id: str, + rule_id: str, + priority_class: int, + priority: int, + conditions_json: str, + actions_json: str, + update_stream: bool = True, + ) -> None: """Specialised version of simple_upsert_txn that picks a push_rule_id using the _push_rule_id_gen if it needs to insert the rule. It assumes that the "push_rules" table is locked""" @@ -600,7 +635,11 @@ async def delete_push_rule(self, user_id: str, rule_id: str) -> None: rule_id: The rule_id of the rule to be deleted """ - def delete_push_rule_txn(txn, stream_id, event_stream_ordering): + def delete_push_rule_txn( + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + ) -> None: # we don't use simple_delete_one_txn because that would fail if the # user did not have a push_rule_enable row. self.db_pool.simple_delete_txn( @@ -661,14 +700,14 @@ async def set_push_rule_enabled( def _set_push_rule_enabled_txn( self, - txn, - stream_id, - event_stream_ordering, - user_id, - rule_id, - enabled, - is_default_rule, - ): + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + user_id: str, + rule_id: str, + enabled: bool, + is_default_rule: bool, + ) -> None: new_id = self._push_rules_enable_id_gen.get_next() if not is_default_rule: @@ -740,7 +779,11 @@ async def set_push_rule_actions( """ actions_json = json_encoder.encode(actions) - def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering): + def set_push_rule_actions_txn( + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + ) -> None: if is_default_rule: # Add a dummy rule to the rules table with the user specified # actions. @@ -794,8 +837,15 @@ def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering): ) def _insert_push_rules_update_txn( - self, txn, stream_id, event_stream_ordering, user_id, rule_id, op, data=None - ): + self, + txn: LoggingTransaction, + stream_id: int, + event_stream_ordering: int, + user_id: str, + rule_id: str, + op: str, + data: Optional[JsonDict] = None, + ) -> None: values = { "stream_id": stream_id, "event_stream_ordering": event_stream_ordering, @@ -814,5 +864,5 @@ def _insert_push_rules_update_txn( self.push_rules_stream_cache.entity_has_changed, user_id, stream_id ) - def get_max_push_rules_stream_id(self): + def get_max_push_rules_stream_id(self) -> int: return self._push_rules_stream_id_gen.get_current_token() diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 48e83592e728..608d40dfa164 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -37,7 +37,12 @@ wrap_as_background_process, ) from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause -from synapse.storage.database import DatabasePool, LoggingDatabaseConnection +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.engines import Sqlite3Engine from synapse.storage.roommember import ( @@ -46,7 +51,7 @@ ProfileInfo, RoomsForUser, ) -from synapse.types import PersistedEventPosition, get_domain_from_id +from synapse.types import JsonDict, PersistedEventPosition, StateMap, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.caches import intern_string from synapse.util.caches.descriptors import _CacheContext, cached, cachedList @@ -115,7 +120,7 @@ def __init__( ) @wrap_as_background_process("_count_known_servers") - async def _count_known_servers(self): + async def _count_known_servers(self) -> int: """ Count the servers that this server knows about. @@ -123,7 +128,7 @@ async def _count_known_servers(self): `synapse_federation_known_servers` LaterGauge to collect. """ - def _transact(txn): + def _transact(txn: LoggingTransaction) -> int: if isinstance(self.database_engine, Sqlite3Engine): query = """ SELECT COUNT(DISTINCT substr(out.user_id, pos+1)) @@ -150,7 +155,9 @@ def _transact(txn): self._known_servers_count = max([count, 1]) return self._known_servers_count - def _check_safe_current_state_events_membership_updated_txn(self, txn): + def _check_safe_current_state_events_membership_updated_txn( + self, txn: LoggingTransaction + ) -> None: """Checks if it is safe to assume the new current_state_events membership column is up to date """ @@ -182,7 +189,7 @@ async def get_users_in_room(self, room_id: str) -> List[str]: "get_users_in_room", self.get_users_in_room_txn, room_id ) - def get_users_in_room_txn(self, txn, room_id: str) -> List[str]: + def get_users_in_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[str]: # If we can assume current_state_events.membership is up to date # then we can avoid a join, which is a Very Good Thing given how # frequently this function gets called. @@ -222,7 +229,9 @@ async def get_users_in_room_with_profiles( A mapping from user ID to ProfileInfo. """ - def _get_users_in_room_with_profiles(txn) -> Dict[str, ProfileInfo]: + def _get_users_in_room_with_profiles( + txn: LoggingTransaction, + ) -> Dict[str, ProfileInfo]: sql = """ SELECT state_key, display_name, avatar_url FROM room_memberships as m INNER JOIN current_state_events as c @@ -250,7 +259,9 @@ async def get_room_summary(self, room_id: str) -> Dict[str, MemberSummary]: dict of membership states, pointing to a MemberSummary named tuple. """ - def _get_room_summary_txn(txn): + def _get_room_summary_txn( + txn: LoggingTransaction, + ) -> Dict[str, MemberSummary]: # first get counts. # We do this all in one transaction to keep the cache small. # FIXME: get rid of this when we have room_stats @@ -279,7 +290,7 @@ def _get_room_summary_txn(txn): """ txn.execute(sql, (room_id,)) - res = {} + res: Dict[str, MemberSummary] = {} for count, membership in txn: res.setdefault(membership, MemberSummary([], count)) @@ -400,7 +411,7 @@ async def get_rooms_for_local_user_where_membership_is( def _get_rooms_for_local_user_where_membership_is_txn( self, - txn, + txn: LoggingTransaction, user_id: str, membership_list: List[str], ) -> List[RoomsForUser]: @@ -488,7 +499,7 @@ async def get_rooms_for_user_with_stream_ordering( ) def _get_rooms_for_user_with_stream_ordering_txn( - self, txn, user_id: str + self, txn: LoggingTransaction, user_id: str ) -> FrozenSet[GetRoomsForUserWithStreamOrdering]: # We use `current_state_events` here and not `local_current_membership` # as a) this gets called with remote users and b) this only gets called @@ -542,7 +553,7 @@ async def get_rooms_for_users_with_stream_ordering( ) def _get_rooms_for_users_with_stream_ordering_txn( - self, txn, user_ids: Collection[str] + self, txn: LoggingTransaction, user_ids: Collection[str] ) -> Dict[str, FrozenSet[GetRoomsForUserWithStreamOrdering]]: clause, args = make_in_list_sql_clause( @@ -575,7 +586,9 @@ def _get_rooms_for_users_with_stream_ordering_txn( txn.execute(sql, [Membership.JOIN] + args) - result = {user_id: set() for user_id in user_ids} + result: Dict[str, Set[GetRoomsForUserWithStreamOrdering]] = { + user_id: set() for user_id in user_ids + } for user_id, room_id, instance, stream_id in txn: result[user_id].add( GetRoomsForUserWithStreamOrdering( @@ -595,7 +608,9 @@ async def get_users_server_still_shares_room_with( if not user_ids: return set() - def _get_users_server_still_shares_room_with_txn(txn): + def _get_users_server_still_shares_room_with_txn( + txn: LoggingTransaction, + ) -> Set[str]: sql = """ SELECT state_key FROM current_state_events WHERE @@ -657,7 +672,7 @@ async def get_users_who_share_room_with_user( async def get_joined_users_from_context( self, event: EventBase, context: EventContext ) -> Dict[str, ProfileInfo]: - state_group = context.state_group + state_group: Union[object, int] = context.state_group if not state_group: # If state_group is None it means it has yet to be assigned a # state group, i.e. we need to make sure that calls with a state_group @@ -666,14 +681,16 @@ async def get_joined_users_from_context( state_group = object() current_state_ids = await context.get_current_state_ids() + assert current_state_ids is not None + assert state_group is not None return await self._get_joined_users_from_context( event.room_id, state_group, current_state_ids, event=event, context=context ) async def get_joined_users_from_state( - self, room_id, state_entry + self, room_id: str, state_entry: "_StateCacheEntry" ) -> Dict[str, ProfileInfo]: - state_group = state_entry.state_group + state_group: Union[object, int] = state_entry.state_group if not state_group: # If state_group is None it means it has yet to be assigned a # state group, i.e. we need to make sure that calls with a state_group @@ -681,6 +698,7 @@ async def get_joined_users_from_state( # To do this we set the state_group to a new object as object() != object() state_group = object() + assert state_group is not None with Measure(self._clock, "get_joined_users_from_state"): return await self._get_joined_users_from_context( room_id, state_group, state_entry.state, context=state_entry @@ -689,12 +707,12 @@ async def get_joined_users_from_state( @cached(num_args=2, cache_context=True, iterable=True, max_entries=100000) async def _get_joined_users_from_context( self, - room_id, - state_group, - current_state_ids, - cache_context, - event=None, - context=None, + room_id: str, + state_group: Union[object, int], + current_state_ids: StateMap[str], + cache_context: _CacheContext, + event: Optional[EventBase] = None, + context: Optional[Union[EventContext, "_StateCacheEntry"]] = None, ) -> Dict[str, ProfileInfo]: # We don't use `state_group`, it's there so that we can cache based # on it. However, it's important that it's never None, since two current_states @@ -765,14 +783,18 @@ async def _get_joined_users_from_context( return users_in_room @cached(max_entries=10000) - def _get_joined_profile_from_event_id(self, event_id): + def _get_joined_profile_from_event_id( + self, event_id: str + ) -> Optional[Tuple[str, ProfileInfo]]: raise NotImplementedError() @cachedList( cached_method_name="_get_joined_profile_from_event_id", list_name="event_ids", ) - async def _get_joined_profiles_from_event_ids(self, event_ids: Iterable[str]): + async def _get_joined_profiles_from_event_ids( + self, event_ids: Iterable[str] + ) -> Dict[str, Optional[Tuple[str, ProfileInfo]]]: """For given set of member event_ids check if they point to a join event and if so return the associated user and profile info. @@ -780,8 +802,7 @@ async def _get_joined_profiles_from_event_ids(self, event_ids: Iterable[str]): event_ids: The member event IDs to lookup Returns: - dict[str, Tuple[str, ProfileInfo]|None]: Map from event ID - to `user_id` and ProfileInfo (or None if not join event). + Map from event ID to `user_id` and ProfileInfo (or None if not join event). """ rows = await self.db_pool.simple_select_many_batch( @@ -847,8 +868,10 @@ async def _check_host_room_membership( return True - async def get_joined_hosts(self, room_id: str, state_entry): - state_group = state_entry.state_group + async def get_joined_hosts( + self, room_id: str, state_entry: "_StateCacheEntry" + ) -> FrozenSet[str]: + state_group: Union[object, int] = state_entry.state_group if not state_group: # If state_group is None it means it has yet to be assigned a # state group, i.e. we need to make sure that calls with a state_group @@ -856,6 +879,7 @@ async def get_joined_hosts(self, room_id: str, state_entry): # To do this we set the state_group to a new object as object() != object() state_group = object() + assert state_group is not None with Measure(self._clock, "get_joined_hosts"): return await self._get_joined_hosts( room_id, state_group, state_entry=state_entry @@ -863,7 +887,10 @@ async def get_joined_hosts(self, room_id: str, state_entry): @cached(num_args=2, max_entries=10000, iterable=True) async def _get_joined_hosts( - self, room_id: str, state_group: int, state_entry: "_StateCacheEntry" + self, + room_id: str, + state_group: Union[object, int], + state_entry: "_StateCacheEntry", ) -> FrozenSet[str]: # We don't use `state_group`, it's there so that we can cache based on # it. However, its important that its never None, since two @@ -881,7 +908,7 @@ async def _get_joined_hosts( # `get_joined_hosts` is called with the "current" state group for the # room, and so consecutive calls will be for consecutive state groups # which point to the previous state group. - cache = await self._get_joined_hosts_cache(room_id) + cache = await self._get_joined_hosts_cache(room_id) # type: ignore[misc] # If the state group in the cache matches, we already have the data we need. if state_entry.state_group == cache.state_group: @@ -897,6 +924,7 @@ async def _get_joined_hosts( elif state_entry.prev_group == cache.state_group: # The cached work is for the previous state group, so we work out # the delta. + assert state_entry.delta_ids is not None for (typ, state_key), event_id in state_entry.delta_ids.items(): if typ != EventTypes.Member: continue @@ -942,7 +970,7 @@ async def did_forget(self, user_id: str, room_id: str) -> bool: Returns False if they have since re-joined.""" - def f(txn): + def f(txn: LoggingTransaction) -> int: sql = ( "SELECT" " COUNT(*)" @@ -973,7 +1001,7 @@ async def get_forgotten_rooms_for_user(self, user_id: str) -> Set[str]: The forgotten rooms. """ - def _get_forgotten_rooms_for_user_txn(txn): + def _get_forgotten_rooms_for_user_txn(txn: LoggingTransaction) -> Set[str]: # This is a slightly convoluted query that first looks up all rooms # that the user has forgotten in the past, then rechecks that list # to see if any have subsequently been updated. This is done so that @@ -1076,7 +1104,9 @@ async def is_local_host_in_room_ignoring_users( clause, ) - def _is_local_host_in_room_ignoring_users_txn(txn): + def _is_local_host_in_room_ignoring_users_txn( + txn: LoggingTransaction, + ) -> bool: txn.execute(sql, (room_id, Membership.JOIN, *args)) return bool(txn.fetchone()) @@ -1110,15 +1140,17 @@ def __init__( where_clause="forgotten = 1", ) - async def _background_add_membership_profile(self, progress, batch_size): + async def _background_add_membership_profile( + self, progress: JsonDict, batch_size: int + ) -> int: target_min_stream_id = progress.get( - "target_min_stream_id_inclusive", self._min_stream_order_on_start + "target_min_stream_id_inclusive", self._min_stream_order_on_start # type: ignore[attr-defined] ) max_stream_id = progress.get( - "max_stream_id_exclusive", self._stream_order_on_start + 1 + "max_stream_id_exclusive", self._stream_order_on_start + 1 # type: ignore[attr-defined] ) - def add_membership_profile_txn(txn): + def add_membership_profile_txn(txn: LoggingTransaction) -> int: sql = """ SELECT stream_ordering, event_id, events.room_id, event_json.json FROM events @@ -1182,13 +1214,17 @@ def add_membership_profile_txn(txn): return result - async def _background_current_state_membership(self, progress, batch_size): + async def _background_current_state_membership( + self, progress: JsonDict, batch_size: int + ) -> int: """Update the new membership column on current_state_events. This works by iterating over all rooms in alphebetical order. """ - def _background_current_state_membership_txn(txn, last_processed_room): + def _background_current_state_membership_txn( + txn: LoggingTransaction, last_processed_room: str + ) -> Tuple[int, bool]: processed = 0 while processed < batch_size: txn.execute( @@ -1242,7 +1278,11 @@ def _background_current_state_membership_txn(txn, last_processed_room): return row_count -class RoomMemberStore(RoomMemberWorkerStore, RoomMemberBackgroundUpdateStore): +class RoomMemberStore( + RoomMemberWorkerStore, + RoomMemberBackgroundUpdateStore, + CacheInvalidationWorkerStore, +): def __init__( self, database: DatabasePool, @@ -1254,7 +1294,7 @@ def __init__( async def forget(self, user_id: str, room_id: str) -> None: """Indicate that user_id wishes to discard history for room_id.""" - def f(txn): + def f(txn: LoggingTransaction) -> None: sql = ( "UPDATE" " room_memberships" @@ -1288,5 +1328,5 @@ class _JoinedHostsCache: # equal to anything else). state_group: Union[object, int] = attr.Factory(object) - def __len__(self): + def __len__(self) -> int: return sum(len(v) for v in self.hosts_to_joined_users.values()) From 5331fb5b478789a3ffaaeddb58f8d1cefd42a9eb Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 17 May 2022 17:06:45 +0100 Subject: [PATCH 208/263] allow `on_invalidate=None` in `@cached` methods (#12769) --- changelog.d/12769.misc | 1 + scripts-dev/mypy_synapse_plugin.py | 25 +++++++++++++------- synapse/storage/databases/main/roommember.py | 3 ++- 3 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 changelog.d/12769.misc diff --git a/changelog.d/12769.misc b/changelog.d/12769.misc new file mode 100644 index 000000000000..27bd53abe376 --- /dev/null +++ b/changelog.d/12769.misc @@ -0,0 +1 @@ +Tweak the mypy plugin so that `@cached` can accept `on_invalidate=None`. diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index c775865212ee..d08517a95382 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -21,7 +21,7 @@ from mypy.nodes import ARG_NAMED_OPT from mypy.plugin import MethodSigContext, Plugin from mypy.typeops import bind_self -from mypy.types import CallableType, NoneType +from mypy.types import CallableType, NoneType, UnionType class SynapsePlugin(Plugin): @@ -72,13 +72,20 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType: # Third, we add an optional "on_invalidate" argument. # - # This is a callable which accepts no input and returns nothing. - calltyp = CallableType( - arg_types=[], - arg_kinds=[], - arg_names=[], - ret_type=NoneType(), - fallback=ctx.api.named_generic_type("builtins.function", []), + # This is a either + # - a callable which accepts no input and returns nothing, or + # - None. + calltyp = UnionType( + [ + NoneType(), + CallableType( + arg_types=[], + arg_kinds=[], + arg_names=[], + ret_type=NoneType(), + fallback=ctx.api.named_generic_type("builtins.function", []), + ), + ] ) arg_types.append(calltyp) @@ -95,7 +102,7 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType: def plugin(version: str) -> Type[SynapsePlugin]: - # This is the entry point of the plugin, and let's us deal with the fact + # This is the entry point of the plugin, and lets us deal with the fact # that the mypy plugin interface is *not* stable by looking at the version # string. # diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 608d40dfa164..cc528fcf2dae 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -15,6 +15,7 @@ import logging from typing import ( TYPE_CHECKING, + Callable, Collection, Dict, FrozenSet, @@ -634,7 +635,7 @@ def _get_users_server_still_shares_room_with_txn( ) async def get_rooms_for_user( - self, user_id: str, on_invalidate=None + self, user_id: str, on_invalidate: Optional[Callable[[], None]] = None ) -> FrozenSet[str]: """Returns a set of room_ids the user is currently joined to. From 182ca78a12c4ae0f37726d43d5e592d669d99ee1 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Tue, 17 May 2022 19:01:06 +0200 Subject: [PATCH 209/263] Delete events from federation_inbound_events_staging table on purge (#12770) --- changelog.d/12770.bugfix | 1 + synapse/storage/databases/main/purge_events.py | 1 + tests/rest/admin/test_room.py | 1 + 3 files changed, 3 insertions(+) create mode 100644 changelog.d/12770.bugfix diff --git a/changelog.d/12770.bugfix b/changelog.d/12770.bugfix new file mode 100644 index 000000000000..a958f9a16ba3 --- /dev/null +++ b/changelog.d/12770.bugfix @@ -0,0 +1 @@ +Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API. diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index 38ba91af4c47..c94d5f9f812b 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -417,6 +417,7 @@ def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]: "room_account_data", "room_tags", "local_current_membership", + "federation_inbound_events_staging", ): logger.info("[purge] removing %s from %s", room_id, table) txn.execute("DELETE FROM %s WHERE room_id=?" % (table,), (room_id,)) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 95282f078e77..608d3f2dc36a 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2489,4 +2489,5 @@ def _block_room(self, room_id: str) -> None: "room_tags", # "state_groups", # Current impl leaves orphaned state groups around. "state_groups_state", + "federation_inbound_events_staging", ] From 0d17357fcdded3fa3f8a37db7b6b9aa0402a10ed Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 17 May 2022 19:05:53 +0100 Subject: [PATCH 210/263] Suggest using docker when testing against postgres (#12765) Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com> --- changelog.d/12765.doc | 1 + docs/development/contributing_guide.md | 31 +++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12765.doc diff --git a/changelog.d/12765.doc b/changelog.d/12765.doc new file mode 100644 index 000000000000..277b037d6b03 --- /dev/null +++ b/changelog.d/12765.doc @@ -0,0 +1 @@ +Recommend using docker to run tests against postgres. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index d356c72bf780..f55a1fbb9002 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -206,7 +206,32 @@ This means that we need to run our unit tests against PostgreSQL too. Our CI doe this automatically for pull requests and release candidates, but it's sometimes useful to reproduce this locally. -To do so, [configure Postgres](../postgres.md) and run `trial` with the +#### Using Docker + +The easiest way to do so is to run Postgres via a docker container. In one +terminal: + +```shell +docker run --rm -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres -e POSTGRES_DB=postgress -p 5432:5432 postgres:14 +``` + +If you see an error like + +``` +docker: Error response from daemon: driver failed programming external connectivity on endpoint nice_ride (b57bbe2e251b70015518d00c9981e8cb8346b5c785250341a6c53e3c899875f1): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use. +``` + +then something is already bound to port 5432. You're probably already running postgres locally. + +Once you have a postgres server running, invoke `trial` in a second terminal: + +```shell +SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_HOST=127.0.0.1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_POSTGRES_PASSWORD=mysecretpassword poetry run trial tests +```` + +#### Using an existing Postgres installation + +If you have postgres already installed on your system, you can run `trial` with the following environment variables matching your configuration: - `SYNAPSE_POSTGRES` to anything nonempty @@ -229,8 +254,8 @@ You don't need to specify the host, user, port or password if your Postgres server is set to authenticate you over the UNIX socket (i.e. if the `psql` command works without further arguments). -Your Postgres account needs to be able to create databases. - +Your Postgres account needs to be able to create databases; see the postgres +docs for [`ALTER ROLE`](https://www.postgresql.org/docs/current/sql-alterrole.html). ## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)). From 37935b5183ab3cbee2f80359d80b1ff2176428f0 Mon Sep 17 00:00:00 2001 From: Adam <65660516+ajr0d@users.noreply.github.com> Date: Wed, 18 May 2022 10:37:48 +0100 Subject: [PATCH 211/263] Move methods that call add_push_rule to PushRuleStore (#12772) Signed-off-by: Adam Roddick --- changelog.d/12772.misc | 1 + synapse/storage/databases/main/push_rule.py | 102 ++++++++++---------- 2 files changed, 52 insertions(+), 51 deletions(-) create mode 100644 changelog.d/12772.misc diff --git a/changelog.d/12772.misc b/changelog.d/12772.misc new file mode 100644 index 000000000000..da66f376fe8e --- /dev/null +++ b/changelog.d/12772.misc @@ -0,0 +1 @@ +Move methods that call `add_push_rule` to the `PushRuleStore` class. diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 0e2855fb446c..ad67901cc1ac 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -243,57 +243,6 @@ async def bulk_get_push_rules( return results - async def copy_push_rule_from_room_to_room( - self, new_room_id: str, user_id: str, rule: dict - ) -> None: - """Copy a single push rule from one room to another for a specific user. - - Args: - new_room_id: ID of the new room. - user_id : ID of user the push rule belongs to. - rule: A push rule. - """ - # Create new rule id - rule_id_scope = "/".join(rule["rule_id"].split("/")[:-1]) - new_rule_id = rule_id_scope + "/" + new_room_id - - # Change room id in each condition - for condition in rule.get("conditions", []): - if condition.get("key") == "room_id": - condition["pattern"] = new_room_id - - # Add the rule for the new room - await self.add_push_rule( # type: ignore[attr-defined] - user_id=user_id, - rule_id=new_rule_id, - priority_class=rule["priority_class"], - conditions=rule["conditions"], - actions=rule["actions"], - ) - - async def copy_push_rules_from_room_to_room_for_user( - self, old_room_id: str, new_room_id: str, user_id: str - ) -> None: - """Copy all of the push rules from one room to another for a specific - user. - - Args: - old_room_id: ID of the old room. - new_room_id: ID of the new room. - user_id: ID of user to copy push rules for. - """ - # Retrieve push rules for this user - user_push_rules = await self.get_push_rules_for_user(user_id) - - # Get rules relating to the old room and copy them to the new room - for rule in user_push_rules: - conditions = rule.get("conditions", []) - if any( - (c.get("key") == "room_id" and c.get("pattern") == old_room_id) - for c in conditions - ): - await self.copy_push_rule_from_room_to_room(new_room_id, user_id, rule) - @cachedList( cached_method_name="get_push_rules_enabled_for_user", list_name="user_ids", @@ -866,3 +815,54 @@ def _insert_push_rules_update_txn( def get_max_push_rules_stream_id(self) -> int: return self._push_rules_stream_id_gen.get_current_token() + + async def copy_push_rule_from_room_to_room( + self, new_room_id: str, user_id: str, rule: dict + ) -> None: + """Copy a single push rule from one room to another for a specific user. + + Args: + new_room_id: ID of the new room. + user_id : ID of user the push rule belongs to. + rule: A push rule. + """ + # Create new rule id + rule_id_scope = "/".join(rule["rule_id"].split("/")[:-1]) + new_rule_id = rule_id_scope + "/" + new_room_id + + # Change room id in each condition + for condition in rule.get("conditions", []): + if condition.get("key") == "room_id": + condition["pattern"] = new_room_id + + # Add the rule for the new room + await self.add_push_rule( + user_id=user_id, + rule_id=new_rule_id, + priority_class=rule["priority_class"], + conditions=rule["conditions"], + actions=rule["actions"], + ) + + async def copy_push_rules_from_room_to_room_for_user( + self, old_room_id: str, new_room_id: str, user_id: str + ) -> None: + """Copy all of the push rules from one room to another for a specific + user. + + Args: + old_room_id: ID of the old room. + new_room_id: ID of the new room. + user_id: ID of user to copy push rules for. + """ + # Retrieve push rules for this user + user_push_rules = await self.get_push_rules_for_user(user_id) + + # Get rules relating to the old room and copy them to the new room + for rule in user_push_rules: + conditions = rule.get("conditions", []) + if any( + (c.get("key") == "room_id" and c.get("pattern") == old_room_id) + for c in conditions + ): + await self.copy_push_rule_from_room_to_room(new_room_id, user_id, rule) From 8afb7b55d0527f8c6af7690b162ebaabe9b5d9f5 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 18 May 2022 06:19:30 -0400 Subject: [PATCH 212/263] Make handling of federation Authorization header (more) compliant with RFC7230 (#12774) The main differences are: - values with delimiters (such as colons) should be quoted, so always quote the origin, since it could contain a colon followed by a port number - should allow more than one space after "X-Matrix" - quoted values with backslash-escaped characters should be unescaped - names should be case insensitive --- changelog.d/12774.misc | 1 + synapse/federation/transport/server/_base.py | 8 +++-- synapse/http/matrixfederationclient.py | 2 +- .../federation/transport/server/test__base.py | 29 ++++++++++++++++++- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12774.misc diff --git a/changelog.d/12774.misc b/changelog.d/12774.misc new file mode 100644 index 000000000000..8651f2e0e062 --- /dev/null +++ b/changelog.d/12774.misc @@ -0,0 +1 @@ +Make handling of federation Authorization header (more) compliant with RFC7230. diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 103861644a70..84100a5a5257 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -169,14 +169,16 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str """ try: header_str = header_bytes.decode("utf-8") - params = header_str.split(" ")[1].split(",") + params = re.split(" +", header_str)[1].split(",") param_dict: Dict[str, str] = { - k: v for k, v in [param.split("=", maxsplit=1) for param in params] + k.lower(): v for k, v in [param.split("=", maxsplit=1) for param in params] } def strip_quotes(value: str) -> str: if value.startswith('"'): - return value[1:-1] + return re.sub( + "\\\\(.)", lambda matchobj: matchobj.group(1), value[1:-1] + ) else: return value diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 725b5c33b8c5..0b9475debdb1 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -747,7 +747,7 @@ def build_auth_headers( for key, sig in request["signatures"][self.server_name].items(): auth_headers.append( ( - 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' + 'X-Matrix origin="%s",key="%s",sig="%s",destination="%s"' % ( self.server_name, key, diff --git a/tests/federation/transport/server/test__base.py b/tests/federation/transport/server/test__base.py index ac3695a8ccab..e63885c1c9d8 100644 --- a/tests/federation/transport/server/test__base.py +++ b/tests/federation/transport/server/test__base.py @@ -17,7 +17,7 @@ from synapse.api.errors import Codes from synapse.federation.transport.server import BaseFederationServlet -from synapse.federation.transport.server._base import Authenticator +from synapse.federation.transport.server._base import Authenticator, _parse_auth_header from synapse.http.server import JsonResource, cancellable from synapse.server import HomeServer from synapse.types import JsonDict @@ -112,3 +112,30 @@ def test_uncancellable_disconnect(self) -> None: expect_cancellation=False, expected_body={"result": True}, ) + + +class BaseFederationAuthorizationTests(unittest.TestCase): + def test_authorization_header(self) -> None: + """Tests that the Authorization header is parsed correctly.""" + + # test a "normal" Authorization header + self.assertEqual( + _parse_auth_header( + b'X-Matrix origin=foo,key="ed25519:1",sig="sig",destination="bar"' + ), + ("foo", "ed25519:1", "sig", "bar"), + ) + # test an Authorization with extra spaces, upper-case names, and escaped + # characters + self.assertEqual( + _parse_auth_header( + b'X-Matrix ORIGIN=foo,KEY="ed25\\519:1",SIG="sig",destination="bar"' + ), + ("foo", "ed25519:1", "sig", "bar"), + ) + self.assertEqual( + _parse_auth_header( + b'X-Matrix origin=foo,key="ed25519:1",sig="sig",destination="bar",extra_field=ignored' + ), + ("foo", "ed25519:1", "sig", "bar"), + ) From d4713d3e335b21d12284ddd8ebd00e38abcfd521 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 18 May 2022 11:28:14 +0100 Subject: [PATCH 213/263] Discard null-containing strings before updating the user directory (#12762) --- changelog.d/12762.misc | 1 + synapse/rest/client/room.py | 4 +-- synapse/storage/databases/main/events.py | 4 +-- .../storage/databases/main/user_directory.py | 9 +++--- synapse/util/stringutils.py | 10 ++++++- tests/handlers/test_user_directory.py | 28 +++++++++++++++++++ 6 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 changelog.d/12762.misc diff --git a/changelog.d/12762.misc b/changelog.d/12762.misc new file mode 100644 index 000000000000..990fb6fe74eb --- /dev/null +++ b/changelog.d/12762.misc @@ -0,0 +1 @@ +Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 4b8bfbffcb36..5a2361a2e691 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -109,10 +109,10 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() def register(self, http_server: HttpServer) -> None: - # /room/$roomid/state/$eventtype + # /rooms/$roomid/state/$eventtype no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" - # /room/$roomid/state/$eventtype/$statekey + # /rooms/$roomid/state/$eventtype/$statekey state_key = ( "/rooms/(?P[^/]*)/state/" "(?P[^/]*)/(?P[^/]*)$" diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 42d484dc98d9..0df8ff53957a 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -52,6 +52,7 @@ from synapse.types import JsonDict, StateMap, get_domain_from_id from synapse.util import json_encoder from synapse.util.iterutils import batch_iter, sorted_topologically +from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: from synapse.server import HomeServer @@ -1728,9 +1729,6 @@ def _store_room_members_txn( not affect the current local state. """ - def non_null_str_or_none(val: Any) -> Optional[str]: - return val if isinstance(val, str) and "\u0000" not in val else None - self.db_pool.simple_insert_many_txn( txn, table="room_memberships", diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index df772d472102..028db69af301 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -29,6 +29,7 @@ from typing_extensions import TypedDict from synapse.api.errors import StoreError +from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: from synapse.server import HomeServer @@ -469,11 +470,9 @@ async def update_profile_in_user_dir( """ Update or add a user's profile in the user directory. """ - # If the display name or avatar URL are unexpected types, overwrite them. - if not isinstance(display_name, str): - display_name = None - if not isinstance(avatar_url, str): - avatar_url = None + # If the display name or avatar URL are unexpected types, replace with None. + display_name = non_null_str_or_none(display_name) + avatar_url = non_null_str_or_none(avatar_url) def _update_profile_in_user_dir_txn(txn: LoggingTransaction) -> None: self.db_pool.simple_upsert_txn( diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index b26546aecdb7..27a363d7e516 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -16,7 +16,7 @@ import re import secrets import string -from typing import Iterable, Optional, Tuple +from typing import Any, Iterable, Optional, Tuple from netaddr import valid_ipv6 @@ -247,3 +247,11 @@ def base62_encode(num: int, minwidth: int = 1) -> str: # pad to minimum width pad = "0" * (minwidth - len(res)) return pad + res + + +def non_null_str_or_none(val: Any) -> Optional[str]: + """Check that the arg is a string containing no null (U+0000) codepoints. + + If so, returns the given string unmodified; otherwise, returns None. + """ + return val if isinstance(val, str) and "\u0000" not in val else None diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index 96e2e3039ba8..4d658d29cab5 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -1007,6 +1007,34 @@ def test_local_user_leaving_room_remains_in_user_directory(self) -> None: self.assertEqual(in_public, {(bob, room1), (bob, room2)}) self.assertEqual(in_private, set()) + def test_ignore_display_names_with_null_codepoints(self) -> None: + MXC_DUMMY = "mxc://dummy" + + # Alice creates a public room. + alice = self.register_user("alice", "pass") + + # Alice has a user directory entry to start with. + self.assertIn( + alice, + self.get_success(self.user_dir_helper.get_profiles_in_user_directory()), + ) + + # Alice changes her name to include a null codepoint. + self.get_success( + self.hs.get_user_directory_handler().handle_local_profile_change( + alice, + ProfileInfo( + display_name="abcd\u0000efgh", + avatar_url=MXC_DUMMY, + ), + ) + ) + # Alice's profile should be updated with the new avatar, but no display name. + self.assertEqual( + self.get_success(self.user_dir_helper.get_profiles_in_user_directory()), + {alice: ProfileInfo(display_name=None, avatar_url=MXC_DUMMY)}, + ) + class TestUserDirSearchDisabled(unittest.HomeserverTestCase): servlets = [ From c22314c4e8a3e9637810d78508bfe15dcdfe50b6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 18 May 2022 11:28:14 +0100 Subject: [PATCH 214/263] Discard null-containing strings before updating the user directory (#12762) --- changelog.d/12762.misc | 1 + synapse/rest/client/room.py | 4 +-- synapse/storage/databases/main/events.py | 4 +-- .../storage/databases/main/user_directory.py | 9 +++--- synapse/util/stringutils.py | 10 ++++++- tests/handlers/test_user_directory.py | 28 +++++++++++++++++++ 6 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 changelog.d/12762.misc diff --git a/changelog.d/12762.misc b/changelog.d/12762.misc new file mode 100644 index 000000000000..990fb6fe74eb --- /dev/null +++ b/changelog.d/12762.misc @@ -0,0 +1 @@ +Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 906fe09e9713..12fed856a8b3 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -109,10 +109,10 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() def register(self, http_server: HttpServer) -> None: - # /room/$roomid/state/$eventtype + # /rooms/$roomid/state/$eventtype no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" - # /room/$roomid/state/$eventtype/$statekey + # /rooms/$roomid/state/$eventtype/$statekey state_key = ( "/rooms/(?P[^/]*)/state/" "(?P[^/]*)/(?P[^/]*)$" diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index ed29a0a5e2db..2c86a870cfcf 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -53,6 +53,7 @@ from synapse.types import StateMap, get_domain_from_id from synapse.util import json_encoder from synapse.util.iterutils import batch_iter, sorted_topologically +from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: from synapse.server import HomeServer @@ -1737,9 +1738,6 @@ def _store_room_members_txn( not affect the current local state. """ - def non_null_str_or_none(val: Any) -> Optional[str]: - return val if isinstance(val, str) and "\u0000" not in val else None - self.db_pool.simple_insert_many_txn( txn, table="room_memberships", diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index df772d472102..028db69af301 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -29,6 +29,7 @@ from typing_extensions import TypedDict from synapse.api.errors import StoreError +from synapse.util.stringutils import non_null_str_or_none if TYPE_CHECKING: from synapse.server import HomeServer @@ -469,11 +470,9 @@ async def update_profile_in_user_dir( """ Update or add a user's profile in the user directory. """ - # If the display name or avatar URL are unexpected types, overwrite them. - if not isinstance(display_name, str): - display_name = None - if not isinstance(avatar_url, str): - avatar_url = None + # If the display name or avatar URL are unexpected types, replace with None. + display_name = non_null_str_or_none(display_name) + avatar_url = non_null_str_or_none(avatar_url) def _update_profile_in_user_dir_txn(txn: LoggingTransaction) -> None: self.db_pool.simple_upsert_txn( diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index b26546aecdb7..27a363d7e516 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -16,7 +16,7 @@ import re import secrets import string -from typing import Iterable, Optional, Tuple +from typing import Any, Iterable, Optional, Tuple from netaddr import valid_ipv6 @@ -247,3 +247,11 @@ def base62_encode(num: int, minwidth: int = 1) -> str: # pad to minimum width pad = "0" * (minwidth - len(res)) return pad + res + + +def non_null_str_or_none(val: Any) -> Optional[str]: + """Check that the arg is a string containing no null (U+0000) codepoints. + + If so, returns the given string unmodified; otherwise, returns None. + """ + return val if isinstance(val, str) and "\u0000" not in val else None diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index 96e2e3039ba8..4d658d29cab5 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -1007,6 +1007,34 @@ def test_local_user_leaving_room_remains_in_user_directory(self) -> None: self.assertEqual(in_public, {(bob, room1), (bob, room2)}) self.assertEqual(in_private, set()) + def test_ignore_display_names_with_null_codepoints(self) -> None: + MXC_DUMMY = "mxc://dummy" + + # Alice creates a public room. + alice = self.register_user("alice", "pass") + + # Alice has a user directory entry to start with. + self.assertIn( + alice, + self.get_success(self.user_dir_helper.get_profiles_in_user_directory()), + ) + + # Alice changes her name to include a null codepoint. + self.get_success( + self.hs.get_user_directory_handler().handle_local_profile_change( + alice, + ProfileInfo( + display_name="abcd\u0000efgh", + avatar_url=MXC_DUMMY, + ), + ) + ) + # Alice's profile should be updated with the new avatar, but no display name. + self.assertEqual( + self.get_success(self.user_dir_helper.get_profiles_in_user_directory()), + {alice: ProfileInfo(display_name=None, avatar_url=MXC_DUMMY)}, + ) + class TestUserDirSearchDisabled(unittest.HomeserverTestCase): servlets = [ From 1aa30f7b3e7767d2845858f37427282710ee94cc Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 18 May 2022 11:41:53 +0100 Subject: [PATCH 215/263] 1.59.1 --- CHANGES.md | 9 +++++++++ changelog.d/12762.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/12762.misc diff --git a/CHANGES.md b/CHANGES.md index 56d1a5a7d7bd..115f4b35ffe6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.59.1 (2022-05-18) +=========================== + +Internal Changes +---------------- + +- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) + + Synapse 1.59.0 (2022-05-17) =========================== diff --git a/changelog.d/12762.misc b/changelog.d/12762.misc deleted file mode 100644 index 990fb6fe74eb..000000000000 --- a/changelog.d/12762.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. diff --git a/debian/changelog b/debian/changelog index cc6152d5b314..dda342a630db 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.59.1) stable; urgency=medium + + * New Synapse release 1.59.1. + + -- Synapse Packaging team Wed, 18 May 2022 11:41:46 +0100 + matrix-synapse-py3 (1.59.0) stable; urgency=medium * New Synapse release 1.59.0. diff --git a/pyproject.toml b/pyproject.toml index e600a1d52e4b..5a5a2eaba73d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.59.0" +version = "1.59.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From d24a1486e5c5d69a8798a9d159fd9e06dfc8c3e3 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 18 May 2022 11:46:05 +0100 Subject: [PATCH 216/263] Fixup changelog --- CHANGES.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 115f4b35ffe6..e10ac0314abf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,12 @@ Synapse 1.59.1 (2022-05-18) =========================== -Internal Changes +This release fixes a long-standing issue which could prevent Synapse's user directory for updating properly. + +Bugfixes ---------------- -- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) +- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. Contributed by Nick @ Beeper. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) Synapse 1.59.0 (2022-05-17) From deca250e3f0f18ffc978f17d1f9e440fc8a4af98 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 18 May 2022 12:21:32 +0100 Subject: [PATCH 217/263] Add some documentation around the `rc_invites` option to the config docs (#12759) --- changelog.d/12759.doc | 1 + docs/usage/configuration/config_documentation.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 changelog.d/12759.doc diff --git a/changelog.d/12759.doc b/changelog.d/12759.doc new file mode 100644 index 000000000000..45d1c9c0ca1a --- /dev/null +++ b/changelog.d/12759.doc @@ -0,0 +1 @@ +Add information regarding the `rc_invites` ratelimiting option to the configuration docs. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 3e2031f08aa6..3ad3085bfac1 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1357,6 +1357,20 @@ This option sets ratelimiting how often invites can be sent in a room or to a specific user. `per_room` defaults to `per_second: 0.3`, `burst_count: 10` and `per_user` defaults to `per_second: 0.003`, `burst_count: 5`. +Client requests that invite user(s) when [creating a +room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom) +will count against the `rc_invites.per_room` limit, whereas +client requests to [invite a single user to a +room](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite) +will count against both the `rc_invites.per_user` and `rc_invites.per_room` limits. + +Federation requests to invite a user will count against the `rc_invites.per_user` +limit only, as Synapse presumes ratelimiting by room will be done by the sending server. + +The `rc_invites.per_user` limit applies to the *receiver* of the invite, rather than the +sender, meaning that a `rc_invite.per_user.burst_count` of 5 mandates that a single user +cannot *receive* more than a burst of 5 invites at a time. + Example configuration: ```yaml rc_invites: From a167304c8bba0dffb4c64dc9034272ff5bcaa4ff Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 18 May 2022 12:29:32 +0100 Subject: [PATCH 218/263] Switch the 'Configuration' link in the docs homepage to the config manual (#12748) --- changelog.d/12748.doc | 1 + docs/welcome_and_overview.md | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12748.doc diff --git a/changelog.d/12748.doc b/changelog.d/12748.doc new file mode 100644 index 000000000000..996ad3a1b926 --- /dev/null +++ b/changelog.d/12748.doc @@ -0,0 +1 @@ +Link to the configuration manual from the welcome page of the documentation. diff --git a/docs/welcome_and_overview.md b/docs/welcome_and_overview.md index aab2d6b4f0f6..451759f06ec6 100644 --- a/docs/welcome_and_overview.md +++ b/docs/welcome_and_overview.md @@ -7,10 +7,10 @@ team. ## Installing and using Synapse This documentation covers topics for **installation**, **configuration** and -**maintainence** of your Synapse process: +**maintenance** of your Synapse process: * Learn how to [install](setup/installation.md) and - [configure](usage/configuration/index.html) your own instance, perhaps with [Single + [configure](usage/configuration/config_documentation.md) your own instance, perhaps with [Single Sign-On](usage/configuration/user_authentication/index.html). * See how to [upgrade](upgrade.md) between Synapse versions. @@ -65,7 +65,7 @@ following documentation: Want to help keep Synapse going but don't know how to code? Synapse is a [Matrix.org Foundation](https://matrix.org) project. Consider becoming a -supportor on [Liberapay](https://liberapay.com/matrixdotorg), +supporter on [Liberapay](https://liberapay.com/matrixdotorg), [Patreon](https://patreon.com/matrixdotorg) or through [PayPal](https://paypal.me/matrixdotorg) via a one-time donation. From df4963548b8f9bf9e36e76558864f7045d7b5215 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 18 May 2022 11:46:06 +0000 Subject: [PATCH 219/263] Give a meaningful error message when a client tries to create a room with an invalid alias localpart. (#12779) --- changelog.d/12779.bugfix | 1 + synapse/handlers/directory.py | 3 +++ synapse/handlers/room.py | 15 +++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 changelog.d/12779.bugfix diff --git a/changelog.d/12779.bugfix b/changelog.d/12779.bugfix new file mode 100644 index 000000000000..7cf7a1f65f24 --- /dev/null +++ b/changelog.d/12779.bugfix @@ -0,0 +1 @@ +Give a meaningful error message when a client tries to create a room with an invalid alias localpart. \ No newline at end of file diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 33d827a45b33..4aa33df884ac 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -71,6 +71,9 @@ async def _create_association( if wchar in room_alias.localpart: raise SynapseError(400, "Invalid characters in room alias") + if ":" in room_alias.localpart: + raise SynapseError(400, "Invalid character in room alias localpart: ':'.") + if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a2973109adc4..53569e521219 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -751,6 +751,21 @@ async def create_room( if wchar in config["room_alias_name"]: raise SynapseError(400, "Invalid characters in room alias") + if ":" in config["room_alias_name"]: + # Prevent someone from trying to pass in a full alias here. + # Note that it's permissible for a room alias to have multiple + # hash symbols at the start (notably bridged over from IRC, too), + # but the first colon in the alias is defined to separate the local + # part from the server name. + # (remember server names can contain port numbers, also separated + # by a colon. But under no circumstances should the local part be + # allowed to contain a colon!) + raise SynapseError( + 400, + "':' is not permitted in the room alias name. " + "Please note this expects a local part — 'wombat', not '#wombat:example.com'.", + ) + room_alias = RoomAlias(config["room_alias_name"], self.hs.hostname) mapping = await self.store.get_association_from_room_alias(room_alias) From 635f0d916bc5155d3f3cba0389a1ebe08a6b5910 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 18 May 2022 13:57:59 +0000 Subject: [PATCH 220/263] Do not keep going if there are 5 back-to-back background update failures. (#12781) --- changelog.d/12781.misc | 1 + synapse/storage/background_updates.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/12781.misc diff --git a/changelog.d/12781.misc b/changelog.d/12781.misc new file mode 100644 index 000000000000..8a045716172a --- /dev/null +++ b/changelog.d/12781.misc @@ -0,0 +1 @@ +Do not keep going if there are 5 back-to-back background update failures. \ No newline at end of file diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index c2bbbb574e75..37f2d6c644f4 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -282,12 +282,20 @@ async def run_background_updates(self, sleep: bool) -> None: self._running = True + back_to_back_failures = 0 + try: logger.info("Starting background schema updates") while self.enabled: try: result = await self.do_next_background_update(sleep) + back_to_back_failures = 0 except Exception: + back_to_back_failures += 1 + if back_to_back_failures >= 5: + raise RuntimeError( + "5 back-to-back background update failures; aborting." + ) logger.exception("Error doing update") else: if result: From 50ae4eafe1f8ba31f1977e5dc11c85f15722f1ee Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 18 May 2022 17:02:10 +0200 Subject: [PATCH 221/263] Add some type hints to `event_federation` datastore (#12753) Co-authored-by: David Robertson --- changelog.d/12753.misc | 1 + mypy.ini | 1 - synapse/handlers/room_batch.py | 2 + .../databases/main/event_federation.py | 187 ++++++++++++------ tests/handlers/test_federation.py | 1 + 5 files changed, 127 insertions(+), 65 deletions(-) create mode 100644 changelog.d/12753.misc diff --git a/changelog.d/12753.misc b/changelog.d/12753.misc new file mode 100644 index 000000000000..e793d08e5e3f --- /dev/null +++ b/changelog.d/12753.misc @@ -0,0 +1 @@ +Add some type hints to datastore. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 45668974b363..4fa020b8764d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -27,7 +27,6 @@ exclude = (?x) |synapse/storage/databases/__init__.py |synapse/storage/databases/main/cache.py |synapse/storage/databases/main/devices.py - |synapse/storage/databases/main/event_federation.py |synapse/storage/schema/ |tests/api/test_auth.py diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py index 29de7e5bed10..fbfd7484065c 100644 --- a/synapse/handlers/room_batch.py +++ b/synapse/handlers/room_batch.py @@ -53,6 +53,7 @@ async def inherit_depth_from_prev_ids(self, prev_event_ids: List[str]) -> int: # We want to use the successor event depth so they appear after `prev_event` because # it has a larger `depth` but before the successor event because the `stream_ordering` # is negative before the successor event. + assert most_recent_prev_event_id is not None successor_event_ids = await self.store.get_successor_events( most_recent_prev_event_id ) @@ -139,6 +140,7 @@ async def get_most_recent_full_state_ids_from_event_id_list( _, ) = await self.store.get_max_depth_of(event_ids) # mapping from (type, state_key) -> state_event_id + assert most_recent_event_id is not None prev_state_map = await self.state_store.get_state_ids_for_event( most_recent_event_id ) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 471022470843..dcfe8caf473a 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -14,7 +14,17 @@ import itertools import logging from queue import Empty, PriorityQueue -from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Optional, Set, Tuple +from typing import ( + TYPE_CHECKING, + Collection, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + cast, +) import attr from prometheus_client import Counter, Gauge @@ -33,7 +43,7 @@ from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.signatures import SignatureWorkerStore from synapse.storage.engines import PostgresEngine -from synapse.storage.types import Cursor +from synapse.types import JsonDict from synapse.util import json_encoder from synapse.util.caches.descriptors import cached from synapse.util.caches.lrucache import LruCache @@ -135,7 +145,7 @@ async def get_auth_chain_ids( # Check if we have indexed the room so we can use the chain cover # algorithm. - room = await self.get_room(room_id) + room = await self.get_room(room_id) # type: ignore[attr-defined] if room["has_auth_chain_index"]: try: return await self.db_pool.runInteraction( @@ -158,7 +168,11 @@ async def get_auth_chain_ids( ) def _get_auth_chain_ids_using_cover_index_txn( - self, txn: Cursor, room_id: str, event_ids: Collection[str], include_given: bool + self, + txn: LoggingTransaction, + room_id: str, + event_ids: Collection[str], + include_given: bool, ) -> Set[str]: """Calculates the auth chain IDs using the chain index.""" @@ -215,9 +229,9 @@ def _get_auth_chain_ids_using_cover_index_txn( chains: Dict[int, int] = {} # Add all linked chains reachable from initial set of chains. - for batch in batch_iter(event_chains, 1000): + for batch2 in batch_iter(event_chains, 1000): clause, args = make_in_list_sql_clause( - txn.database_engine, "origin_chain_id", batch + txn.database_engine, "origin_chain_id", batch2 ) txn.execute(sql % (clause,), args) @@ -297,7 +311,7 @@ def _get_auth_chain_ids_txn( front = set(event_ids) while front: - new_front = set() + new_front: Set[str] = set() for chunk in batch_iter(front, 100): # Pull the auth events either from the cache or DB. to_fetch: List[str] = [] # Event IDs to fetch from DB @@ -316,7 +330,7 @@ def _get_auth_chain_ids_txn( # Note we need to batch up the results by event ID before # adding to the cache. - to_cache = {} + to_cache: Dict[str, List[Tuple[str, int]]] = {} for event_id, auth_event_id, auth_event_depth in txn: to_cache.setdefault(event_id, []).append( (auth_event_id, auth_event_depth) @@ -349,7 +363,7 @@ async def get_auth_chain_difference( # Check if we have indexed the room so we can use the chain cover # algorithm. - room = await self.get_room(room_id) + room = await self.get_room(room_id) # type: ignore[attr-defined] if room["has_auth_chain_index"]: try: return await self.db_pool.runInteraction( @@ -370,7 +384,7 @@ async def get_auth_chain_difference( ) def _get_auth_chain_difference_using_cover_index_txn( - self, txn: Cursor, room_id: str, state_sets: List[Set[str]] + self, txn: LoggingTransaction, room_id: str, state_sets: List[Set[str]] ) -> Set[str]: """Calculates the auth chain difference using the chain index. @@ -444,9 +458,9 @@ def _get_auth_chain_difference_using_cover_index_txn( # (We need to take a copy of `seen_chains` as we want to mutate it in # the loop) - for batch in batch_iter(set(seen_chains), 1000): + for batch2 in batch_iter(set(seen_chains), 1000): clause, args = make_in_list_sql_clause( - txn.database_engine, "origin_chain_id", batch + txn.database_engine, "origin_chain_id", batch2 ) txn.execute(sql % (clause,), args) @@ -529,7 +543,7 @@ def _get_auth_chain_difference_using_cover_index_txn( return result def _get_auth_chain_difference_txn( - self, txn, state_sets: List[Set[str]] + self, txn: LoggingTransaction, state_sets: List[Set[str]] ) -> Set[str]: """Calculates the auth chain difference using a breadth first search. @@ -602,7 +616,7 @@ def _get_auth_chain_difference_txn( # I think building a temporary list with fetchall is more efficient than # just `search.extend(txn)`, but this is unconfirmed - search.extend(txn.fetchall()) + search.extend(cast(List[Tuple[int, str]], txn.fetchall())) # sort by depth search.sort() @@ -645,7 +659,7 @@ def _get_auth_chain_difference_txn( # We parse the results and add the to the `found` set and the # cache (note we need to batch up the results by event ID before # adding to the cache). - to_cache = {} + to_cache: Dict[str, List[Tuple[str, int]]] = {} for event_id, auth_event_id, auth_event_depth in txn: to_cache.setdefault(event_id, []).append( (auth_event_id, auth_event_depth) @@ -696,7 +710,7 @@ def _get_auth_chain_difference_txn( return {eid for eid, n in event_to_missing_sets.items() if n} async def get_oldest_event_ids_with_depth_in_room( - self, room_id + self, room_id: str ) -> List[Tuple[str, int]]: """Gets the oldest events(backwards extremities) in the room along with the aproximate depth. @@ -713,7 +727,9 @@ async def get_oldest_event_ids_with_depth_in_room( List of (event_id, depth) tuples """ - def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): + def get_oldest_event_ids_with_depth_in_room_txn( + txn: LoggingTransaction, room_id: str + ) -> List[Tuple[str, int]]: # Assemble a dictionary with event_id -> depth for the oldest events # we know of in the room. Backwards extremeties are the oldest # events we know of in the room but we only know of them because @@ -743,7 +759,7 @@ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): txn.execute(sql, (room_id, False)) - return txn.fetchall() + return cast(List[Tuple[str, int]], txn.fetchall()) return await self.db_pool.runInteraction( "get_oldest_event_ids_with_depth_in_room", @@ -752,7 +768,7 @@ def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id): ) async def get_insertion_event_backward_extremities_in_room( - self, room_id + self, room_id: str ) -> List[Tuple[str, int]]: """Get the insertion events we know about that we haven't backfilled yet. @@ -768,7 +784,9 @@ async def get_insertion_event_backward_extremities_in_room( List of (event_id, depth) tuples """ - def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): + def get_insertion_event_backward_extremities_in_room_txn( + txn: LoggingTransaction, room_id: str + ) -> List[Tuple[str, int]]: sql = """ SELECT b.event_id, MAX(e.depth) FROM insertion_events as i /* We only want insertion events that are also marked as backwards extremities */ @@ -780,7 +798,7 @@ def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): """ txn.execute(sql, (room_id,)) - return txn.fetchall() + return cast(List[Tuple[str, int]], txn.fetchall()) return await self.db_pool.runInteraction( "get_insertion_event_backward_extremities_in_room", @@ -788,7 +806,7 @@ def get_insertion_event_backward_extremities_in_room_txn(txn, room_id): room_id, ) - async def get_max_depth_of(self, event_ids: List[str]) -> Tuple[str, int]: + async def get_max_depth_of(self, event_ids: List[str]) -> Tuple[Optional[str], int]: """Returns the event ID and depth for the event that has the max depth from a set of event IDs Args: @@ -817,7 +835,7 @@ async def get_max_depth_of(self, event_ids: List[str]) -> Tuple[str, int]: return max_depth_event_id, current_max_depth - async def get_min_depth_of(self, event_ids: List[str]) -> Tuple[str, int]: + async def get_min_depth_of(self, event_ids: List[str]) -> Tuple[Optional[str], int]: """Returns the event ID and depth for the event that has the min depth from a set of event IDs Args: @@ -865,7 +883,9 @@ async def get_prev_events_for_room(self, room_id: str) -> List[str]: "get_prev_events_for_room", self._get_prev_events_for_room_txn, room_id ) - def _get_prev_events_for_room_txn(self, txn, room_id: str): + def _get_prev_events_for_room_txn( + self, txn: LoggingTransaction, room_id: str + ) -> List[str]: # we just use the 10 newest events. Older events will become # prev_events of future events. @@ -896,7 +916,7 @@ async def get_rooms_with_many_extremities( sorted by extremity count. """ - def _get_rooms_with_many_extremities_txn(txn): + def _get_rooms_with_many_extremities_txn(txn: LoggingTransaction) -> List[str]: where_clause = "1=1" if room_id_filter: where_clause = "room_id NOT IN (%s)" % ( @@ -937,7 +957,9 @@ async def get_min_depth(self, room_id: str) -> Optional[int]: "get_min_depth", self._get_min_depth_interaction, room_id ) - def _get_min_depth_interaction(self, txn, room_id): + def _get_min_depth_interaction( + self, txn: LoggingTransaction, room_id: str + ) -> Optional[int]: min_depth = self.db_pool.simple_select_one_onecol_txn( txn, table="room_depth", @@ -966,22 +988,24 @@ async def get_forward_extremities_for_room_at_stream_ordering( """ # We want to make the cache more effective, so we clamp to the last # change before the given ordering. - last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id) + last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id) # type: ignore[attr-defined] # We don't always have a full stream_to_exterm_id table, e.g. after # the upgrade that introduced it, so we make sure we never ask for a # stream_ordering from before a restart - last_change = max(self._stream_order_on_start, last_change) + last_change = max(self._stream_order_on_start, last_change) # type: ignore[attr-defined] # provided the last_change is recent enough, we now clamp the requested # stream_ordering to it. - if last_change > self.stream_ordering_month_ago: + if last_change > self.stream_ordering_month_ago: # type: ignore[attr-defined] stream_ordering = min(last_change, stream_ordering) return await self._get_forward_extremeties_for_room(room_id, stream_ordering) @cached(max_entries=5000, num_args=2) - async def _get_forward_extremeties_for_room(self, room_id, stream_ordering): + async def _get_forward_extremeties_for_room( + self, room_id: str, stream_ordering: int + ) -> List[str]: """For a given room_id and stream_ordering, return the forward extremeties of the room at that point in "time". @@ -989,7 +1013,7 @@ async def _get_forward_extremeties_for_room(self, room_id, stream_ordering): stream_orderings from that point. """ - if stream_ordering <= self.stream_ordering_month_ago: + if stream_ordering <= self.stream_ordering_month_ago: # type: ignore[attr-defined] raise StoreError(400, "stream_ordering too old %s" % (stream_ordering,)) sql = """ @@ -1002,7 +1026,7 @@ async def _get_forward_extremeties_for_room(self, room_id, stream_ordering): WHERE room_id = ? """ - def get_forward_extremeties_for_room_txn(txn): + def get_forward_extremeties_for_room_txn(txn: LoggingTransaction) -> List[str]: txn.execute(sql, (stream_ordering, room_id)) return [event_id for event_id, in txn] @@ -1104,8 +1128,8 @@ def _get_connected_prev_event_backfill_results_txn( ] async def get_backfill_events( - self, room_id: str, seed_event_id_list: list, limit: int - ): + self, room_id: str, seed_event_id_list: List[str], limit: int + ) -> List[EventBase]: """Get a list of Events for a given topic that occurred before (and including) the events in seed_event_id_list. Return a list of max size `limit` @@ -1123,10 +1147,19 @@ async def get_backfill_events( ) events = await self.get_events_as_list(event_ids) return sorted( - events, key=lambda e: (-e.depth, -e.internal_metadata.stream_ordering) + # type-ignore: mypy doesn't like negating the Optional[int] stream_ordering. + # But it's never None, because these events were previously persisted to the DB. + events, + key=lambda e: (-e.depth, -e.internal_metadata.stream_ordering), # type: ignore[operator] ) - def _get_backfill_events(self, txn, room_id, seed_event_id_list, limit): + def _get_backfill_events( + self, + txn: LoggingTransaction, + room_id: str, + seed_event_id_list: List[str], + limit: int, + ) -> Set[str]: """ We want to make sure that we do a breadth-first, "depth" ordered search. We also handle navigating historical branches of history connected by @@ -1139,7 +1172,7 @@ def _get_backfill_events(self, txn, room_id, seed_event_id_list, limit): limit, ) - event_id_results = set() + event_id_results: Set[str] = set() # In a PriorityQueue, the lowest valued entries are retrieved first. # We're using depth as the priority in the queue and tie-break based on @@ -1147,7 +1180,7 @@ def _get_backfill_events(self, txn, room_id, seed_event_id_list, limit): # highest and newest-in-time message. We add events to the queue with a # negative depth so that we process the newest-in-time messages first # going backwards in time. stream_ordering follows the same pattern. - queue = PriorityQueue() + queue: "PriorityQueue[Tuple[int, int, str, str]]" = PriorityQueue() for seed_event_id in seed_event_id_list: event_lookup_result = self.db_pool.simple_select_one_txn( @@ -1253,7 +1286,13 @@ def _get_backfill_events(self, txn, room_id, seed_event_id_list, limit): return event_id_results - async def get_missing_events(self, room_id, earliest_events, latest_events, limit): + async def get_missing_events( + self, + room_id: str, + earliest_events: List[str], + latest_events: List[str], + limit: int, + ) -> List[EventBase]: ids = await self.db_pool.runInteraction( "get_missing_events", self._get_missing_events, @@ -1264,11 +1303,18 @@ async def get_missing_events(self, room_id, earliest_events, latest_events, limi ) return await self.get_events_as_list(ids) - def _get_missing_events(self, txn, room_id, earliest_events, latest_events, limit): + def _get_missing_events( + self, + txn: LoggingTransaction, + room_id: str, + earliest_events: List[str], + latest_events: List[str], + limit: int, + ) -> List[str]: seen_events = set(earliest_events) front = set(latest_events) - seen_events - event_results = [] + event_results: List[str] = [] query = ( "SELECT prev_event_id FROM event_edges " @@ -1311,7 +1357,7 @@ async def get_successor_events(self, event_id: str) -> List[str]: @wrap_as_background_process("delete_old_forward_extrem_cache") async def _delete_old_forward_extrem_cache(self) -> None: - def _delete_old_forward_extrem_cache_txn(txn): + def _delete_old_forward_extrem_cache_txn(txn: LoggingTransaction) -> None: # Delete entries older than a month, while making sure we don't delete # the only entries for a room. sql = """ @@ -1324,7 +1370,7 @@ def _delete_old_forward_extrem_cache_txn(txn): ) AND stream_ordering < ? """ txn.execute( - sql, (self.stream_ordering_month_ago, self.stream_ordering_month_ago) + sql, (self.stream_ordering_month_ago, self.stream_ordering_month_ago) # type: ignore[attr-defined] ) await self.db_pool.runInteraction( @@ -1382,7 +1428,9 @@ async def remove_received_event_from_staging( """ if self.db_pool.engine.supports_returning: - def _remove_received_event_from_staging_txn(txn): + def _remove_received_event_from_staging_txn( + txn: LoggingTransaction, + ) -> Optional[int]: sql = """ DELETE FROM federation_inbound_events_staging WHERE origin = ? AND event_id = ? @@ -1390,21 +1438,24 @@ def _remove_received_event_from_staging_txn(txn): """ txn.execute(sql, (origin, event_id)) - return txn.fetchone() + row = cast(Optional[Tuple[int]], txn.fetchone()) - row = await self.db_pool.runInteraction( + if row is None: + return None + + return row[0] + + return await self.db_pool.runInteraction( "remove_received_event_from_staging", _remove_received_event_from_staging_txn, db_autocommit=True, ) - if row is None: - return None - - return row[0] else: - def _remove_received_event_from_staging_txn(txn): + def _remove_received_event_from_staging_txn( + txn: LoggingTransaction, + ) -> Optional[int]: received_ts = self.db_pool.simple_select_one_onecol_txn( txn, table="federation_inbound_events_staging", @@ -1437,7 +1488,9 @@ async def get_next_staged_event_id_for_room( ) -> Optional[Tuple[str, str]]: """Get the next event ID in the staging area for the given room.""" - def _get_next_staged_event_id_for_room_txn(txn): + def _get_next_staged_event_id_for_room_txn( + txn: LoggingTransaction, + ) -> Optional[Tuple[str, str]]: sql = """ SELECT origin, event_id FROM federation_inbound_events_staging @@ -1448,7 +1501,7 @@ def _get_next_staged_event_id_for_room_txn(txn): txn.execute(sql, (room_id,)) - return txn.fetchone() + return cast(Optional[Tuple[str, str]], txn.fetchone()) return await self.db_pool.runInteraction( "get_next_staged_event_id_for_room", _get_next_staged_event_id_for_room_txn @@ -1461,7 +1514,9 @@ async def get_next_staged_event_for_room( ) -> Optional[Tuple[str, EventBase]]: """Get the next event in the staging area for the given room.""" - def _get_next_staged_event_for_room_txn(txn): + def _get_next_staged_event_for_room_txn( + txn: LoggingTransaction, + ) -> Optional[Tuple[str, str, str]]: sql = """ SELECT event_json, internal_metadata, origin FROM federation_inbound_events_staging @@ -1471,7 +1526,7 @@ def _get_next_staged_event_for_room_txn(txn): """ txn.execute(sql, (room_id,)) - return txn.fetchone() + return cast(Optional[Tuple[str, str, str]], txn.fetchone()) row = await self.db_pool.runInteraction( "get_next_staged_event_for_room", _get_next_staged_event_for_room_txn @@ -1599,18 +1654,20 @@ async def get_all_rooms_with_staged_incoming_events(self) -> List[str]: ) @wrap_as_background_process("_get_stats_for_federation_staging") - async def _get_stats_for_federation_staging(self): + async def _get_stats_for_federation_staging(self) -> None: """Update the prometheus metrics for the inbound federation staging area.""" - def _get_stats_for_federation_staging_txn(txn): + def _get_stats_for_federation_staging_txn( + txn: LoggingTransaction, + ) -> Tuple[int, int]: txn.execute("SELECT count(*) FROM federation_inbound_events_staging") - (count,) = txn.fetchone() + (count,) = cast(Tuple[int], txn.fetchone()) txn.execute( "SELECT min(received_ts) FROM federation_inbound_events_staging" ) - (received_ts,) = txn.fetchone() + (received_ts,) = cast(Tuple[Optional[int]], txn.fetchone()) # If there is nothing in the staging area default it to 0. age = 0 @@ -1651,19 +1708,21 @@ def __init__( self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth ) - async def clean_room_for_join(self, room_id): - return await self.db_pool.runInteraction( + async def clean_room_for_join(self, room_id: str) -> None: + await self.db_pool.runInteraction( "clean_room_for_join", self._clean_room_for_join_txn, room_id ) - def _clean_room_for_join_txn(self, txn, room_id): + def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None: query = "DELETE FROM event_forward_extremities WHERE room_id = ?" txn.execute(query, (room_id,)) txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,)) - async def _background_delete_non_state_event_auth(self, progress, batch_size): - def delete_event_auth(txn): + async def _background_delete_non_state_event_auth( + self, progress: JsonDict, batch_size: int + ) -> int: + def delete_event_auth(txn: LoggingTransaction) -> bool: target_min_stream_id = progress.get("target_min_stream_id_inclusive") max_stream_id = progress.get("max_stream_id_exclusive") diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 060ba5f5174f..e95dfdce2086 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -332,6 +332,7 @@ def test_backfill_floating_outlier_membership_auth(self) -> None: most_recent_prev_event_depth, ) = self.get_success(self.store.get_max_depth_of(prev_event_ids)) # mapping from (type, state_key) -> state_event_id + assert most_recent_prev_event_id is not None prev_state_map = self.get_success( self.state_store.get_state_ids_for_event(most_recent_prev_event_id) ) From 3d8839c30c96b49588196c60e2bbf056ed6465eb Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 18 May 2022 17:56:23 +0100 Subject: [PATCH 222/263] Add documentation for cancellation of request processing (#12761) Signed-off-by: Sean Quah --- changelog.d/12761.doc | 1 + docs/SUMMARY.md | 1 + .../synapse_architecture/cancellation.md | 392 ++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 changelog.d/12761.doc create mode 100644 docs/development/synapse_architecture/cancellation.md diff --git a/changelog.d/12761.doc b/changelog.d/12761.doc new file mode 100644 index 000000000000..2eb2c0976f1b --- /dev/null +++ b/changelog.d/12761.doc @@ -0,0 +1 @@ +Add documentation for cancellation of request processing. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 65570cefbe1b..8400a6539a4e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -89,6 +89,7 @@ - [Database Schemas](development/database_schema.md) - [Experimental features](development/experimental_features.md) - [Synapse Architecture]() + - [Cancellation](development/synapse_architecture/cancellation.md) - [Log Contexts](log_contexts.md) - [Replication](replication.md) - [TCP Replication](tcp_replication.md) diff --git a/docs/development/synapse_architecture/cancellation.md b/docs/development/synapse_architecture/cancellation.md new file mode 100644 index 000000000000..ef9e0226353b --- /dev/null +++ b/docs/development/synapse_architecture/cancellation.md @@ -0,0 +1,392 @@ +# Cancellation +Sometimes, requests take a long time to service and clients disconnect +before Synapse produces a response. To avoid wasting resources, Synapse +can cancel request processing for select endpoints marked with the +`@cancellable` decorator. + +Synapse makes use of Twisted's `Deferred.cancel()` feature to make +cancellation work. The `@cancellable` decorator does nothing by itself +and merely acts as a flag, signalling to developers and other code alike +that a method can be cancelled. + +## Enabling cancellation for an endpoint +1. Check that the endpoint method, and any `async` functions in its call + tree handle cancellation correctly. See + [Handling cancellation correctly](#handling-cancellation-correctly) + for a list of things to look out for. +2. Add the `@cancellable` decorator to the `on_GET/POST/PUT/DELETE` + method. It's not recommended to make non-`GET` methods cancellable, + since cancellation midway through some database updates is less + likely to be handled correctly. + +## Mechanics +There are two stages to cancellation: downward propagation of a +`cancel()` call, followed by upwards propagation of a `CancelledError` +out of a blocked `await`. +Both Twisted and asyncio have a cancellation mechanism. + +| | Method | Exception | Exception inherits from | +|---------------|---------------------|-----------------------------------------|-------------------------| +| Twisted | `Deferred.cancel()` | `twisted.internet.defer.CancelledError` | `Exception` (!) | +| asyncio | `Task.cancel()` | `asyncio.CancelledError` | `BaseException` | + +### Deferred.cancel() +When Synapse starts handling a request, it runs the async method +responsible for handling it using `defer.ensureDeferred`, which returns +a `Deferred`. For example: + +```python +def do_something() -> Deferred[None]: + ... + +@cancellable +async def on_GET() -> Tuple[int, JsonDict]: + d = make_deferred_yieldable(do_something()) + await d + return 200, {} + +request = defer.ensureDeferred(on_GET()) +``` + +When a client disconnects early, Synapse checks for the presence of the +`@cancellable` decorator on `on_GET`. Since `on_GET` is cancellable, +`Deferred.cancel()` is called on the `Deferred` from +`defer.ensureDeferred`, ie. `request`. Twisted knows which `Deferred` +`request` is waiting on and passes the `cancel()` call on to `d`. + +The `Deferred` being waited on, `d`, may have its own handling for +`cancel()` and pass the call on to other `Deferred`s. + +Eventually, a `Deferred` handles the `cancel()` call by resolving itself +with a `CancelledError`. + +### CancelledError +The `CancelledError` gets raised out of the `await` and bubbles up, as +per normal Python exception handling. + +## Handling cancellation correctly +In general, when writing code that might be subject to cancellation, two +things must be considered: + * The effect of `CancelledError`s raised out of `await`s. + * The effect of `Deferred`s being `cancel()`ed. + +Examples of code that handles cancellation incorrectly include: + * `try-except` blocks which swallow `CancelledError`s. + * Code that shares the same `Deferred`, which may be cancelled, between + multiple requests. + * Code that starts some processing that's exempt from cancellation, but + uses a logging context from cancellable code. The logging context + will be finished upon cancellation, while the uncancelled processing + is still using it. + +Some common patterns are listed below in more detail. + +### `async` function calls +Most functions in Synapse are relatively straightforward from a +cancellation standpoint: they don't do anything with `Deferred`s and +purely call and `await` other `async` functions. + +An `async` function handles cancellation correctly if its own code +handles cancellation correctly and all the async function it calls +handle cancellation correctly. For example: +```python +async def do_two_things() -> None: + check_something() + await do_something() + await do_something_else() +``` +`do_two_things` handles cancellation correctly if `do_something` and +`do_something_else` handle cancellation correctly. + +That is, when checking whether a function handles cancellation +correctly, its implementation and all its `async` function calls need to +be checked, recursively. + +As `check_something` is not `async`, it does not need to be checked. + +### CancelledErrors +Because Twisted's `CancelledError`s are `Exception`s, it's easy to +accidentally catch and suppress them. Care must be taken to ensure that +`CancelledError`s are allowed to propagate upwards. + + + + + + + + + + +
+ +**Bad**: +```python +try: + await do_something() +except Exception: + # `CancelledError` gets swallowed here. + logger.info(...) +``` + + +**Good**: +```python +try: + await do_something() +except CancelledError: + raise +except Exception: + logger.info(...) +``` +
+ +**OK**: +```python +try: + check_something() + # A `CancelledError` won't ever be raised here. +except Exception: + logger.info(...) +``` + + +**Good**: +```python +try: + await do_something() +except ValueError: + logger.info(...) +``` +
+ +#### defer.gatherResults +`defer.gatherResults` produces a `Deferred` which: + * broadcasts `cancel()` calls to every `Deferred` being waited on. + * wraps the first exception it sees in a `FirstError`. + +Together, this means that `CancelledError`s will be wrapped in +a `FirstError` unless unwrapped. Such `FirstError`s are liable to be +swallowed, so they must be unwrapped. + + + + + + +
+ +**Bad**: +```python +async def do_something() -> None: + await make_deferred_yieldable( + defer.gatherResults([...], consumeErrors=True) + ) + +try: + await do_something() +except CancelledError: + raise +except Exception: + # `FirstError(CancelledError)` gets swallowed here. + logger.info(...) +``` + + + +**Good**: +```python +async def do_something() -> None: + await make_deferred_yieldable( + defer.gatherResults([...], consumeErrors=True) + ).addErrback(unwrapFirstError) + +try: + await do_something() +except CancelledError: + raise +except Exception: + logger.info(...) +``` +
+ +### Creation of `Deferred`s +If a function creates a `Deferred`, the effect of cancelling it must be considered. `Deferred`s that get shared are likely to have unintended behaviour when cancelled. + + + + + + + + + +
+ +**Bad**: +```python +cache: Dict[str, Deferred[None]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + deferred = cache.get(room_id) + if deferred is None: + deferred = Deferred() + cache[room_id] = deferred + # `deferred` can have multiple waiters. + # All of them will observe a `CancelledError` + # if any one of them is cancelled. + return make_deferred_yieldable(deferred) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +``` + + +**Good**: +```python +cache: Dict[str, Deferred[None]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + deferred = cache.get(room_id) + if deferred is None: + deferred = Deferred() + cache[room_id] = deferred + # `deferred` will never be cancelled now. + # A `CancelledError` will still come out of + # the `await`. + # `delay_cancellation` may also be used. + return make_deferred_yieldable(stop_cancellation(deferred)) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +``` +
+ + +**Good**: +```python +cache: Dict[str, List[Deferred[None]]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + if room_id not in cache: + cache[room_id] = [] + # Each request gets its own `Deferred` to wait on. + deferred = Deferred() + cache[room_id]].append(deferred) + return make_deferred_yieldable(deferred) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +``` +
+ +### Uncancelled processing +Some `async` functions may kick off some `async` processing which is +intentionally protected from cancellation, by `stop_cancellation` or +other means. If the `async` processing inherits the logcontext of the +request which initiated it, care must be taken to ensure that the +logcontext is not finished before the `async` processing completes. + + + + + + + + + + +
+ +**Bad**: +```python +cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + # `do_something_else` will never be cancelled and + # can outlive the `request-1` logging context. + run_in_background(do_something_else, to_resolve) + + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +``` + + +**Good**: +```python +cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + run_in_background(do_something_else, to_resolve) + # We'll wait until `do_something_else` is + # done before raising a `CancelledError`. + await make_deferred_yieldable( + delay_cancellation(cache.observe()) + ) + else: + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +``` +
+ +**OK**: +```python +cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + # `do_something_else` will get its own independent + # logging context. `request-1` will not count any + # metrics from `do_something_else`. + run_as_background_process( + "do_something_else", + do_something_else, + to_resolve, + ) + + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +``` + +
From 19d79b6ebe3070ad7352f24549fbafb9dee44b75 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 18 May 2022 10:15:52 -0700 Subject: [PATCH 223/263] Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. (#12775) --- changelog.d/12775.misc | 1 + synapse/state/__init__.py | 35 +++++++++++++----------- synapse/storage/databases/state/store.py | 2 +- synapse/storage/state.py | 12 ++++---- tests/test_state.py | 13 +++++++++ 5 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 changelog.d/12775.misc diff --git a/changelog.d/12775.misc b/changelog.d/12775.misc new file mode 100644 index 000000000000..eac326cde3a7 --- /dev/null +++ b/changelog.d/12775.misc @@ -0,0 +1 @@ +Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. \ No newline at end of file diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 0219091c4e8b..4b4ed42cff33 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -288,7 +288,6 @@ async def compute_event_context( # # first of all, figure out the state before the event # - if old_state: # if we're given the state before the event, then we use that state_ids_before_event: StateMap[str] = { @@ -419,33 +418,37 @@ async def resolve_state_groups_for_events( """ logger.debug("resolve_state_groups event_ids %s", event_ids) - # map from state group id to the state in that state group (where - # 'state' is a map from state key to event id) - # dict[int, dict[(str, str), str]] - state_groups_ids = await self.state_store.get_state_groups_ids( - room_id, event_ids - ) - - if len(state_groups_ids) == 0: - return _StateCacheEntry(state={}, state_group=None) - elif len(state_groups_ids) == 1: - name, state_list = list(state_groups_ids.items()).pop() + state_groups = await self.state_store.get_state_group_for_events(event_ids) - prev_group, delta_ids = await self.state_store.get_state_group_delta(name) + state_group_ids = state_groups.values() + # check if each event has same state group id, if so there's no state to resolve + state_group_ids_set = set(state_group_ids) + if len(state_group_ids_set) == 1: + (state_group_id,) = state_group_ids_set + state = await self.state_store.get_state_for_groups(state_group_ids_set) + prev_group, delta_ids = await self.state_store.get_state_group_delta( + state_group_id + ) return _StateCacheEntry( - state=state_list, - state_group=name, + state=state[state_group_id], + state_group=state_group_id, prev_group=prev_group, delta_ids=delta_ids, ) + elif len(state_group_ids_set) == 0: + return _StateCacheEntry(state={}, state_group=None) room_version = await self.store.get_room_version_id(room_id) + state_to_resolve = await self.state_store.get_state_for_groups( + state_group_ids_set + ) + result = await self._state_resolution_handler.resolve_state_groups( room_id, room_version, - state_groups_ids, + state_to_resolve, None, state_res_store=StateResolutionStore(self.store), ) diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 7614d76ac646..609a2b88bfbf 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -189,7 +189,7 @@ def _get_state_for_group_using_cache( group: int, state_filter: StateFilter, ) -> Tuple[MutableStateMap[str], bool]: - """Checks if group is in cache. See `_get_state_for_groups` + """Checks if group is in cache. See `get_state_for_groups` Args: cache: the state group cache to use diff --git a/synapse/storage/state.py b/synapse/storage/state.py index d4a1bd4f9d7d..a6c60de50434 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -586,7 +586,7 @@ async def get_state_groups_ids( if not event_ids: return {} - event_to_groups = await self._get_state_group_for_events(event_ids) + event_to_groups = await self.get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups(groups) @@ -602,7 +602,7 @@ async def get_state_ids_for_group(self, state_group: int) -> StateMap[str]: Returns: Resolves to a map of (type, state_key) -> event_id """ - group_to_state = await self._get_state_for_groups((state_group,)) + group_to_state = await self.get_state_for_groups((state_group,)) return group_to_state[state_group] @@ -675,7 +675,7 @@ async def get_state_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self._get_state_group_for_events(event_ids) + event_to_groups = await self.get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -716,7 +716,7 @@ async def get_state_ids_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self._get_state_group_for_events(event_ids) + event_to_groups = await self.get_state_group_for_events(event_ids) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -774,7 +774,7 @@ async def get_state_ids_for_event( ) return state_map[event_id] - def _get_state_for_groups( + def get_state_for_groups( self, groups: Iterable[int], state_filter: Optional[StateFilter] = None ) -> Awaitable[Dict[int, MutableStateMap[str]]]: """Gets the state at each of a list of state groups, optionally @@ -792,7 +792,7 @@ def _get_state_for_groups( groups, state_filter or StateFilter.all() ) - async def _get_state_group_for_events( + async def get_state_group_for_events( self, event_ids: Collection[str], await_full_state: bool = True, diff --git a/tests/test_state.py b/tests/test_state.py index 651ec1c7d4bd..74a8ce6096b9 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -129,6 +129,19 @@ def register_event_id_state_group(self, event_id, state_group): async def get_room_version_id(self, room_id): return RoomVersions.V1.identifier + async def get_state_group_for_events(self, event_ids): + res = {} + for event in event_ids: + res[event] = self._event_to_state_group[event] + return res + + async def get_state_for_groups(self, groups): + res = {} + for group in groups: + state = self._group_to_state[group] + res[group] = state + return res + class DictObj(dict): def __init__(self, **kwargs): From 0fce474a4019b441940b6e12ac8a50ffac09727e Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 18 May 2022 18:24:44 +0100 Subject: [PATCH 224/263] Fix YAML parsing error in `url_preview_accept_language` (#12785) --- changelog.d/12785.doc | 1 + docs/usage/configuration/config_documentation.md | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12785.doc diff --git a/changelog.d/12785.doc b/changelog.d/12785.doc new file mode 100644 index 000000000000..5209dfeb053e --- /dev/null +++ b/changelog.d/12785.doc @@ -0,0 +1 @@ +Fix invalid YAML syntax in the example documentation for the `url_preview_accept_language` config option. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 3ad3085bfac1..6b9ffc09d2b3 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1194,7 +1194,7 @@ For more information on using Synapse with Postgres, see [here](../../postgres.md). Example SQLite configuration: -``` +```yaml database: name: sqlite3 args: @@ -1202,7 +1202,7 @@ database: ``` Example Postgres configuration: -``` +```yaml database: name: psycopg2 txn_limit: 10000 @@ -1679,10 +1679,10 @@ Defaults to "en". Example configuration: ```yaml url_preview_accept_language: - - en-UK - - en-US;q=0.9 - - fr;q=0.8 - - *;q=0.7 + - 'en-UK' + - 'en-US;q=0.9' + - 'fr;q=0.8' + - '*;q=0.7' ``` ---- Config option: `oembed` From d38c73e9abbd7fe40f4f24d4c96107415e6f15a1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 18 May 2022 20:33:57 +0200 Subject: [PATCH 225/263] Skip waiting for full state if a StateFilter does not require it (#12498) If `StateFilter` specifies a state set which we will have regardless of state-syncing, then we may as well return it immediately. --- changelog.d/12498.misc | 1 + synapse/storage/state.py | 63 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12498.misc diff --git a/changelog.d/12498.misc b/changelog.d/12498.misc new file mode 100644 index 000000000000..8a00b94fbeef --- /dev/null +++ b/changelog.d/12498.misc @@ -0,0 +1 @@ +Preparation for faster-room-join work: return subsets of room state which we already have, immediately. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index a6c60de50434..e58301a8f087 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1,4 +1,5 @@ # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2022 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +16,7 @@ from typing import ( TYPE_CHECKING, Awaitable, + Callable, Collection, Dict, Iterable, @@ -532,6 +534,44 @@ def approx_difference(self, other: "StateFilter") -> "StateFilter": new_all, new_excludes, new_wildcards, new_concrete_keys ) + def must_await_full_state(self, is_mine_id: Callable[[str], bool]) -> bool: + """Check if we need to wait for full state to complete to calculate this state + + If we have a state filter which is completely satisfied even with partial + state, then we don't need to await_full_state before we can return it. + + Args: + is_mine_id: a callable which confirms if a given state_key matches a mxid + of a local user + """ + + # TODO(faster_joins): it's not entirely clear that this is safe. In particular, + # there may be circumstances in which we return a piece of state that, once we + # resync the state, we discover is invalid. For example: if it turns out that + # the sender of a piece of state wasn't actually in the room, then clearly that + # state shouldn't have been returned. + # We should at least add some tests around this to see what happens. + + # if we haven't requested membership events, then it depends on the value of + # 'include_others' + if EventTypes.Member not in self.types: + return self.include_others + + # if we're looking for *all* membership events, then we have to wait + member_state_keys = self.types[EventTypes.Member] + if member_state_keys is None: + return True + + # otherwise, consider whose membership we are looking for. If it's entirely + # local users, then we don't need to wait. + for state_key in member_state_keys: + if not is_mine_id(state_key): + # remote user + return True + + # local users only + return False + _ALL_STATE_FILTER = StateFilter(types=frozendict(), include_others=True) _ALL_NON_MEMBER_STATE_FILTER = StateFilter( @@ -544,6 +584,7 @@ class StateGroupStorage: """High level interface to fetching state for event.""" def __init__(self, hs: "HomeServer", stores: "Databases"): + self._is_mine_id = hs.is_mine_id self.stores = stores self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) @@ -675,7 +716,13 @@ async def get_state_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.get_state_group_for_events(event_ids) + await_full_state = True + if state_filter and not state_filter.must_await_full_state(self._is_mine_id): + await_full_state = False + + event_to_groups = await self.get_state_group_for_events( + event_ids, await_full_state=await_full_state + ) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -699,7 +746,9 @@ async def get_state_for_events( return {event: event_to_state[event] for event in event_ids} async def get_state_ids_for_events( - self, event_ids: Collection[str], state_filter: Optional[StateFilter] = None + self, + event_ids: Collection[str], + state_filter: Optional[StateFilter] = None, ) -> Dict[str, StateMap[str]]: """ Get the state dicts corresponding to a list of events, containing the event_ids @@ -716,7 +765,13 @@ async def get_state_ids_for_events( RuntimeError if we don't have a state group for one or more of the events (ie they are outliers or unknown) """ - event_to_groups = await self.get_state_group_for_events(event_ids) + await_full_state = True + if state_filter and not state_filter.must_await_full_state(self._is_mine_id): + await_full_state = False + + event_to_groups = await self.get_state_group_for_events( + event_ids, await_full_state=await_full_state + ) groups = set(event_to_groups.values()) group_to_state = await self.stores.state._get_state_for_groups( @@ -802,7 +857,7 @@ async def get_state_group_for_events( Args: event_ids: events to get state groups for await_full_state: if true, will block if we do not yet have complete - state at this event. + state at these events. """ if await_full_state: await self._partial_state_events_tracker.await_full_state(event_ids) From a1cb05b3e8b439a2e68d3762ea7373785b8be4e1 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 18 May 2022 14:49:33 -0400 Subject: [PATCH 226/263] Fix federation in demo scripts. (#12783) --- changelog.d/12783.misc | 1 + demo/start.sh | 7 +++++-- docs/development/demo.md | 9 +++++---- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 changelog.d/12783.misc diff --git a/changelog.d/12783.misc b/changelog.d/12783.misc new file mode 100644 index 000000000000..97575608bb8b --- /dev/null +++ b/changelog.d/12783.misc @@ -0,0 +1 @@ +Fix federation when using the demo scripts. diff --git a/demo/start.sh b/demo/start.sh index 5a9972d24c2c..96b3a2ceab2f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -12,6 +12,7 @@ export PYTHONPATH echo "$PYTHONPATH" +# Create servers which listen on HTTP at 808x and HTTPS at 848x. for port in 8080 8081 8082; do echo "Starting server on port $port... " @@ -19,10 +20,12 @@ for port in 8080 8081 8082; do mkdir -p demo/$port pushd demo/$port || exit - # Generate the configuration for the homeserver at localhost:848x. + # Generate the configuration for the homeserver at localhost:848x, note that + # the homeserver name needs to match the HTTPS listening port for federation + # to properly work.. python3 -m synapse.app.homeserver \ --generate-config \ - --server-name "localhost:$port" \ + --server-name "localhost:$https_port" \ --config-path "$port.config" \ --report-stats no diff --git a/docs/development/demo.md b/docs/development/demo.md index 4277252ceb60..893ed6998ebb 100644 --- a/docs/development/demo.md +++ b/docs/development/demo.md @@ -5,7 +5,7 @@ Requires you to have a [Synapse development environment setup](https://matrix-org.github.io/synapse/develop/development/contributing_guide.html#4-install-the-dependencies). The demo setup allows running three federation Synapse servers, with server -names `localhost:8080`, `localhost:8081`, and `localhost:8082`. +names `localhost:8480`, `localhost:8481`, and `localhost:8482`. You can access them via any Matrix client over HTTP at `localhost:8080`, `localhost:8081`, and `localhost:8082` or over HTTPS at `localhost:8480`, @@ -20,9 +20,10 @@ and the servers are configured in a highly insecure way, including: The servers are configured to store their data under `demo/8080`, `demo/8081`, and `demo/8082`. This includes configuration, logs, SQLite databases, and media. -Note that when joining a public room on a different HS via "#foo:bar.net", then -you are (in the current impl) joining a room with room_id "foo". This means that -it won't work if your HS already has a room with that name. +Note that when joining a public room on a different homeserver via "#foo:bar.net", +then you are (in the current implementation) joining a room with room_id "foo". +This means that it won't work if your homeserver already has a room with that +name. ## Using the demo scripts From 6ff99e3bea481790782c252c5433e9a88f65c4b0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 18 May 2022 20:10:21 +0100 Subject: [PATCH 227/263] Downgrade some OIDC exceptions to warnings (#12723) --- changelog.d/12723.misc | 1 + synapse/handlers/oidc.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12723.misc diff --git a/changelog.d/12723.misc b/changelog.d/12723.misc new file mode 100644 index 000000000000..4f5bffeda639 --- /dev/null +++ b/changelog.d/12723.misc @@ -0,0 +1 @@ +Downgrade some OIDC errors to warnings in the logs, to reduce the noise of Sentry reports. diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index f6ffb7d18d91..9de61d554f41 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -224,7 +224,7 @@ async def handle_oidc_callback(self, request: SynapseRequest) -> None: self._sso_handler.render_error(request, "invalid_session", str(e)) return except MacaroonInvalidSignatureException as e: - logger.exception("Could not verify session for OIDC callback") + logger.warning("Could not verify session for OIDC callback: %s", e) self._sso_handler.render_error(request, "mismatching_session", str(e)) return @@ -827,7 +827,7 @@ async def handle_oidc_callback( logger.debug("Exchanging OAuth2 code for a token") token = await self._exchange_code(code) except OidcError as e: - logger.exception("Could not exchange OAuth2 code") + logger.warning("Could not exchange OAuth2 code: %s", e) self._sso_handler.render_error(request, e.error, e.error_description) return From 5675cebfaa511b6a93db15dd3db3b7f27ce0677a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 19 May 2022 10:28:18 +0100 Subject: [PATCH 228/263] openid.md: fix some links docbook doesn't auto-linkify links --- docs/openid.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/openid.md b/docs/openid.md index e899db63d63b..9d615a573759 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -293,7 +293,7 @@ can be used to retrieve information on the authenticated user. As the Synapse login mechanism needs an attribute to uniquely identify users, and that endpoint does not return a `sub` property, an alternative `subject_claim` has to be set. -1. Create a new OAuth application: https://github.com/settings/applications/new. +1. Create a new OAuth application: [https://github.com/settings/applications/new](https://github.com/settings/applications/new). 2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`. Synapse config: @@ -322,10 +322,10 @@ oidc_providers: [Google][google-idp] is an OpenID certified authentication and authorisation provider. -1. Set up a project in the Google API Console (see - https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup). -2. Add an "OAuth Client ID" for a Web Application under "Credentials". -3. Copy the Client ID and Client Secret, and add the following to your synapse config: +1. Set up a project in the Google API Console (see + [documentation](https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup)). +3. Add an "OAuth Client ID" for a Web Application under "Credentials". +4. Copy the Client ID and Client Secret, and add the following to your synapse config: ```yaml oidc_providers: - idp_id: google @@ -501,8 +501,8 @@ As well as the private key file, you will need: * Team ID: a 10-character ID associated with your developer account. * Key ID: the 10-character identifier for the key. -https://help.apple.com/developer-account/?lang=en#/dev77c875b7e has more -documentation on setting up SiWA. +[Apple's developer documentation](https://help.apple.com/developer-account/?lang=en#/dev77c875b7e) +has more information on setting up SiWA. The synapse config will look like this: @@ -535,8 +535,8 @@ needed to add OAuth2 capabilities to your Django projects. It supports Configuration on Django's side: -1. Add an application: https://example.com/admin/oauth2_provider/application/add/ and choose parameters like this: -* `Redirect uris`: https://synapse.example.com/_synapse/client/oidc/callback +1. Add an application: `https://example.com/admin/oauth2_provider/application/add/` and choose parameters like this: +* `Redirect uris`: `https://synapse.example.com/_synapse/client/oidc/callback` * `Client type`: `Confidential` * `Authorization grant type`: `Authorization code` * `Algorithm`: `HMAC with SHA-2 256` From 47619017f97e04733e2a2fe511d6865d0294f063 Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Thu, 19 May 2022 14:03:12 +0300 Subject: [PATCH 229/263] Add missing user directory search endpoint to the generic worker documentation (#12773) Signed-off-by: Sami Olmari --- changelog.d/12773.doc | 1 + docs/workers.md | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 changelog.d/12773.doc diff --git a/changelog.d/12773.doc b/changelog.d/12773.doc new file mode 100644 index 000000000000..6de371653427 --- /dev/null +++ b/changelog.d/12773.doc @@ -0,0 +1 @@ +Add missing user directory endpoint from the generic worker documentation. Contributed by @olmari. \ No newline at end of file diff --git a/docs/workers.md b/docs/workers.md index 553792d2384c..779069b8177f 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -251,6 +251,8 @@ information. # Presence requests ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ + # User directory search requests + ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ Additionally, the following REST endpoints can be handled for GET requests: @@ -448,6 +450,14 @@ update_user_directory_from_worker: worker_name This work cannot be load-balanced; please ensure the main process is restarted after setting this option in the shared configuration! +User directory updates allow REST endpoints matching the following regular +expressions to work: + + ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ + +The above endpoints can be routed to any worker, though you may choose to route +it to the chosen user directory worker. + This style of configuration supersedes the legacy `synapse.app.user_dir` worker application type. From d25935cd3d92a77a43719df8f6763be0cbcf665b Mon Sep 17 00:00:00 2001 From: Aminda Suomalainen Date: Thu, 19 May 2022 14:28:10 +0300 Subject: [PATCH 230/263] Implement MSC3818: copy room type on upgrade (#12786) Resolves: #11896 Signed-off-by: Aminda Suomalainen --- changelog.d/12786.feature | 1 + synapse/handlers/room.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/12786.feature diff --git a/changelog.d/12786.feature b/changelog.d/12786.feature new file mode 100644 index 000000000000..c90ddd411ee2 --- /dev/null +++ b/changelog.d/12786.feature @@ -0,0 +1 @@ +Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 53569e521219..b7d64a2f5a29 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -469,14 +469,14 @@ async def clone_existing_room( (EventTypes.PowerLevels, ""), ] - # If the old room was a space, copy over the room type and the rooms in - # the space. - if ( - old_room_create_event.content.get(EventContentFields.ROOM_TYPE) - == RoomTypes.SPACE - ): - creation_content[EventContentFields.ROOM_TYPE] = RoomTypes.SPACE - types_to_copy.append((EventTypes.SpaceChild, None)) + # Copy the room type as per MSC3818. + room_type = old_room_create_event.content.get(EventContentFields.ROOM_TYPE) + if room_type is not None: + creation_content[EventContentFields.ROOM_TYPE] = room_type + + # If the old room was a space, copy over the rooms in the space. + if room_type == RoomTypes.SPACE: + types_to_copy.append((EventTypes.SpaceChild, None)) old_room_state_ids = await self.store.get_filtered_current_state_ids( old_room_id, StateFilter.from_types(types_to_copy) From b935c9529c1193416621a18567c11ff4d1d5edca Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 19 May 2022 13:49:58 +0100 Subject: [PATCH 231/263] Simplify untyped-defs config in mypy.ini (#12790) --- changelog.d/12790.misc | 1 + mypy.ini | 173 ++++------------------------------------- 2 files changed, 18 insertions(+), 156 deletions(-) create mode 100644 changelog.d/12790.misc diff --git a/changelog.d/12790.misc b/changelog.d/12790.misc new file mode 100644 index 000000000000..b78156cf4e1d --- /dev/null +++ b/changelog.d/12790.misc @@ -0,0 +1 @@ +Simplify `disallow_untyped_defs` config in `mypy.ini`. diff --git a/mypy.ini b/mypy.ini index 4fa020b8764d..df2622df983a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -10,6 +10,7 @@ warn_unreachable = True warn_unused_ignores = True local_partial_types = True no_implicit_optional = True +disallow_untyped_defs = True files = docker/, @@ -86,177 +87,37 @@ exclude = (?x) |tests/utils.py )$ -[mypy-synapse._scripts.*] -disallow_untyped_defs = True - -[mypy-synapse.api.*] -disallow_untyped_defs = True - -[mypy-synapse.app.*] -disallow_untyped_defs = True - -[mypy-synapse.appservice.*] -disallow_untyped_defs = True - -[mypy-synapse.config.*] -disallow_untyped_defs = True - -[mypy-synapse.crypto.*] -disallow_untyped_defs = True - -[mypy-synapse.event_auth] -disallow_untyped_defs = True - -[mypy-synapse.events.*] -disallow_untyped_defs = True - -[mypy-synapse.federation.*] -disallow_untyped_defs = True - [mypy-synapse.federation.transport.client] disallow_untyped_defs = False -[mypy-synapse.groups.*] -disallow_untyped_defs = True - -[mypy-synapse.handlers.*] -disallow_untyped_defs = True - -[mypy-synapse.http.federation.*] -disallow_untyped_defs = True - -[mypy-synapse.http.connectproxyclient] -disallow_untyped_defs = True - -[mypy-synapse.http.proxyagent] -disallow_untyped_defs = True - -[mypy-synapse.http.request_metrics] -disallow_untyped_defs = True - -[mypy-synapse.http.server] -disallow_untyped_defs = True - -[mypy-synapse.logging._remote] -disallow_untyped_defs = True - -[mypy-synapse.logging.context] -disallow_untyped_defs = True +[mypy-synapse.http.client] +disallow_untyped_defs = False -[mypy-synapse.logging.formatter] -disallow_untyped_defs = True +[mypy-synapse.http.matrixfederationclient] +disallow_untyped_defs = False -[mypy-synapse.logging.handlers] -disallow_untyped_defs = True +[mypy-synapse.logging.opentracing] +disallow_untyped_defs = False -[mypy-synapse.metrics.*] -disallow_untyped_defs = True +[mypy-synapse.logging.scopecontextmanager] +disallow_untyped_defs = False [mypy-synapse.metrics._reactor_metrics] +disallow_untyped_defs = False # This module imports select.epoll. That exists on Linux, but doesn't on macOS. # See https://github.com/matrix-org/synapse/pull/11771. warn_unused_ignores = False -[mypy-synapse.module_api.*] -disallow_untyped_defs = True - -[mypy-synapse.notifier] -disallow_untyped_defs = True - -[mypy-synapse.push.*] -disallow_untyped_defs = True - -[mypy-synapse.replication.*] -disallow_untyped_defs = True - -[mypy-synapse.rest.*] -disallow_untyped_defs = True - -[mypy-synapse.server_notices.*] -disallow_untyped_defs = True - -[mypy-synapse.state.*] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.background_updates] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.account_data] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.client_ips] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.directory] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.e2e_room_keys] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.end_to_end_keys] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.event_push_actions] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.events_bg_updates] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.events_worker] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.room] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.room_batch] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.profile] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.stats] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.state_deltas] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.stream] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.transactions] -disallow_untyped_defs = True - -[mypy-synapse.storage.databases.main.user_erasure_store] -disallow_untyped_defs = True - -[mypy-synapse.storage.engines.*] -disallow_untyped_defs = True - -[mypy-synapse.storage.prepare_database] -disallow_untyped_defs = True - -[mypy-synapse.storage.persist_events] -disallow_untyped_defs = True - -[mypy-synapse.storage.state] -disallow_untyped_defs = True - -[mypy-synapse.storage.types] -disallow_untyped_defs = True - -[mypy-synapse.storage.util.*] -disallow_untyped_defs = True - -[mypy-synapse.streams.*] -disallow_untyped_defs = True +[mypy-synapse.util.caches.treecache] +disallow_untyped_defs = False -[mypy-synapse.types] -disallow_untyped_defs = True +[mypy-synapse.server] +disallow_untyped_defs = False -[mypy-synapse.util.*] -disallow_untyped_defs = True +[mypy-synapse.storage.database] +disallow_untyped_defs = False -[mypy-synapse.util.caches.treecache] +[mypy-tests.*] disallow_untyped_defs = False [mypy-tests.handlers.test_user_directory] From f16ec055cc235eed1ae02f7cede99c366fedca5e Mon Sep 17 00:00:00 2001 From: Jae Lo Presti Date: Thu, 19 May 2022 14:03:13 +0100 Subject: [PATCH 232/263] hash_password: raise an error if no config file is specified (#12789) --- changelog.d/12789.misc | 1 + synapse/_scripts/hash_password.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/12789.misc diff --git a/changelog.d/12789.misc b/changelog.d/12789.misc new file mode 100644 index 000000000000..3398d00110c8 --- /dev/null +++ b/changelog.d/12789.misc @@ -0,0 +1 @@ +The `hash_password` script now fails when it is called without specifying a config file. diff --git a/synapse/_scripts/hash_password.py b/synapse/_scripts/hash_password.py index 3aa29de5bd8a..3bed367be29d 100755 --- a/synapse/_scripts/hash_password.py +++ b/synapse/_scripts/hash_password.py @@ -46,14 +46,14 @@ def main() -> None: "Path to server config file. " "Used to read in bcrypt_rounds and password_pepper." ), + required=True, ) args = parser.parse_args() - if "config" in args and args.config: - config = yaml.safe_load(args.config) - bcrypt_rounds = config.get("bcrypt_rounds", bcrypt_rounds) - password_config = config.get("password_config", None) or {} - password_pepper = password_config.get("pepper", password_pepper) + config = yaml.safe_load(args.config) + bcrypt_rounds = config.get("bcrypt_rounds", bcrypt_rounds) + password_config = config.get("password_config", None) or {} + password_pepper = password_config.get("pepper", password_pepper) password = args.password if not password: From 66a5f6c40018018cccffd79aded0850d13efe513 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 19 May 2022 14:16:49 +0100 Subject: [PATCH 233/263] Add a unique index to `state_group_edges` to prevent duplicates being accidentally introduced and the consequential impact to performance. (#12687) --- changelog.d/12687.bugfix | 1 + docs/upgrade.md | 90 +++++++++++++++++++ synapse/storage/background_updates.py | 15 ++++ synapse/storage/databases/state/bg_updates.py | 16 ++++ .../delta/70/08_state_group_edges_unique.sql | 17 ++++ 5 files changed, 139 insertions(+) create mode 100644 changelog.d/12687.bugfix create mode 100644 synapse/storage/schema/state/delta/70/08_state_group_edges_unique.sql diff --git a/changelog.d/12687.bugfix b/changelog.d/12687.bugfix new file mode 100644 index 000000000000..196d9766707a --- /dev/null +++ b/changelog.d/12687.bugfix @@ -0,0 +1 @@ +Add a unique index to `state_group_edges` to prevent duplicates being accidentally introduced and the consequential impact to performance. \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index fa4b3ef5902d..92ca31b2f8de 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -89,6 +89,96 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.60.0 + +## Adding a new unique index to `state_group_edges` could fail if your database is corrupted + +This release of Synapse will add a unique index to the `state_group_edges` table, in order +to prevent accidentally introducing duplicate information (for example, because a database +backup was restored multiple times). + +Duplicate rows being present in this table could cause drastic performance problems; see +[issue 11779](https://github.com/matrix-org/synapse/issues/11779) for more details. + +If your Synapse database already has had duplicate rows introduced into this table, +this could fail, with either of these errors: + + +**On Postgres:** +``` +synapse.storage.background_updates - 623 - INFO - background_updates-0 - Adding index state_group_edges_unique_idx to state_group_edges +synapse.storage.background_updates - 282 - ERROR - background_updates-0 - Error doing update +... +psycopg2.errors.UniqueViolation: could not create unique index "state_group_edges_unique_idx" +DETAIL: Key (state_group, prev_state_group)=(2, 1) is duplicated. +``` +(The numbers may be different.) + +**On SQLite:** +``` +synapse.storage.background_updates - 623 - INFO - background_updates-0 - Adding index state_group_edges_unique_idx to state_group_edges +synapse.storage.background_updates - 282 - ERROR - background_updates-0 - Error doing update +... +sqlite3.IntegrityError: UNIQUE constraint failed: state_group_edges.state_group, state_group_edges.prev_state_group +``` + + +
+Expand this section for steps to resolve this problem + +### On Postgres + +Connect to your database with `psql`. + +```sql +BEGIN; +DELETE FROM state_group_edges WHERE (ctid, state_group, prev_state_group) IN ( + SELECT row_id, state_group, prev_state_group + FROM ( + SELECT + ctid AS row_id, + MIN(ctid) OVER (PARTITION BY state_group, prev_state_group) AS min_row_id, + state_group, + prev_state_group + FROM state_group_edges + ) AS t1 + WHERE row_id <> min_row_id +); +COMMIT; +``` + + +### On SQLite + +At the command-line, use `sqlite3 path/to/your-homeserver-database.db`: + +```sql +BEGIN; +DELETE FROM state_group_edges WHERE (rowid, state_group, prev_state_group) IN ( + SELECT row_id, state_group, prev_state_group + FROM ( + SELECT + rowid AS row_id, + MIN(rowid) OVER (PARTITION BY state_group, prev_state_group) AS min_row_id, + state_group, + prev_state_group + FROM state_group_edges + ) + WHERE row_id <> min_row_id +); +COMMIT; +``` + + +### For more details + +[This comment on issue 11779](https://github.com/matrix-org/synapse/issues/11779#issuecomment-1131545970) +has queries that can be used to check a database for this problem in advance. + +
+ + + # Upgrading to v1.59.0 ## Device name lookup over federation has been disabled by default diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 37f2d6c644f4..b1e5208c7603 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -535,6 +535,7 @@ def register_background_index_update( where_clause: Optional[str] = None, unique: bool = False, psql_only: bool = False, + replaces_index: Optional[str] = None, ) -> None: """Helper for store classes to do a background index addition @@ -554,6 +555,8 @@ def register_background_index_update( unique: true to make a UNIQUE index psql_only: true to only create this index on psql databases (useful for virtual sqlite tables) + replaces_index: The name of an index that this index replaces. + The named index will be dropped upon completion of the new index. """ def create_index_psql(conn: Connection) -> None: @@ -585,6 +588,12 @@ def create_index_psql(conn: Connection) -> None: } logger.debug("[SQL] %s", sql) c.execute(sql) + + if replaces_index is not None: + # We drop the old index as the new index has now been created. + sql = f"DROP INDEX IF EXISTS {replaces_index}" + logger.debug("[SQL] %s", sql) + c.execute(sql) finally: conn.set_session(autocommit=False) # type: ignore @@ -613,6 +622,12 @@ def create_index_sqlite(conn: Connection) -> None: logger.debug("[SQL] %s", sql) c.execute(sql) + if replaces_index is not None: + # We drop the old index as the new index has now been created. + sql = f"DROP INDEX IF EXISTS {replaces_index}" + logger.debug("[SQL] %s", sql) + c.execute(sql) + if isinstance(self.db_pool.engine, engines.PostgresEngine): runner: Optional[Callable[[Connection], None]] = create_index_psql elif psql_only: diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 5de70f31d294..fa9eadaca7ea 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -195,6 +195,7 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication" STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index" STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" + STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" def __init__( self, @@ -217,6 +218,21 @@ def __init__( columns=["room_id"], ) + # `state_group_edges` can cause severe performance issues if duplicate + # rows are introduced, which can accidentally be done by well-meaning + # server admins when trying to restore a database dump, etc. + # See https://github.com/matrix-org/synapse/issues/11779. + # Introduce a unique index to guard against that. + self.db_pool.updates.register_background_index_update( + self.STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME, + index_name="state_group_edges_unique_idx", + table="state_group_edges", + columns=["state_group", "prev_state_group"], + unique=True, + # The old index was on (state_group) and was not unique. + replaces_index="state_group_edges_idx", + ) + async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/state/delta/70/08_state_group_edges_unique.sql b/synapse/storage/schema/state/delta/70/08_state_group_edges_unique.sql new file mode 100644 index 000000000000..b8c0ee0fa03e --- /dev/null +++ b/synapse/storage/schema/state/delta/70/08_state_group_edges_unique.sql @@ -0,0 +1,17 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (7008, 'state_group_edges_unique_idx', '{}'); From 684feeaf2f32d853b02951794789830e48e75a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Van=C4=9Bk?= Date: Thu, 19 May 2022 16:23:59 +0200 Subject: [PATCH 234/263] Properly close providers.json file stream. (#12794) --- changelog.d/12794.bugfix | 1 + synapse/config/oembed.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/12794.bugfix diff --git a/changelog.d/12794.bugfix b/changelog.d/12794.bugfix new file mode 100644 index 000000000000..2d1a2838e128 --- /dev/null +++ b/changelog.d/12794.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in 1.43.0 where a file (`providers.json`) was never closed. Contributed by @arkamar. diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py index 690ffb52963e..e9edea073123 100644 --- a/synapse/config/oembed.py +++ b/synapse/config/oembed.py @@ -57,9 +57,9 @@ def _parse_and_validate_providers( """ # Whether to use the packaged providers.json file. if not oembed_config.get("disable_default_providers") or False: - providers = json.load( - pkg_resources.resource_stream("synapse", "res/providers.json") - ) + with pkg_resources.resource_stream("synapse", "res/providers.json") as s: + providers = json.load(s) + yield from self._parse_and_validate_provider( providers, config_path=("oembed",) ) From ab2a615cfb13f8ff91919c4332fcb182640d5484 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 19 May 2022 07:46:33 -0700 Subject: [PATCH 235/263] Update configuration manual to document size-related suffixes (#12777) --- changelog.d/12777.doc | 2 ++ docs/usage/configuration/config_documentation.md | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 changelog.d/12777.doc diff --git a/changelog.d/12777.doc b/changelog.d/12777.doc new file mode 100644 index 000000000000..cc9c07704d02 --- /dev/null +++ b/changelog.d/12777.doc @@ -0,0 +1,2 @@ +Update configuration manual documentation to document size-related suffixes. + diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 6b9ffc09d2b3..525e1c7a9145 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -23,6 +23,14 @@ followed by a letter. Letters have the following meanings: For example, setting `redaction_retention_period: 5m` would remove redacted messages from the database after 5 minutes, rather than 5 months. +In addition, configuration options referring to size use the following suffixes: + +* `M` = MiB, or 1,048,576 bytes +* `K` = KiB, or 1024 bytes + +For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes +for a user avatar. + ### YAML The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules apply if you want your config file to be read properly. A few helpful things to know: From eb4aaa1b4b828c7d2ab501f03ebe79b13c75b7e0 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 19 May 2022 07:47:07 -0700 Subject: [PATCH 236/263] Add detail to `cache_autotuning` config option documentation (#12776) --- changelog.d/12776.doc | 2 ++ docs/usage/configuration/config_documentation.md | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12776.doc diff --git a/changelog.d/12776.doc b/changelog.d/12776.doc new file mode 100644 index 000000000000..c00489a8ce14 --- /dev/null +++ b/changelog.d/12776.doc @@ -0,0 +1,2 @@ +Add additional info to documentation of config option `cache_autotuning`. + diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 525e1c7a9145..0f5bda32b941 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1130,14 +1130,19 @@ Caching can be configured through the following sub-options: * `cache_autotuning` and its sub-options `max_cache_memory_usage`, `target_cache_memory_usage`, and `min_cache_ttl` work in conjunction with each other to maintain a balance between cache memory usage and cache entry availability. You must be using [jemalloc](https://github.com/matrix-org/synapse#help-synapse-is-slow-and-eats-all-my-ramcpu) - to utilize this option, and all three of the options must be specified for this feature to work. + to utilize this option, and all three of the options must be specified for this feature to work. This option + defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work + and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. + Please see the [Config Conventions](#config-conventions) for information on how to specify memory size and cache expiry + durations. * `max_cache_memory_usage` sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in - the flag below, or until the `min_cache_ttl` is hit. - * `target_memory_usage` sets a rough target for the desired memory usage of the caches. + the setting below, or until the `min_cache_ttl` is hit. There is no default value for this option. + * `target_memory_usage` sets a rough target for the desired memory usage of the caches. There is no default value + for this option. * `min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches - from being emptied while Synapse is evicting due to memory. + from being emptied while Synapse is evicting due to memory. There is no default value for this option. Example configuration: ```yaml From 177b884ad7cc1ecdd92ff74188732734df203150 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 19 May 2022 16:29:08 +0100 Subject: [PATCH 237/263] Lay some foundation work to allow workers to only subscribe to some kinds of messages, reducing replication traffic. (#12672) --- changelog.d/12672.misc | 1 + synapse/replication/tcp/handler.py | 34 ++++++++++++- synapse/replication/tcp/redis.py | 35 +++++++++---- tests/replication/_base.py | 54 +++++++++++++++----- tests/replication/tcp/test_handler.py | 73 +++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 changelog.d/12672.misc create mode 100644 tests/replication/tcp/test_handler.py diff --git a/changelog.d/12672.misc b/changelog.d/12672.misc new file mode 100644 index 000000000000..265e0a801f78 --- /dev/null +++ b/changelog.d/12672.misc @@ -0,0 +1 @@ +Lay some foundation work to allow workers to only subscribe to some kinds of messages, reducing replication traffic. \ No newline at end of file diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 9aba1cd45111..e1cbfa50ebd2 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -1,5 +1,5 @@ # Copyright 2017 Vector Creations Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. +# Copyright 2020, 2022 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,6 +101,9 @@ def __init__(self, hs: "HomeServer"): self._instance_id = hs.get_instance_id() self._instance_name = hs.get_instance_name() + # Additional Redis channel suffixes to subscribe to. + self._channels_to_subscribe_to: List[str] = [] + self._is_presence_writer = ( hs.get_instance_name() in hs.config.worker.writers.presence ) @@ -243,6 +246,31 @@ def __init__(self, hs: "HomeServer"): # If we're NOT using Redis, this must be handled by the master self._should_insert_client_ips = hs.get_instance_name() == "master" + if self._is_master or self._should_insert_client_ips: + self.subscribe_to_channel("USER_IP") + + def subscribe_to_channel(self, channel_name: str) -> None: + """ + Indicates that we wish to subscribe to a Redis channel by name. + + (The name will later be prefixed with the server name; i.e. subscribing + to the 'ABC' channel actually subscribes to 'example.com/ABC' Redis-side.) + + Raises: + - If replication has already started, then it's too late to subscribe + to new channels. + """ + + if self._factory is not None: + # We don't allow subscribing after the fact to avoid the chance + # of missing an important message because we didn't subscribe in time. + raise RuntimeError( + "Cannot subscribe to more channels after replication started." + ) + + if channel_name not in self._channels_to_subscribe_to: + self._channels_to_subscribe_to.append(channel_name) + def _add_command_to_stream_queue( self, conn: IReplicationConnection, cmd: Union[RdataCommand, PositionCommand] ) -> None: @@ -321,7 +349,9 @@ def start_replication(self, hs: "HomeServer") -> None: # Now create the factory/connection for the subscription stream. self._factory = RedisDirectTcpReplicationClientFactory( - hs, outbound_redis_connection + hs, + outbound_redis_connection, + channel_names=self._channels_to_subscribe_to, ) hs.get_reactor().connectTCP( hs.config.redis.redis_host, diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 989c5be0327e..73294654eff1 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -14,7 +14,7 @@ import logging from inspect import isawaitable -from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, List, Optional, Type, TypeVar, cast import attr import txredisapi @@ -85,14 +85,15 @@ class RedisSubscriber(txredisapi.SubscriberProtocol): Attributes: synapse_handler: The command handler to handle incoming commands. - synapse_stream_name: The *redis* stream name to subscribe to and publish + synapse_stream_prefix: The *redis* stream name to subscribe to and publish from (not anything to do with Synapse replication streams). synapse_outbound_redis_connection: The connection to redis to use to send commands. """ synapse_handler: "ReplicationCommandHandler" - synapse_stream_name: str + synapse_stream_prefix: str + synapse_channel_names: List[str] synapse_outbound_redis_connection: txredisapi.ConnectionHandler def __init__(self, *args: Any, **kwargs: Any): @@ -117,8 +118,13 @@ async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we # have successfully subscribed to the stream - otherwise we might miss the # POSITION response sent back by the other end. - logger.info("Sending redis SUBSCRIBE for %s", self.synapse_stream_name) - await make_deferred_yieldable(self.subscribe(self.synapse_stream_name)) + fully_qualified_stream_names = [ + f"{self.synapse_stream_prefix}/{stream_suffix}" + for stream_suffix in self.synapse_channel_names + ] + [self.synapse_stream_prefix] + logger.info("Sending redis SUBSCRIBE for %r", fully_qualified_stream_names) + await make_deferred_yieldable(self.subscribe(fully_qualified_stream_names)) + logger.info( "Successfully subscribed to redis stream, sending REPLICATE command" ) @@ -217,7 +223,7 @@ async def _async_send_command(self, cmd: Command) -> None: await make_deferred_yieldable( self.synapse_outbound_redis_connection.publish( - self.synapse_stream_name, encoded_string + self.synapse_stream_prefix, encoded_string ) ) @@ -300,20 +306,27 @@ def format_address(address: IAddress) -> str: class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): """This is a reconnecting factory that connects to redis and immediately - subscribes to a stream. + subscribes to some streams. Args: hs outbound_redis_connection: A connection to redis that will be used to send outbound commands (this is separate to the redis connection used to subscribe). + channel_names: A list of channel names to append to the base channel name + to additionally subscribe to. + e.g. if ['ABC', 'DEF'] is specified then we'll listen to: + example.com; example.com/ABC; and example.com/DEF. """ maxDelay = 5 protocol = RedisSubscriber def __init__( - self, hs: "HomeServer", outbound_redis_connection: txredisapi.ConnectionHandler + self, + hs: "HomeServer", + outbound_redis_connection: txredisapi.ConnectionHandler, + channel_names: List[str], ): super().__init__( @@ -326,7 +339,8 @@ def __init__( ) self.synapse_handler = hs.get_replication_command_handler() - self.synapse_stream_name = hs.hostname + self.synapse_stream_prefix = hs.hostname + self.synapse_channel_names = channel_names self.synapse_outbound_redis_connection = outbound_redis_connection @@ -340,7 +354,8 @@ def buildProtocol(self, addr: IAddress) -> RedisSubscriber: # protocol. p.synapse_handler = self.synapse_handler p.synapse_outbound_redis_connection = self.synapse_outbound_redis_connection - p.synapse_stream_name = self.synapse_stream_name + p.synapse_stream_prefix = self.synapse_stream_prefix + p.synapse_channel_names = self.synapse_channel_names return p diff --git a/tests/replication/_base.py b/tests/replication/_base.py index a7602b4c96ae..970d5e533b35 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Tuple +from collections import defaultdict +from typing import Any, Dict, List, Optional, Set, Tuple from twisted.internet.address import IPv4Address from twisted.internet.protocol import Protocol @@ -32,6 +33,7 @@ from tests import unittest from tests.server import FakeTransport +from tests.utils import USE_POSTGRES_FOR_TESTS try: import hiredis @@ -475,22 +477,25 @@ class FakeRedisPubSubServer: """A fake Redis server for pub/sub.""" def __init__(self): - self._subscribers = set() + self._subscribers_by_channel: Dict[ + bytes, Set["FakeRedisPubSubProtocol"] + ] = defaultdict(set) - def add_subscriber(self, conn): + def add_subscriber(self, conn, channel: bytes): """A connection has called SUBSCRIBE""" - self._subscribers.add(conn) + self._subscribers_by_channel[channel].add(conn) def remove_subscriber(self, conn): - """A connection has called UNSUBSCRIBE""" - self._subscribers.discard(conn) + """A connection has lost connection""" + for subscribers in self._subscribers_by_channel.values(): + subscribers.discard(conn) - def publish(self, conn, channel, msg) -> int: + def publish(self, conn, channel: bytes, msg) -> int: """A connection want to publish a message to subscribers.""" - for sub in self._subscribers: + for sub in self._subscribers_by_channel[channel]: sub.send(["message", channel, msg]) - return len(self._subscribers) + return len(self._subscribers_by_channel) def buildProtocol(self, addr): return FakeRedisPubSubProtocol(self) @@ -531,9 +536,10 @@ def handle_command(self, command, *args): num_subscribers = self._server.publish(self, channel, message) self.send(num_subscribers) elif command == b"SUBSCRIBE": - (channel,) = args - self._server.add_subscriber(self) - self.send(["subscribe", channel, 1]) + for idx, channel in enumerate(args): + num_channels = idx + 1 + self._server.add_subscriber(self, channel) + self.send(["subscribe", channel, num_channels]) # Since we use SET/GET to cache things we can safely no-op them. elif command == b"SET": @@ -576,3 +582,27 @@ def encode(self, obj): def connectionLost(self, reason): self._server.remove_subscriber(self) + + +class RedisMultiWorkerStreamTestCase(BaseMultiWorkerStreamTestCase): + """ + A test case that enables Redis, providing a fake Redis server. + """ + + if not hiredis: + skip = "Requires hiredis" + + if not USE_POSTGRES_FOR_TESTS: + # Redis replication only takes place on Postgres + skip = "Requires Postgres" + + def default_config(self) -> Dict[str, Any]: + """ + Overrides the default config to enable Redis. + Even if the test only uses make_worker_hs, the main process needs Redis + enabled otherwise it won't create a Fake Redis server to listen on the + Redis port and accept fake TCP connections. + """ + base = super().default_config() + base["redis"] = {"enabled": True} + return base diff --git a/tests/replication/tcp/test_handler.py b/tests/replication/tcp/test_handler.py new file mode 100644 index 000000000000..e6a19eafd578 --- /dev/null +++ b/tests/replication/tcp/test_handler.py @@ -0,0 +1,73 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests.replication._base import RedisMultiWorkerStreamTestCase + + +class ChannelsTestCase(RedisMultiWorkerStreamTestCase): + def test_subscribed_to_enough_redis_channels(self) -> None: + # The default main process is subscribed to the USER_IP channel. + self.assertCountEqual( + self.hs.get_replication_command_handler()._channels_to_subscribe_to, + ["USER_IP"], + ) + + def test_background_worker_subscribed_to_user_ip(self) -> None: + # The default main process is subscribed to the USER_IP channel. + worker1 = self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "worker_name": "worker1", + "run_background_tasks_on": "worker1", + "redis": {"enabled": True}, + }, + ) + self.assertIn( + "USER_IP", + worker1.get_replication_command_handler()._channels_to_subscribe_to, + ) + + # Advance so the Redis subscription gets processed + self.pump(0.1) + + # The counts are 2 because both the main process and the worker are subscribed. + self.assertEqual(len(self._redis_server._subscribers_by_channel[b"test"]), 2) + self.assertEqual( + len(self._redis_server._subscribers_by_channel[b"test/USER_IP"]), 2 + ) + + def test_non_background_worker_not_subscribed_to_user_ip(self) -> None: + # The default main process is subscribed to the USER_IP channel. + worker2 = self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "worker_name": "worker2", + "run_background_tasks_on": "worker1", + "redis": {"enabled": True}, + }, + ) + self.assertNotIn( + "USER_IP", + worker2.get_replication_command_handler()._channels_to_subscribe_to, + ) + + # Advance so the Redis subscription gets processed + self.pump(0.1) + + # The count is 2 because both the main process and the worker are subscribed. + self.assertEqual(len(self._redis_server._subscribers_by_channel[b"test"]), 2) + # For USER_IP, the count is 1 because only the main process is subscribed. + self.assertEqual( + len(self._redis_server._subscribers_by_channel[b"test/USER_IP"]), 1 + ) From 96df31239cdbcd4f50f503bf329fe3bb86c39a20 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 19 May 2022 18:32:48 +0100 Subject: [PATCH 238/263] Add a unit test for copying over arbitrary room types when upgrading a room (#12792) --- changelog.d/12792.feature | 1 + synapse/handlers/room.py | 2 +- tests/rest/client/test_upgrade_room.py | 32 +++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12792.feature diff --git a/changelog.d/12792.feature b/changelog.d/12792.feature new file mode 100644 index 000000000000..4778b8a394d4 --- /dev/null +++ b/changelog.d/12792.feature @@ -0,0 +1 @@ +Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). \ No newline at end of file diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b7d64a2f5a29..794f94f6b353 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -427,7 +427,7 @@ async def clone_existing_room( requester: the user requesting the upgrade old_room_id : the id of the room to be replaced new_room_id: the id to give the new room (should already have been - created with _gemerate_room_id()) + created with _generate_room_id()) new_room_version: the new room version to use tombstone_event_id: the ID of the tombstone event in the old room. """ diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index c86fc5df0d08..a21cbe9fa874 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -76,7 +76,7 @@ def test_not_in_room(self) -> None: """ Upgrading a room should work fine. """ - # THe user isn't in the room. + # The user isn't in the room. roomless = self.register_user("roomless", "pass") roomless_token = self.login(roomless, "pass") @@ -263,3 +263,33 @@ def test_space(self) -> None: self.assertIn((EventTypes.SpaceChild, self.room_id), state_ids) # The child that was removed should not be copied over. self.assertNotIn((EventTypes.SpaceChild, old_room_id), state_ids) + + def test_custom_room_type(self) -> None: + """Test upgrading a room that has a custom room type set.""" + test_room_type = "com.example.my_custom_room_type" + + # Create a room with a custom room type. + room_id = self.helper.create_room_as( + self.creator, + tok=self.creator_token, + extra_content={ + "creation_content": {EventContentFields.ROOM_TYPE: test_room_type} + }, + ) + + # Upgrade the room! + channel = self._upgrade_room(room_id=room_id) + self.assertEqual(200, channel.code, channel.result) + self.assertIn("replacement_room", channel.json_body) + + new_room_id = channel.json_body["replacement_room"] + + state_ids = self.get_success(self.store.get_current_state_ids(new_room_id)) + + # Ensure the new room is the same type as the old room. + create_event = self.get_success( + self.store.get_event(state_ids[(EventTypes.Create, "")]) + ) + self.assertEqual( + create_event.content.get(EventContentFields.ROOM_TYPE), test_room_type + ) From 2be5a2b07becdbf0957008ece74be3b3f957b233 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 19 May 2022 20:17:10 +0100 Subject: [PATCH 239/263] Fix `RetryDestinationLimiter` re-starting finished log contexts (#12803) Signed-off-by: Sean Quah --- changelog.d/12803.bugfix | 1 + synapse/util/retryutils.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12803.bugfix diff --git a/changelog.d/12803.bugfix b/changelog.d/12803.bugfix new file mode 100644 index 000000000000..6ddd3d24e05f --- /dev/null +++ b/changelog.d/12803.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where finished log contexts would be re-started when failing to contact remote homeservers. diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 81bfed268ee7..d0a69ff843e5 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -16,8 +16,8 @@ from types import TracebackType from typing import TYPE_CHECKING, Any, Optional, Type -import synapse.logging.context from synapse.api.errors import CodeMessageException +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage import DataStore from synapse.util import Clock @@ -265,4 +265,4 @@ async def store_retry_timings() -> None: logger.exception("Failed to store destination_retry_timings") # we deliberately do this in the background. - synapse.logging.context.run_in_background(store_retry_timings) + run_as_background_process("store_retry_timings", store_retry_timings) From 71e8afe34d2103c5ccc9f2d1c99587d14b2acc56 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 20 May 2022 01:54:12 -0700 Subject: [PATCH 240/263] Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible (#12791) --- changelog.d/12791.misc | 1 + synapse/events/snapshot.py | 19 +++++++++++++++---- synapse/handlers/federation.py | 9 +++++++-- synapse/handlers/federation_event.py | 8 +++++++- synapse/handlers/message.py | 14 +++++++++++--- synapse/handlers/room.py | 5 ++++- synapse/handlers/room_member.py | 9 +++++++-- synapse/push/bulk_push_rule_evaluator.py | 9 +++++++-- synapse/storage/state.py | 7 +++++-- tests/test_state.py | 2 +- 10 files changed, 65 insertions(+), 18 deletions(-) create mode 100644 changelog.d/12791.misc diff --git a/changelog.d/12791.misc b/changelog.d/12791.misc new file mode 100644 index 000000000000..b6e92b7eafad --- /dev/null +++ b/changelog.d/12791.misc @@ -0,0 +1 @@ +Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 9ccd24b298bb..7a91544119f7 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from synapse.storage import Storage from synapse.storage.databases.main import DataStore + from synapse.storage.state import StateFilter @attr.s(slots=True, auto_attribs=True) @@ -196,7 +197,9 @@ def state_group(self) -> Optional[int]: return self._state_group - async def get_current_state_ids(self) -> Optional[StateMap[str]]: + async def get_current_state_ids( + self, state_filter: Optional["StateFilter"] = None + ) -> Optional[StateMap[str]]: """ Gets the room state map, including this event - ie, the state in ``state_group`` @@ -204,6 +207,9 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: not make it into the room state. This method will raise an exception if ``rejected`` is set. + Arg: + state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules + Returns: Returns None if state_group is None, which happens when the associated event is an outlier. @@ -216,7 +222,7 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: assert self._state_delta_due_to_event is not None - prev_state_ids = await self.get_prev_state_ids() + prev_state_ids = await self.get_prev_state_ids(state_filter) if self._state_delta_due_to_event: prev_state_ids = dict(prev_state_ids) @@ -224,12 +230,17 @@ async def get_current_state_ids(self) -> Optional[StateMap[str]]: return prev_state_ids - async def get_prev_state_ids(self) -> StateMap[str]: + async def get_prev_state_ids( + self, state_filter: Optional["StateFilter"] = None + ) -> StateMap[str]: """ Gets the room state map, excluding this event. For a non-state event, this will be the same as get_current_state_ids(). + Args: + state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules + Returns: Returns {} if state_group is None, which happens when the associated event is an outlier. @@ -239,7 +250,7 @@ async def get_prev_state_ids(self) -> StateMap[str]: """ assert self.state_group_before_event is not None return await self._storage.state.get_state_ids_for_group( - self.state_group_before_event + self.state_group_before_event, state_filter ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index be5099b507f6..0386d0a07bba 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -54,6 +54,7 @@ ReplicationStoreRoomOnOutlierMembershipRestServlet, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour +from synapse.storage.state import StateFilter from synapse.types import JsonDict, StateMap, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.retryutils import NotRetryingDestination @@ -1259,7 +1260,9 @@ async def add_display_name_to_third_party_invite( event.content["third_party_invite"]["signed"]["token"], ) original_invite = None - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.ThirdPartyInvite, None)]) + ) original_invite_id = prev_state_ids.get(key) if original_invite_id: original_invite = await self.store.get_event( @@ -1308,7 +1311,9 @@ async def _check_signature(self, event: EventBase, context: EventContext) -> Non signed = event.content["third_party_invite"]["signed"] token = signed["token"] - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.ThirdPartyInvite, None)]) + ) invite_event_id = prev_state_ids.get((EventTypes.ThirdPartyInvite, token)) invite_event = None diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 761caa04b726..05c122f22491 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -30,6 +30,7 @@ from prometheus_client import Counter +from synapse import event_auth from synapse.api.constants import ( EventContentFields, EventTypes, @@ -63,6 +64,7 @@ ) from synapse.state import StateResolutionStore from synapse.storage.databases.main.events_worker import EventRedactBehaviour +from synapse.storage.state import StateFilter from synapse.types import ( PersistedEventPosition, RoomStreamToken, @@ -1500,7 +1502,11 @@ async def _check_event_auth( return context # now check auth against what we think the auth events *should* be. - prev_state_ids = await context.get_prev_state_ids() + event_types = event_auth.auth_types_for_event(event.room_version, event) + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types(event_types) + ) + auth_events_ids = self._event_auth_handler.compute_auth_events( event, prev_state_ids, for_verification=True ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 0951b9c71f75..e566ff1f8ed8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -634,7 +634,9 @@ async def create_event( # federation as well as those created locally. As of room v3, aliases events # can be created by users that are not in the room, therefore we have to # tolerate them in event_auth.check(). - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.Member, None)]) + ) prev_event_id = prev_state_ids.get((EventTypes.Member, event.sender)) prev_event = ( await self.store.get_event(prev_event_id, allow_none=True) @@ -761,7 +763,9 @@ async def deduplicate_state_event( # This can happen due to out of band memberships return None - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(event.type, None)]) + ) prev_event_id = prev_state_ids.get((event.type, event.state_key)) if not prev_event_id: return None @@ -1547,7 +1551,11 @@ async def persist_and_notify_client_event( "Redacting MSC2716 events is not supported in this room version", ) - prev_state_ids = await context.get_prev_state_ids() + event_types = event_auth.auth_types_for_event(event.room_version, event) + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types(event_types) + ) + auth_events_ids = self._event_auth_handler.compute_auth_events( event, prev_state_ids, for_verification=True ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 794f94f6b353..92e1de050071 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -303,7 +303,10 @@ async def _upgrade_room( context=tombstone_context, ) - old_room_state = await tombstone_context.get_current_state_ids() + state_filter = StateFilter.from_types( + [(EventTypes.CanonicalAlias, ""), (EventTypes.PowerLevels, "")] + ) + old_room_state = await tombstone_context.get_current_state_ids(state_filter) # We know the tombstone event isn't an outlier so it has current state. assert old_room_state is not None diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 802e57c4d0cc..ea876c168de7 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -38,6 +38,7 @@ from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN +from synapse.storage.state import StateFilter from synapse.types import ( JsonDict, Requester, @@ -362,7 +363,9 @@ async def _local_membership_update( historical=historical, ) - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.Member, None)]) + ) prev_member_event_id = prev_state_ids.get((EventTypes.Member, user_id), None) @@ -1160,7 +1163,9 @@ async def send_membership_event( else: requester = types.create_requester(target_user) - prev_state_ids = await context.get_prev_state_ids() + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types([(EventTypes.GuestAccess, None)]) + ) if event.membership == Membership.JOIN: if requester.is_guest: guest_can_join = await self._can_guest_join(prev_state_ids) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 4ac2c546bf2a..4cc8a2ecca7a 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -20,7 +20,7 @@ from prometheus_client import Counter from synapse.api.constants import EventTypes, Membership, RelationTypes -from synapse.event_auth import get_user_power_level +from synapse.event_auth import auth_types_for_event, get_user_power_level from synapse.events import EventBase, relation_from_event from synapse.events.snapshot import EventContext from synapse.state import POWER_KEY @@ -31,6 +31,7 @@ from synapse.util.caches.lrucache import LruCache from synapse.util.metrics import measure_func +from ..storage.state import StateFilter from .push_rule_evaluator import PushRuleEvaluatorForEvent if TYPE_CHECKING: @@ -168,8 +169,12 @@ def _get_rules_for_room(self, room_id: str) -> "RulesForRoomData": async def _get_power_levels_and_sender_level( self, event: EventBase, context: EventContext ) -> Tuple[dict, int]: - prev_state_ids = await context.get_prev_state_ids() + event_types = auth_types_for_event(event.room_version, event) + prev_state_ids = await context.get_prev_state_ids( + StateFilter.from_types(event_types) + ) pl_event_id = prev_state_ids.get(POWER_KEY) + if pl_event_id: # fastpath: if there's a power level event, that's all we need, and # not having a power level event is an extreme edge case diff --git a/synapse/storage/state.py b/synapse/storage/state.py index e58301a8f087..ab630953ac93 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -634,16 +634,19 @@ async def get_state_groups_ids( return group_to_state - async def get_state_ids_for_group(self, state_group: int) -> StateMap[str]: + async def get_state_ids_for_group( + self, state_group: int, state_filter: Optional[StateFilter] = None + ) -> StateMap[str]: """Get the event IDs of all the state in the given state group Args: state_group: A state group for which we want to get the state IDs. + state_filter: specifies the type of state event to fetch from DB, example: EventTypes.JoinRules Returns: Resolves to a map of (type, state_key) -> event_id """ - group_to_state = await self.get_state_for_groups((state_group,)) + group_to_state = await self.get_state_for_groups((state_group,), state_filter) return group_to_state[state_group] diff --git a/tests/test_state.py b/tests/test_state.py index 74a8ce6096b9..c6baea3d7604 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -88,7 +88,7 @@ async def get_state_groups_ids(self, room_id, event_ids): return groups - async def get_state_ids_for_group(self, state_group): + async def get_state_ids_for_group(self, state_group, state_filter=None): return self._group_to_state[state_group] async def store_state_group( From 10280fc9437038f7ef715873e491d54b0a6d2208 Mon Sep 17 00:00:00 2001 From: David Teller Date: Fri, 20 May 2022 14:53:25 +0200 Subject: [PATCH 241/263] Uniformize spam-checker API, part 1: the `Code` enum. (#12703) --- changelog.d/12703.misc | 1 + synapse/api/errors.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/12703.misc diff --git a/changelog.d/12703.misc b/changelog.d/12703.misc new file mode 100644 index 000000000000..9aaa1bbaa3d0 --- /dev/null +++ b/changelog.d/12703.misc @@ -0,0 +1 @@ +Convert namespace class `Codes` into a string enum. \ No newline at end of file diff --git a/synapse/api/errors.py b/synapse/api/errors.py index cb3b7323d568..9614be6b4e46 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -17,6 +17,7 @@ import logging import typing +from enum import Enum from http import HTTPStatus from typing import Any, Dict, List, Optional, Union @@ -30,7 +31,11 @@ logger = logging.getLogger(__name__) -class Codes: +class Codes(str, Enum): + """ + All known error codes, as an enum of strings. + """ + UNRECOGNIZED = "M_UNRECOGNIZED" UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" @@ -265,7 +270,9 @@ class UnrecognizedRequestError(SynapseError): """An error indicating we don't understand the request you're trying to make""" def __init__( - self, msg: str = "Unrecognized request", errcode: str = Codes.UNRECOGNIZED + self, + msg: str = "Unrecognized request", + errcode: str = Codes.UNRECOGNIZED, ): super().__init__(400, msg, errcode) From 39dee30f0120290d6ef3504815655df1a6cf47a5 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Fri, 20 May 2022 15:28:23 +0100 Subject: [PATCH 242/263] Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. (#12809) --- changelog.d/12672.feature | 1 + changelog.d/12672.misc | 1 - changelog.d/12809.feature | 1 + synapse/replication/tcp/commands.py | 12 ++++++++++++ synapse/replication/tcp/redis.py | 6 +++--- 5 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12672.feature delete mode 100644 changelog.d/12672.misc create mode 100644 changelog.d/12809.feature diff --git a/changelog.d/12672.feature b/changelog.d/12672.feature new file mode 100644 index 000000000000..b989e0d208c4 --- /dev/null +++ b/changelog.d/12672.feature @@ -0,0 +1 @@ +Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. \ No newline at end of file diff --git a/changelog.d/12672.misc b/changelog.d/12672.misc deleted file mode 100644 index 265e0a801f78..000000000000 --- a/changelog.d/12672.misc +++ /dev/null @@ -1 +0,0 @@ -Lay some foundation work to allow workers to only subscribe to some kinds of messages, reducing replication traffic. \ No newline at end of file diff --git a/changelog.d/12809.feature b/changelog.d/12809.feature new file mode 100644 index 000000000000..b989e0d208c4 --- /dev/null +++ b/changelog.d/12809.feature @@ -0,0 +1 @@ +Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. \ No newline at end of file diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index fe34948168ab..32f52e54d8c7 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -58,6 +58,15 @@ def get_logcontext_id(self) -> str: # by default, we just use the command name. return self.NAME + def redis_channel_name(self, prefix: str) -> str: + """ + Returns the Redis channel name upon which to publish this command. + + Args: + prefix: The prefix for the channel. + """ + return prefix + SC = TypeVar("SC", bound="_SimpleCommand") @@ -395,6 +404,9 @@ def __repr__(self) -> str: f"{self.user_agent!r}, {self.device_id!r}, {self.last_seen})" ) + def redis_channel_name(self, prefix: str) -> str: + return f"{prefix}/USER_IP" + class RemoteServerUpCommand(_SimpleCommand): """Sent when a worker has detected that a remote server is no longer diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 73294654eff1..fd1c0ec6afa2 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -221,10 +221,10 @@ async def _async_send_command(self, cmd: Command) -> None: # remote instances. tcp_outbound_commands_counter.labels(cmd.NAME, "redis").inc() + channel_name = cmd.redis_channel_name(self.synapse_stream_prefix) + await make_deferred_yieldable( - self.synapse_outbound_redis_connection.publish( - self.synapse_stream_prefix, encoded_string - ) + self.synapse_outbound_redis_connection.publish(channel_name, encoded_string) ) From fbf904bd54071ca22c8918e0e106dd2fb008d0fb Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 23 May 2022 10:28:56 +0100 Subject: [PATCH 243/263] Fix media thumbnails being unusable before the index had been added in the background. (#12823) --- changelog.d/12823.bugfix | 1 + synapse/storage/database.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/12823.bugfix diff --git a/changelog.d/12823.bugfix b/changelog.d/12823.bugfix new file mode 100644 index 000000000000..1a1f5957e712 --- /dev/null +++ b/changelog.d/12823.bugfix @@ -0,0 +1 @@ +Fix a bug, introduced in Synapse 1.21.0, that led to media thumbnails being unusable before the index has been added in the background. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 5ddb58a8a2ca..a78d68a9d7fe 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -90,6 +90,8 @@ "device_lists_remote_extremeties": "device_lists_remote_extremeties_unique_idx", "device_lists_remote_cache": "device_lists_remote_cache_unique_idx", "event_search": "event_search_event_id_idx", + "local_media_repository_thumbnails": "local_media_repository_thumbnails_method_idx", + "remote_media_cache_thumbnails": "remote_media_repository_thumbnails_method_idx", } From 4fef76ca348209b7c9dd3c17d5f3d8ef12623c1b Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 23 May 2022 10:29:24 +0100 Subject: [PATCH 244/263] Remove Caddy from the Synapse workers image used in Complement. (#12818) --- changelog.d/12818.misc | 1 + docker/complement/SynapseWorkers.Dockerfile | 12 +--- .../conf-workers/caddy.complement.json | 72 ------------------- .../conf-workers/caddy.supervisord.conf | 7 -- .../start-complement-synapse-workers.sh | 18 ++++- docker/conf-workers/nginx.conf.j2 | 16 +++++ docker/configure_workers_and_start.py | 5 ++ 7 files changed, 38 insertions(+), 93 deletions(-) create mode 100644 changelog.d/12818.misc delete mode 100644 docker/complement/conf-workers/caddy.complement.json delete mode 100644 docker/complement/conf-workers/caddy.supervisord.conf diff --git a/changelog.d/12818.misc b/changelog.d/12818.misc new file mode 100644 index 000000000000..2f9dacc21dd9 --- /dev/null +++ b/changelog.d/12818.misc @@ -0,0 +1 @@ +Remove Caddy from the Synapse workers image used in Complement. \ No newline at end of file diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 9a4438e7303b..99a09cbc2bab 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -6,12 +6,6 @@ # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse FROM matrixdotorg/synapse-workers -# Download a caddy server to stand in front of nginx and terminate TLS using Complement's -# custom CA. -# We include this near the top of the file in order to cache the result. -RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \ - tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root - # Install postgresql RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13 @@ -31,16 +25,12 @@ COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml WORKDIR /data -# Copy the caddy config -COPY conf-workers/caddy.complement.json /root/caddy.json - COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf -COPY conf-workers/caddy.supervisord.conf /etc/supervisor/conf.d/caddy.conf # Copy the entrypoint COPY conf-workers/start-complement-synapse-workers.sh / -# Expose caddy's listener ports +# Expose nginx's listener ports EXPOSE 8008 8448 ENTRYPOINT ["/start-complement-synapse-workers.sh"] diff --git a/docker/complement/conf-workers/caddy.complement.json b/docker/complement/conf-workers/caddy.complement.json deleted file mode 100644 index 09e2136af2e2..000000000000 --- a/docker/complement/conf-workers/caddy.complement.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "apps": { - "http": { - "servers": { - "srv0": { - "listen": [ - ":8448" - ], - "routes": [ - { - "match": [ - { - "host": [ - "{{ server_name }}" - ] - } - ], - "handle": [ - { - "handler": "subroute", - "routes": [ - { - "handle": [ - { - "handler": "reverse_proxy", - "upstreams": [ - { - "dial": "localhost:8008" - } - ] - } - ] - } - ] - } - ], - "terminal": true - } - ] - } - } - }, - "tls": { - "automation": { - "policies": [ - { - "subjects": [ - "{{ server_name }}" - ], - "issuers": [ - { - "module": "internal" - } - ], - "on_demand": true - } - ] - } - }, - "pki": { - "certificate_authorities": { - "local": { - "name": "Complement CA", - "root": { - "certificate": "/complement/ca/ca.crt", - "private_key": "/complement/ca/ca.key" - } - } - } - } - } - } diff --git a/docker/complement/conf-workers/caddy.supervisord.conf b/docker/complement/conf-workers/caddy.supervisord.conf deleted file mode 100644 index d9ddb51dac46..000000000000 --- a/docker/complement/conf-workers/caddy.supervisord.conf +++ /dev/null @@ -1,7 +0,0 @@ -[program:caddy] -command=/usr/local/bin/prefix-log /root/caddy run --config /root/caddy.json -autorestart=unexpected -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh index b9a6b55bbe8e..a10b57a53f5e 100755 --- a/docker/complement/conf-workers/start-complement-synapse-workers.sh +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -9,9 +9,6 @@ function log { echo "$d $@" } -# Replace the server name in the caddy config -sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json - # Set the server name of the homeserver export SYNAPSE_SERVER_NAME=${SERVER_NAME} @@ -39,6 +36,21 @@ export SYNAPSE_WORKER_TYPES="\ appservice, \ pusher" + +# Generate a TLS key, then generate a certificate by having Complement's CA sign it +# Note that both the key and certificate are in PEM format (not DER). +openssl genrsa -out /conf/server.tls.key 2048 + +openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \ + -subj "/CN=${SERVER_NAME}" + +openssl x509 -req -in /conf/server.tls.csr \ + -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \ + -out /conf/server.tls.crt + +export SYNAPSE_TLS_CERT=/conf/server.tls.crt +export SYNAPSE_TLS_KEY=/conf/server.tls.key + # Run the script that writes the necessary config files and starts supervisord, which in turn # starts everything else exec /configure_workers_and_start.py diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2 index 1081979e06a0..967fc65e798c 100644 --- a/docker/conf-workers/nginx.conf.j2 +++ b/docker/conf-workers/nginx.conf.j2 @@ -9,6 +9,22 @@ server { listen 8008; listen [::]:8008; + {% if tls_cert_path is not none and tls_key_path is not none %} + listen 8448 ssl; + listen [::]:8448 ssl; + + ssl_certificate {{ tls_cert_path }}; + ssl_certificate_key {{ tls_key_path }}; + + # Some directives from cipherlist.eu (fka cipherli.st): + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; + ssl_ecdh_curve secp384r1; # Requires nginx >= 1.1.0 + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; # Requires nginx >= 1.5.9 + {% endif %} + server_name localhost; # Nginx by default only allows file uploads up to 1M in size diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index b2b7938ae801..f46b9b675e90 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -21,6 +21,9 @@ # * SYNAPSE_REPORT_STATS: Whether to report stats. # * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG # below. Leave empty for no workers, or set to '*' for all possible workers. +# * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format. +# * SYNAPSE_TLS_KEY: Path to a TLS key. If this and SYNAPSE_TLS_CERT are specified, +# Nginx will be configured to serve TLS on port 8448. # # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined # in the project's README), this script may be run multiple times, and functionality should @@ -501,6 +504,8 @@ def generate_worker_files( "/etc/nginx/conf.d/matrix-synapse.conf", worker_locations=nginx_location_config, upstream_directives=nginx_upstream_config, + tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"), + tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"), ) # Supervisord config From a6ab3f56196d0067a5be25917c24988a734f0d51 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 23 May 2022 11:28:14 +0100 Subject: [PATCH 245/263] Add a windows->unix file endings commit to git blame ignore file (#12824) --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 83ddd568c207..50d28c68eeb8 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -6,3 +6,6 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56 # Update black to 20.8b1 (#9381). 0a00b7ff14890987f09112a2ae696c61001e6cf1 + +# Convert tests/rest/admin/test_room.py to unix file endings (#7953). +c4268e3da64f1abb5b31deaeb5769adb6510c0a7 \ No newline at end of file From 438925c422fec9bffe6e90633abe8875c0c5fb5c Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 23 May 2022 12:20:30 +0100 Subject: [PATCH 246/263] Fix Complement `TestCanRegisterAdmin` with workers, by adding Complement's shared registration secret. (#12819) --- changelog.d/12819.misc | 1 + docker/complement/conf-workers/workers-shared.yaml | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 changelog.d/12819.misc diff --git a/changelog.d/12819.misc b/changelog.d/12819.misc new file mode 100644 index 000000000000..7a03102a632d --- /dev/null +++ b/changelog.d/12819.misc @@ -0,0 +1 @@ +Add Complement's shared registration secret to the Complement worker image. This fixes tests that depend on it. \ No newline at end of file diff --git a/docker/complement/conf-workers/workers-shared.yaml b/docker/complement/conf-workers/workers-shared.yaml index 86ee11ecd0e5..cd7b50c65cc3 100644 --- a/docker/complement/conf-workers/workers-shared.yaml +++ b/docker/complement/conf-workers/workers-shared.yaml @@ -5,6 +5,12 @@ enable_registration: true enable_registration_without_verification: true bcrypt_rounds: 4 +## Registration ## + +# Needed by Complement to register admin users +# DO NOT USE in a production configuration! This should be a random secret. +registration_shared_secret: complement + ## Federation ## # trust certs signed by Complement's CA From 444588c5fc5e4fd0f3796d389fe5f062acc55286 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 23 May 2022 13:23:26 +0200 Subject: [PATCH 247/263] Add some type hints to tests files (#12833) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/12833.misc | 1 + mypy.ini | 8 -------- tests/http/test_servlet.py | 14 ++++++++------ tests/http/test_site.py | 2 +- tests/scripts/test_new_matrix_user.py | 13 +++++++------ tests/storage/test_base.py | 2 +- tests/storage/test_roommember.py | 2 +- 7 files changed, 19 insertions(+), 23 deletions(-) create mode 100644 changelog.d/12833.misc diff --git a/changelog.d/12833.misc b/changelog.d/12833.misc new file mode 100644 index 000000000000..fad5df1afa34 --- /dev/null +++ b/changelog.d/12833.misc @@ -0,0 +1 @@ +Add some type hints to test files. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index df2622df983a..fe3e3f9b8efd 100644 --- a/mypy.ini +++ b/mypy.ini @@ -41,16 +41,11 @@ exclude = (?x) |tests/events/test_utils.py |tests/federation/test_federation_catch_up.py |tests/federation/test_federation_sender.py - |tests/federation/test_federation_server.py |tests/federation/transport/test_knocking.py - |tests/federation/transport/test_server.py |tests/handlers/test_typing.py |tests/http/federation/test_matrix_federation_agent.py |tests/http/federation/test_srv_resolver.py - |tests/http/test_fedclient.py |tests/http/test_proxyagent.py - |tests/http/test_servlet.py - |tests/http/test_site.py |tests/logging/__init__.py |tests/logging/test_terse_json.py |tests/module_api/test_api.py @@ -59,12 +54,9 @@ exclude = (?x) |tests/push/test_push_rule_evaluator.py |tests/rest/client/test_transactions.py |tests/rest/media/v1/test_media_storage.py - |tests/scripts/test_new_matrix_user.py |tests/server.py |tests/server_notices/test_resource_limits_server_notices.py |tests/state/test_v2.py - |tests/storage/test_base.py - |tests/storage/test_roommember.py |tests/test_metrics.py |tests/test_server.py |tests/test_state.py diff --git a/tests/http/test_servlet.py b/tests/http/test_servlet.py index ad521525cfaa..b3655d7b44c2 100644 --- a/tests/http/test_servlet.py +++ b/tests/http/test_servlet.py @@ -49,19 +49,21 @@ def test_parse_json_value(self): """Basic tests for parse_json_value_from_request.""" # Test round-tripping. obj = {"foo": 1} - result = parse_json_value_from_request(make_request(obj)) - self.assertEqual(result, obj) + result1 = parse_json_value_from_request(make_request(obj)) + self.assertEqual(result1, obj) # Results don't have to be objects. - result = parse_json_value_from_request(make_request(b'["foo"]')) - self.assertEqual(result, ["foo"]) + result2 = parse_json_value_from_request(make_request(b'["foo"]')) + self.assertEqual(result2, ["foo"]) # Test empty. with self.assertRaises(SynapseError): parse_json_value_from_request(make_request(b"")) - result = parse_json_value_from_request(make_request(b""), allow_empty_body=True) - self.assertIsNone(result) + result3 = parse_json_value_from_request( + make_request(b""), allow_empty_body=True + ) + self.assertIsNone(result3) # Invalid UTF-8. with self.assertRaises(SynapseError): diff --git a/tests/http/test_site.py b/tests/http/test_site.py index 8c13b4f6931e..b2dbf76d33b1 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -36,7 +36,7 @@ def test_large_request(self): # as a control case, first send a regular request. # complete the connection and wire it up to a fake transport - client_address = IPv6Address("TCP", "::1", "2345") + client_address = IPv6Address("TCP", "::1", 2345) protocol = factory.buildProtocol(client_address) transport = StringTransport() protocol.makeConnection(transport) diff --git a/tests/scripts/test_new_matrix_user.py b/tests/scripts/test_new_matrix_user.py index 19a145eeb65e..22f99c6ab1ce 100644 --- a/tests/scripts/test_new_matrix_user.py +++ b/tests/scripts/test_new_matrix_user.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List from unittest.mock import Mock, patch from synapse._scripts.register_new_matrix_user import request_registration @@ -49,8 +50,8 @@ def post(url, json=None, verify=None): requests.post = post # The fake stdout will be written here - out = [] - err_code = [] + out: List[str] = [] + err_code: List[int] = [] with patch("synapse._scripts.register_new_matrix_user.requests", requests): request_registration( @@ -85,8 +86,8 @@ def get(url, verify=None): requests.get = get # The fake stdout will be written here - out = [] - err_code = [] + out: List[str] = [] + err_code: List[int] = [] with patch("synapse._scripts.register_new_matrix_user.requests", requests): request_registration( @@ -137,8 +138,8 @@ def post(url, json=None, verify=None): requests.post = post # The fake stdout will be written here - out = [] - err_code = [] + out: List[str] = [] + err_code: List[int] = [] with patch("synapse._scripts.register_new_matrix_user.requests", requests): request_registration( diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index a8ffb52c0503..cce8e75c7475 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -60,7 +60,7 @@ def runWithConnection(func, *args, **kwargs): db = DatabasePool(Mock(), Mock(config=sqlite_config), fake_engine) db._db_pool = self.db_pool - self.datastore = SQLBaseStore(db, None, hs) + self.datastore = SQLBaseStore(db, None, hs) # type: ignore[arg-type] @defer.inlineCallbacks def test_insert_1col(self): diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index a2a9c05f24c8..1218786d79d8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -34,7 +34,7 @@ class RoomMemberStoreTestCase(unittest.HomeserverTestCase): room.register_servlets, ] - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: TestHomeServer) -> None: + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: TestHomeServer) -> None: # type: ignore[override] # We can't test the RoomMemberStore on its own without the other event # storage logic From 67aae05ece9b6e07fedc73f737c0d6db6351d6c7 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 23 May 2022 14:11:06 +0100 Subject: [PATCH 248/263] Support registering Application Services when running with workers under Complement. (#12826) Co-authored-by: Patrick Cloke --- changelog.d/12826.misc | 1 + .../start-complement-synapse-workers.sh | 5 +++++ docker/conf-workers/shared.yaml.j2 | 11 ++++++++++- docker/configure_workers_and_start.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12826.misc diff --git a/changelog.d/12826.misc b/changelog.d/12826.misc new file mode 100644 index 000000000000..f5e91f1ed592 --- /dev/null +++ b/changelog.d/12826.misc @@ -0,0 +1 @@ +Support registering Application Services when running with workers under Complement. \ No newline at end of file diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh index a10b57a53f5e..b7e24440006f 100755 --- a/docker/complement/conf-workers/start-complement-synapse-workers.sh +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -36,6 +36,11 @@ export SYNAPSE_WORKER_TYPES="\ appservice, \ pusher" +# Add Complement's appservice registration directory, if there is one +# (It can be absent when there are no application services in this test!) +if [ -d /complement/appservice ]; then + export SYNAPSE_AS_REGISTRATION_DIR=/complement/appservice +fi # Generate a TLS key, then generate a certificate by having Complement's CA sign it # Note that both the key and certificate are in PEM format (not DER). diff --git a/docker/conf-workers/shared.yaml.j2 b/docker/conf-workers/shared.yaml.j2 index f94b8c6aca0f..644ed788f3d5 100644 --- a/docker/conf-workers/shared.yaml.j2 +++ b/docker/conf-workers/shared.yaml.j2 @@ -6,4 +6,13 @@ redis: enabled: true -{{ shared_worker_config }} \ No newline at end of file +{% if appservice_registrations is not none %} +## Application Services ## +# A list of application service config files to use. +app_service_config_files: +{%- for path in appservice_registrations %} + - "{{ path }}" +{%- endfor %} +{%- endif %} + +{{ shared_worker_config }} diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index f46b9b675e90..b6ad14117325 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -21,6 +21,8 @@ # * SYNAPSE_REPORT_STATS: Whether to report stats. # * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG # below. Leave empty for no workers, or set to '*' for all possible workers. +# * SYNAPSE_AS_REGISTRATION_DIR: If specified, a directory in which .yaml and .yml files +# will be treated as Application Service registration files. # * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format. # * SYNAPSE_TLS_KEY: Path to a TLS key. If this and SYNAPSE_TLS_CERT are specified, # Nginx will be configured to serve TLS on port 8448. @@ -32,6 +34,7 @@ import os import subprocess import sys +from pathlib import Path from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set import jinja2 @@ -491,11 +494,23 @@ def generate_worker_files( master_log_config = generate_worker_log_config(environ, "master", data_dir) shared_config["log_config"] = master_log_config + # Find application service registrations + appservice_registrations = None + appservice_registration_dir = os.environ.get("SYNAPSE_AS_REGISTRATION_DIR") + if appservice_registration_dir: + # Scan for all YAML files that should be application service registrations. + appservice_registrations = [ + str(reg_path.resolve()) + for reg_path in Path(appservice_registration_dir).iterdir() + if reg_path.suffix.lower() in (".yaml", ".yml") + ] + # Shared homeserver config convert( "/conf/shared.yaml.j2", "/conf/workers/shared.yaml", shared_worker_config=yaml.dump(shared_config), + appservice_registrations=appservice_registrations, ) # Nginx config From 7a68203cde312c57137735a19c274a6d8470a2bf Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 23 May 2022 17:27:05 +0100 Subject: [PATCH 249/263] Disable 'faster room join' Complement tests when testing against Synapse with workers. (#12842) --- changelog.d/12842.misc | 1 + scripts-dev/complement.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12842.misc diff --git a/changelog.d/12842.misc b/changelog.d/12842.misc new file mode 100644 index 000000000000..cec3f97d86fd --- /dev/null +++ b/changelog.d/12842.misc @@ -0,0 +1 @@ +Disable 'faster room join' Complement tests when testing against Synapse with workers. \ No newline at end of file diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 190df6909a6a..ca476d9a5e61 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -45,6 +45,8 @@ docker build -t matrixdotorg/synapse -f "docker/Dockerfile" . extra_test_args=() +test_tags="synapse_blacklist,msc2716,msc3030" + # If we're using workers, modify the docker files slightly. if [[ -n "$WORKERS" ]]; then # Build the workers docker image (from the base Synapse image). @@ -65,6 +67,10 @@ if [[ -n "$WORKERS" ]]; then else export COMPLEMENT_BASE_IMAGE=complement-synapse COMPLEMENT_DOCKERFILE=Dockerfile + + # We only test faster room joins on monoliths, because they are purposefully + # being developed without worker support to start with. + test_tags="$test_tags,faster_joins" fi # Build the Complement image from the Synapse image we just built. @@ -73,4 +79,5 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF # Run the tests! echo "Images built; running complement" cd "$COMPLEMENT_DIR" -go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "${extra_test_args[@]}" "$@" ./tests/... + +go test -v -tags $test_tags -count=1 "${extra_test_args[@]}" "$@" ./tests/... From a608ac847b36dd72634f21502be42e785add8b65 Mon Sep 17 00:00:00 2001 From: Jess Porter Date: Mon, 23 May 2022 17:36:21 +0100 Subject: [PATCH 250/263] add SpamChecker callback for silently dropping inbound federated events (#12744) Signed-off-by: jesopo --- changelog.d/12744.feature | 1 + docs/modules/spam_checker_callbacks.md | 18 ++++++++++ synapse/events/spamcheck.py | 40 +++++++++++++++++++++ synapse/federation/federation_server.py | 48 ++++++++++++++++++++++--- synapse/module_api/__init__.py | 5 +++ 5 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 changelog.d/12744.feature diff --git a/changelog.d/12744.feature b/changelog.d/12744.feature new file mode 100644 index 000000000000..9836d94f8ca6 --- /dev/null +++ b/changelog.d/12744.feature @@ -0,0 +1 @@ +Add a `drop_federated_event` callback to `SpamChecker` to disregard inbound federated events before they take up much processing power, in an emergency. diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 472d95718087..27c5a0ed5cfe 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -249,6 +249,24 @@ callback returns `False`, Synapse falls through to the next one. The value of th callback that does not return `False` will be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. +### `should_drop_federated_event` + +_First introduced in Synapse v1.60.0_ + +```python +async def should_drop_federated_event(event: "synapse.events.EventBase") -> bool +``` + +Called when checking whether a remote server can federate an event with us. **Returning +`True` from this function will silently drop a federated event and split-brain our view +of a room's DAG, and thus you shouldn't use this callback unless you know what you are +doing.** + +If multiple modules implement this callback, they will be considered in order. If a +callback returns `False`, Synapse falls through to the next one. The value of the first +callback that does not return `False` will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback. + ## Example The example below is a module that implements the spam checker callback diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index f30207376ae2..61bcbe2abe60 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -44,6 +44,10 @@ ["synapse.events.EventBase"], Awaitable[Union[bool, str]], ] +SHOULD_DROP_FEDERATED_EVENT_CALLBACK = Callable[ + ["synapse.events.EventBase"], + Awaitable[Union[bool, str]], +] USER_MAY_JOIN_ROOM_CALLBACK = Callable[[str, str, bool], Awaitable[bool]] USER_MAY_INVITE_CALLBACK = Callable[[str, str, str], Awaitable[bool]] USER_MAY_SEND_3PID_INVITE_CALLBACK = Callable[[str, str, str, str], Awaitable[bool]] @@ -168,6 +172,9 @@ def __init__(self, hs: "synapse.server.HomeServer") -> None: self.clock = hs.get_clock() self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] + self._should_drop_federated_event_callbacks: List[ + SHOULD_DROP_FEDERATED_EVENT_CALLBACK + ] = [] self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = [] self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = [] self._user_may_send_3pid_invite_callbacks: List[ @@ -191,6 +198,9 @@ def __init__(self, hs: "synapse.server.HomeServer") -> None: def register_callbacks( self, check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None, + should_drop_federated_event: Optional[ + SHOULD_DROP_FEDERATED_EVENT_CALLBACK + ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, @@ -209,6 +219,11 @@ def register_callbacks( if check_event_for_spam is not None: self._check_event_for_spam_callbacks.append(check_event_for_spam) + if should_drop_federated_event is not None: + self._should_drop_federated_event_callbacks.append( + should_drop_federated_event + ) + if user_may_join_room is not None: self._user_may_join_room_callbacks.append(user_may_join_room) @@ -268,6 +283,31 @@ async def check_event_for_spam( return False + async def should_drop_federated_event( + self, event: "synapse.events.EventBase" + ) -> Union[bool, str]: + """Checks if a given federated event is considered "spammy" by this + server. + + If the server considers an event spammy, it will be silently dropped, + and in doing so will split-brain our view of the room's DAG. + + Args: + event: the event to be checked + + Returns: + True if the event should be silently dropped + """ + for callback in self._should_drop_federated_event_callbacks: + with Measure( + self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) + ): + res: Union[bool, str] = await delay_cancellation(callback(event)) + if res: + return res + + return False + async def user_may_join_room( self, user_id: str, room_id: str, is_invited: bool ) -> bool: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 884b5d60b4f9..b8232e5257d2 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -110,6 +110,7 @@ def __init__(self, hs: "HomeServer"): self.handler = hs.get_federation_handler() self.storage = hs.get_storage() + self._spam_checker = hs.get_spam_checker() self._federation_event_handler = hs.get_federation_event_handler() self.state = hs.get_state_handler() self._event_auth_handler = hs.get_event_auth_handler() @@ -1019,6 +1020,12 @@ async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None: except SynapseError as e: raise FederationError("ERROR", e.code, e.msg, affected=pdu.event_id) + if await self._spam_checker.should_drop_federated_event(pdu): + logger.warning( + "Unstaged federated event contains spam, dropping %s", pdu.event_id + ) + return + # Add the event to our staging area await self.store.insert_received_event_to_staging(origin, pdu) @@ -1032,6 +1039,41 @@ async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None: pdu.room_id, room_version, lock, origin, pdu ) + async def _get_next_nonspam_staged_event_for_room( + self, room_id: str, room_version: RoomVersion + ) -> Optional[Tuple[str, EventBase]]: + """Fetch the first non-spam event from staging queue. + + Args: + room_id: the room to fetch the first non-spam event in. + room_version: the version of the room. + + Returns: + The first non-spam event in that room. + """ + + while True: + # We need to do this check outside the lock to avoid a race between + # a new event being inserted by another instance and it attempting + # to acquire the lock. + next = await self.store.get_next_staged_event_for_room( + room_id, room_version + ) + + if next is None: + return None + + origin, event = next + + if await self._spam_checker.should_drop_federated_event(event): + logger.warning( + "Staged federated event contains spam, dropping %s", + event.event_id, + ) + continue + + return next + @wrap_as_background_process("_process_incoming_pdus_in_room_inner") async def _process_incoming_pdus_in_room_inner( self, @@ -1109,12 +1151,10 @@ async def _process_incoming_pdus_in_room_inner( (self._clock.time_msec() - received_ts) / 1000 ) - # We need to do this check outside the lock to avoid a race between - # a new event being inserted by another instance and it attempting - # to acquire the lock. - next = await self.store.get_next_staged_event_for_room( + next = await self._get_next_nonspam_staged_event_for_room( room_id, room_version ) + if not next: break diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 73f92d2df8d6..c4f661bb9382 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -47,6 +47,7 @@ CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK, CHECK_REGISTRATION_FOR_SPAM_CALLBACK, CHECK_USERNAME_FOR_SPAM_CALLBACK, + SHOULD_DROP_FEDERATED_EVENT_CALLBACK, USER_MAY_CREATE_ROOM_ALIAS_CALLBACK, USER_MAY_CREATE_ROOM_CALLBACK, USER_MAY_INVITE_CALLBACK, @@ -234,6 +235,9 @@ def register_spam_checker_callbacks( self, *, check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None, + should_drop_federated_event: Optional[ + SHOULD_DROP_FEDERATED_EVENT_CALLBACK + ] = None, user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None, user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None, user_may_send_3pid_invite: Optional[USER_MAY_SEND_3PID_INVITE_CALLBACK] = None, @@ -254,6 +258,7 @@ def register_spam_checker_callbacks( """ return self._spam_checker.register_callbacks( check_event_for_spam=check_event_for_spam, + should_drop_federated_event=should_drop_federated_event, user_may_join_room=user_may_join_room, user_may_invite=user_may_invite, user_may_send_3pid_invite=user_may_send_3pid_invite, From 4cc4229cd7a55d2556c798fecbb1c9660dc821c8 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Mon, 23 May 2022 19:18:23 +0200 Subject: [PATCH 251/263] Prevent expired events from being filtered out when retention is disabled (#12611) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Co-authored-by: Patrick Cloke --- changelog.d/12611.bugfix | 1 + synapse/handlers/pagination.py | 2 +- synapse/storage/databases/main/room.py | 45 ++++++++++++++------------ synapse/types.py | 6 ++++ synapse/visibility.py | 6 ++-- tests/rest/client/test_relations.py | 8 ++--- tests/rest/client/test_retention.py | 35 +++++++++++++++++--- 7 files changed, 71 insertions(+), 32 deletions(-) create mode 100644 changelog.d/12611.bugfix diff --git a/changelog.d/12611.bugfix b/changelog.d/12611.bugfix new file mode 100644 index 000000000000..093c45a20b7f --- /dev/null +++ b/changelog.d/12611.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.7.0 that would prevent events from being sent to clients if there's a retention policy in the room when the support for retention policies is disabled. diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 6ae88add9526..19a440705027 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -239,7 +239,7 @@ async def purge_history_for_rooms_in_range( # defined in the server's configuration, we can safely assume that's the # case and use it for this room. max_lifetime = ( - retention_policy["max_lifetime"] or self._retention_default_max_lifetime + retention_policy.max_lifetime or self._retention_default_max_lifetime ) # Cap the effective max_lifetime to be within the range allowed in the diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 87e9482c6054..ded15b92ef84 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -45,7 +45,7 @@ from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator -from synapse.types import JsonDict, ThirdPartyInstanceID +from synapse.types import JsonDict, RetentionPolicy, ThirdPartyInstanceID from synapse.util import json_encoder from synapse.util.caches.descriptors import cached from synapse.util.stringutils import MXC_REGEX @@ -699,7 +699,7 @@ def delete_ratelimit_txn(txn: LoggingTransaction) -> None: await self.db_pool.runInteraction("delete_ratelimit", delete_ratelimit_txn) @cached() - async def get_retention_policy_for_room(self, room_id: str) -> Dict[str, int]: + async def get_retention_policy_for_room(self, room_id: str) -> RetentionPolicy: """Get the retention policy for a given room. If no retention policy has been found for this room, returns a policy defined @@ -707,12 +707,20 @@ async def get_retention_policy_for_room(self, room_id: str) -> Dict[str, int]: the 'max_lifetime' if no default policy has been defined in the server's configuration). + If support for retention policies is disabled, a policy with a 'min_lifetime' and + 'max_lifetime' of None is returned. + Args: room_id: The ID of the room to get the retention policy of. Returns: A dict containing "min_lifetime" and "max_lifetime" for this room. """ + # If the room retention feature is disabled, return a policy with no minimum nor + # maximum. This prevents incorrectly filtering out events when sending to + # the client. + if not self.config.retention.retention_enabled: + return RetentionPolicy() def get_retention_policy_for_room_txn( txn: LoggingTransaction, @@ -736,10 +744,10 @@ def get_retention_policy_for_room_txn( # If we don't know this room ID, ret will be None, in this case return the default # policy. if not ret: - return { - "min_lifetime": self.config.retention.retention_default_min_lifetime, - "max_lifetime": self.config.retention.retention_default_max_lifetime, - } + return RetentionPolicy( + min_lifetime=self.config.retention.retention_default_min_lifetime, + max_lifetime=self.config.retention.retention_default_max_lifetime, + ) min_lifetime = ret[0]["min_lifetime"] max_lifetime = ret[0]["max_lifetime"] @@ -754,10 +762,10 @@ def get_retention_policy_for_room_txn( if max_lifetime is None: max_lifetime = self.config.retention.retention_default_max_lifetime - return { - "min_lifetime": min_lifetime, - "max_lifetime": max_lifetime, - } + return RetentionPolicy( + min_lifetime=min_lifetime, + max_lifetime=max_lifetime, + ) async def get_media_mxcs_in_room(self, room_id: str) -> Tuple[List[str], List[str]]: """Retrieves all the local and remote media MXC URIs in a given room @@ -994,7 +1002,7 @@ def _quarantine_media_txn( async def get_rooms_for_retention_period_in_range( self, min_ms: Optional[int], max_ms: Optional[int], include_null: bool = False - ) -> Dict[str, Dict[str, Optional[int]]]: + ) -> Dict[str, RetentionPolicy]: """Retrieves all of the rooms within the given retention range. Optionally includes the rooms which don't have a retention policy. @@ -1016,7 +1024,7 @@ async def get_rooms_for_retention_period_in_range( def get_rooms_for_retention_period_in_range_txn( txn: LoggingTransaction, - ) -> Dict[str, Dict[str, Optional[int]]]: + ) -> Dict[str, RetentionPolicy]: range_conditions = [] args = [] @@ -1047,10 +1055,10 @@ def get_rooms_for_retention_period_in_range_txn( rooms_dict = {} for row in rows: - rooms_dict[row["room_id"]] = { - "min_lifetime": row["min_lifetime"], - "max_lifetime": row["max_lifetime"], - } + rooms_dict[row["room_id"]] = RetentionPolicy( + min_lifetime=row["min_lifetime"], + max_lifetime=row["max_lifetime"], + ) if include_null: # If required, do a second query that retrieves all of the rooms we know @@ -1065,10 +1073,7 @@ def get_rooms_for_retention_period_in_range_txn( # policy in its state), add it with a null policy. for row in rows: if row["room_id"] not in rooms_dict: - rooms_dict[row["room_id"]] = { - "min_lifetime": None, - "max_lifetime": None, - } + rooms_dict[row["room_id"]] = RetentionPolicy() return rooms_dict diff --git a/synapse/types.py b/synapse/types.py index bd8071d51d78..6f7128ddd604 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -932,3 +932,9 @@ class UserProfile(TypedDict): user_id: str display_name: Optional[str] avatar_url: Optional[str] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class RetentionPolicy: + min_lifetime: Optional[int] = None + max_lifetime: Optional[int] = None diff --git a/synapse/visibility.py b/synapse/visibility.py index de6d2ffc526a..da4af02796c3 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -22,7 +22,7 @@ from synapse.events.utils import prune_event from synapse.storage import Storage from synapse.storage.state import StateFilter -from synapse.types import StateMap, get_domain_from_id +from synapse.types import RetentionPolicy, StateMap, get_domain_from_id logger = logging.getLogger(__name__) @@ -94,7 +94,7 @@ async def filter_events_for_client( if filter_send_to_client: room_ids = {e.room_id for e in events} - retention_policies = {} + retention_policies: Dict[str, RetentionPolicy] = {} for room_id in room_ids: retention_policies[ @@ -137,7 +137,7 @@ def allowed(event: EventBase) -> Optional[EventBase]: # events. if not event.is_state(): retention_policy = retention_policies[event.room_id] - max_lifetime = retention_policy.get("max_lifetime") + max_lifetime = retention_policy.max_lifetime if max_lifetime is not None: oldest_allowed_ts = storage.main.clock.time_msec() - max_lifetime diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 27dee8f6975d..bc9cc51b92d5 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -995,7 +995,7 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: bundled_aggregations, ) - self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 7) + self._test_bundled_aggregations(RelationTypes.ANNOTATION, assert_annotations, 6) def test_annotation_to_annotation(self) -> None: """Any relation to an annotation should be ignored.""" @@ -1031,7 +1031,7 @@ def assert_annotations(bundled_aggregations: JsonDict) -> None: bundled_aggregations, ) - self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 7) + self._test_bundled_aggregations(RelationTypes.REFERENCE, assert_annotations, 6) def test_thread(self) -> None: """ @@ -1060,7 +1060,7 @@ def assert_thread(bundled_aggregations: JsonDict) -> None: bundled_aggregations.get("latest_event"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 10) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) def test_thread_with_bundled_aggregations_for_latest(self) -> None: """ @@ -1106,7 +1106,7 @@ def assert_thread(bundled_aggregations: JsonDict) -> None: bundled_aggregations["latest_event"].get("unsigned"), ) - self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 10) + self._test_bundled_aggregations(RelationTypes.THREAD, assert_thread, 9) def test_nested_thread(self) -> None: """ diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 7b8fe6d02522..2cd7a9e6c5f8 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Dict from unittest.mock import Mock from twisted.test.proto_helpers import MemoryReactor @@ -252,16 +253,24 @@ class RetentionNoDefaultPolicyTestCase(unittest.HomeserverTestCase): room.register_servlets, ] - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - config = self.default_config() - config["retention"] = { + def default_config(self) -> Dict[str, Any]: + config = super().default_config() + + retention_config = { "enabled": True, } + # Update this config with what's in the default config so that + # override_config works as expected. + retention_config.update(config.get("retention", {})) + config["retention"] = retention_config + + return config + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: mock_federation_client = Mock(spec=["backfill"]) self.hs = self.setup_test_homeserver( - config=config, federation_client=mock_federation_client, ) return self.hs @@ -295,6 +304,24 @@ def test_state_policy(self) -> None: self._test_retention(room_id, expected_code_for_first_event=404) + @unittest.override_config({"retention": {"enabled": False}}) + def test_visibility_when_disabled(self) -> None: + """Retention policies should be ignored when the retention feature is disabled.""" + room_id = self.helper.create_room_as(self.user_id, tok=self.token) + + self.helper.send_state( + room_id=room_id, + event_type=EventTypes.Retention, + body={"max_lifetime": one_day_ms}, + tok=self.token, + ) + + resp = self.helper.send(room_id=room_id, body="test", tok=self.token) + + self.reactor.advance(one_day_ms * 2 / 1000) + + self.get_event(room_id, resp["event_id"]) + def _test_retention( self, room_id: str, expected_code_for_first_event: int = 200 ) -> None: From 28199e93579b5a73841a95ed4d355322227432b5 Mon Sep 17 00:00:00 2001 From: David Teller Date: Mon, 23 May 2022 19:27:39 +0200 Subject: [PATCH 252/263] Uniformize spam-checker API, part 2: check_event_for_spam (#12808) Signed-off-by: David Teller --- changelog.d/12808.feature | 1 + docs/modules/spam_checker_callbacks.md | 27 ++++++++------ docs/upgrade.md | 29 +++++++++++++++ synapse/api/errors.py | 4 +-- synapse/events/spamcheck.py | 49 ++++++++++++++++++++------ synapse/federation/federation_base.py | 5 +-- synapse/handlers/message.py | 11 +++--- synapse/module_api/__init__.py | 5 +++ synapse/module_api/errors.py | 2 ++ synapse/spam_checker_api/__init__.py | 27 +++++++++++++- 10 files changed, 129 insertions(+), 31 deletions(-) create mode 100644 changelog.d/12808.feature diff --git a/changelog.d/12808.feature b/changelog.d/12808.feature new file mode 100644 index 000000000000..561c8b9d34a4 --- /dev/null +++ b/changelog.d/12808.feature @@ -0,0 +1 @@ +Update to `check_event_for_spam`. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). \ No newline at end of file diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index 27c5a0ed5cfe..71f6f9f0ab45 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -11,22 +11,29 @@ The available spam checker callbacks are: ### `check_event_for_spam` _First introduced in Synapse v1.37.0_ +_Signature extended to support Allow and Code in Synapse v1.60.0_ +_Boolean and string return value types deprecated in Synapse v1.60.0_ ```python -async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str] +async def check_event_for_spam(event: "synapse.module_api.EventBase") -> Union["synapse.module_api.ALLOW", "synapse.module_api.error.Codes", str, bool] ``` -Called when receiving an event from a client or via federation. The callback must return -either: -- an error message string, to indicate the event must be rejected because of spam and - give a rejection reason to forward to clients; -- the boolean `True`, to indicate that the event is spammy, but not provide further details; or -- the booelan `False`, to indicate that the event is not considered spammy. +Called when receiving an event from a client or via federation. The callback must return either: + - `synapse.module_api.ALLOW`, to allow the operation. Other callbacks + may still decide to reject it. + - `synapse.api.Codes` to reject the operation with an error code. In case + of doubt, `synapse.api.error.Codes.FORBIDDEN` is a good error code. + - (deprecated) a `str` to reject the operation and specify an error message. Note that clients + typically will not localize the error message to the user's preferred locale. + - (deprecated) on `False`, behave as `ALLOW`. Deprecated as confusing, as some + callbacks in expect `True` to allow and others `True` to reject. + - (deprecated) on `True`, behave as `synapse.api.error.Codes.FORBIDDEN`. Deprecated as confusing, as + some callbacks in expect `True` to allow and others `True` to reject. If multiple modules implement this callback, they will be considered in order. If a -callback returns `False`, Synapse falls through to the next one. The value of the first -callback that does not return `False` will be used. If this happens, Synapse will not call -any of the subsequent implementations of this callback. +callback returns `synapse.module_api.ALLOW`, Synapse falls through to the next one. The value of the +first callback that does not return `synapse.module_api.ALLOW` will be used. If this happens, Synapse +will not call any of the subsequent implementations of this callback. ### `user_may_join_room` diff --git a/docs/upgrade.md b/docs/upgrade.md index 92ca31b2f8de..e7eadadb64bf 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -177,7 +177,36 @@ has queries that can be used to check a database for this problem in advance. +## SpamChecker API's `check_event_for_spam` has a new signature. +The previous signature has been deprecated. + +Whereas `check_event_for_spam` callbacks used to return `Union[str, bool]`, they should now return `Union["synapse.module_api.Allow", "synapse.module_api.errors.Codes"]`. + +This is part of an ongoing refactoring of the SpamChecker API to make it less ambiguous and more powerful. + +If your module implements `check_event_for_spam` as follows: + +```python +async def check_event_for_spam(event): + if ...: + # Event is spam + return True + # Event is not spam + return False +``` + +you should rewrite it as follows: + +```python +async def check_event_for_spam(event): + if ...: + # Event is spam, mark it as forbidden (you may use some more precise error + # code if it is useful). + return synapse.module_api.errors.Codes.FORBIDDEN + # Event is not spam, mark it as `ALLOW`. + return synapse.module_api.ALLOW +``` # Upgrading to v1.59.0 diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 9614be6b4e46..6650e826d5af 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -270,9 +270,7 @@ class UnrecognizedRequestError(SynapseError): """An error indicating we don't understand the request you're trying to make""" def __init__( - self, - msg: str = "Unrecognized request", - errcode: str = Codes.UNRECOGNIZED, + self, msg: str = "Unrecognized request", errcode: str = Codes.UNRECOGNIZED ): super().__init__(400, msg, errcode) diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index 61bcbe2abe60..7984874e21df 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -27,9 +27,10 @@ Union, ) +from synapse.api.errors import Codes from synapse.rest.media.v1._base import FileInfo from synapse.rest.media.v1.media_storage import ReadableFileWrapper -from synapse.spam_checker_api import RegistrationBehaviour +from synapse.spam_checker_api import Allow, Decision, RegistrationBehaviour from synapse.types import RoomAlias, UserProfile from synapse.util.async_helpers import delay_cancellation, maybe_awaitable from synapse.util.metrics import Measure @@ -40,9 +41,19 @@ logger = logging.getLogger(__name__) + CHECK_EVENT_FOR_SPAM_CALLBACK = Callable[ ["synapse.events.EventBase"], - Awaitable[Union[bool, str]], + Awaitable[ + Union[ + Allow, + Codes, + # Deprecated + bool, + # Deprecated + str, + ] + ], ] SHOULD_DROP_FEDERATED_EVENT_CALLBACK = Callable[ ["synapse.events.EventBase"], @@ -259,7 +270,7 @@ def register_callbacks( async def check_event_for_spam( self, event: "synapse.events.EventBase" - ) -> Union[bool, str]: + ) -> Union[Decision, str]: """Checks if a given event is considered "spammy" by this server. If the server considers an event spammy, then it will be rejected if @@ -270,18 +281,36 @@ async def check_event_for_spam( event: the event to be checked Returns: - True or a string if the event is spammy. If a string is returned it - will be used as the error message returned to the user. + - on `ALLOW`, the event is considered good (non-spammy) and should + be let through. Other spamcheck filters may still reject it. + - on `Code`, the event is considered spammy and is rejected with a specific + error message/code. + - on `str`, the event is considered spammy and the string is used as error + message. This usage is generally discouraged as it doesn't support + internationalization. """ for callback in self._check_event_for_spam_callbacks: with Measure( self.clock, "{}.{}".format(callback.__module__, callback.__qualname__) ): - res: Union[bool, str] = await delay_cancellation(callback(event)) - if res: - return res - - return False + res: Union[Decision, str, bool] = await delay_cancellation( + callback(event) + ) + if res is False or res is Allow.ALLOW: + # This spam-checker accepts the event. + # Other spam-checkers may reject it, though. + continue + elif res is True: + # This spam-checker rejects the event with deprecated + # return value `True` + return Codes.FORBIDDEN + else: + # This spam-checker rejects the event either with a `str` + # or with a `Codes`. In either case, we stop here. + return res + + # No spam-checker has rejected the event, let it pass. + return Allow.ALLOW async def should_drop_federated_event( self, event: "synapse.events.EventBase" diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 41ac49fdc8bf..1e866b19d87b 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -15,6 +15,7 @@ import logging from typing import TYPE_CHECKING +import synapse from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import EventFormatVersions, RoomVersion @@ -98,9 +99,9 @@ async def _check_sigs_and_hash( ) return redacted_event - result = await self.spam_checker.check_event_for_spam(pdu) + spam_check = await self.spam_checker.check_event_for_spam(pdu) - if result: + if spam_check is not synapse.spam_checker_api.Allow.ALLOW: logger.warning("Event contains spam, soft-failing %s", pdu.event_id) # we redact (to save disk space) as well as soft-failing (to stop # using the event in prev_events). diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e566ff1f8ed8..cb1bc4c06f1c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -23,6 +23,7 @@ from twisted.internet.interfaces import IDelayedCall +import synapse from synapse import event_auth from synapse.api.constants import ( EventContentFields, @@ -885,11 +886,11 @@ async def create_and_send_nonmember_event( event.sender, ) - spam_error = await self.spam_checker.check_event_for_spam(event) - if spam_error: - if not isinstance(spam_error, str): - spam_error = "Spam is not permitted here" - raise SynapseError(403, spam_error, Codes.FORBIDDEN) + spam_check = await self.spam_checker.check_event_for_spam(event) + if spam_check is not synapse.spam_checker_api.Allow.ALLOW: + raise SynapseError( + 403, "This message had been rejected as probable spam", spam_check + ) ev = await self.handle_new_client_event( requester=requester, diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index c4f661bb9382..95f3b2792793 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -35,6 +35,7 @@ from twisted.internet import defer from twisted.web.resource import Resource +from synapse import spam_checker_api from synapse.api.errors import SynapseError from synapse.events import EventBase from synapse.events.presence_router import ( @@ -140,6 +141,9 @@ PRESENCE_ALL_USERS = PresenceRouter.ALL_USERS +ALLOW = spam_checker_api.Allow.ALLOW +# Singleton value used to mark a message as permitted. + __all__ = [ "errors", "make_deferred_yieldable", @@ -147,6 +151,7 @@ "respond_with_html", "run_in_background", "cached", + "Allow", "UserID", "DatabasePool", "LoggingTransaction", diff --git a/synapse/module_api/errors.py b/synapse/module_api/errors.py index e58e0e60feab..bedd045d6fe1 100644 --- a/synapse/module_api/errors.py +++ b/synapse/module_api/errors.py @@ -15,6 +15,7 @@ """Exception types which are exposed as part of the stable module API""" from synapse.api.errors import ( + Codes, InvalidClientCredentialsError, RedirectException, SynapseError, @@ -24,6 +25,7 @@ from synapse.storage.push_rule import RuleNotFoundException __all__ = [ + "Codes", "InvalidClientCredentialsError", "RedirectException", "SynapseError", diff --git a/synapse/spam_checker_api/__init__.py b/synapse/spam_checker_api/__init__.py index 73018f2d002e..95132c80b70e 100644 --- a/synapse/spam_checker_api/__init__.py +++ b/synapse/spam_checker_api/__init__.py @@ -12,13 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. from enum import Enum +from typing import Union + +from synapse.api.errors import Codes class RegistrationBehaviour(Enum): """ - Enum to define whether a registration request should allowed, denied, or shadow-banned. + Enum to define whether a registration request should be allowed, denied, or shadow-banned. """ ALLOW = "allow" SHADOW_BAN = "shadow_ban" DENY = "deny" + + +# We define the following singleton enum rather than a string to be able to +# write `Union[Allow, ..., str]` in some of the callbacks for the spam-checker +# API, where the `str` is required to maintain backwards compatibility with +# previous versions of the API. +class Allow(Enum): + """ + Singleton to allow events to pass through in SpamChecker APIs. + """ + + ALLOW = "allow" + + +Decision = Union[Allow, Codes] +""" +Union to define whether a request should be allowed or rejected. + +To accept a request, return `ALLOW`. + +To reject a request without any specific information, use `Codes.FORBIDDEN`. +""" From 7c2a78bb3bd2091439722e9f1e4601837bcb8fc4 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 23 May 2022 20:43:37 -0500 Subject: [PATCH 253/263] Marker events as state - MSC2716 (#12718) Sending marker events as state now so they are always able to be seen by homeservers (not lost in some timeline gap). Part of [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) Complement tests: https://github.com/matrix-org/complement/pull/371 As initially discussed at https://github.com/matrix-org/matrix-spec-proposals/pull/2716#discussion_r782629097 and https://github.com/matrix-org/matrix-spec-proposals/pull/2716#discussion_r876684431 When someone joins a room, process all of the marker events we see in the current state. Marker events should be sent with a unique `state_key` so that they can all resolve in the current state to easily be discovered. Marker events as state - If we re-use the same `state_key` (like `""`), then we would have to fetch previous snapshots of state up through time to find all of the marker events. This way we can avoid all of that. This PR was originally doing this but then thought of the smarter way to tackle in an [out of band discussion with @erikjohnston](https://docs.google.com/document/d/1JJDuPfcPNX75fprdTWlxlaKjWOdbdJylbpZ03hzo638/edit#bookmark=id.sm92fqyq7vpp). - Also avoids state resolution conflicts where only one of the marker events win As a homeserver, when we see new marker state, we know there is new history imported somewhere back in time and should process it to fetch the insertion event where the historical messages are and set it as an insertion extremity. This way we know where to backfill more messages when someone asks for scrollback. --- changelog.d/12718.feature | 1 + synapse/handlers/federation_event.py | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12718.feature diff --git a/changelog.d/12718.feature b/changelog.d/12718.feature new file mode 100644 index 000000000000..1056f519a4c1 --- /dev/null +++ b/changelog.d/12718.feature @@ -0,0 +1 @@ +Update [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to process marker events from the current state to avoid markers being lost in timeline gaps for federated servers which would cause the imported history to be undiscovered. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 05c122f22491..ca82df8a6d9e 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -477,7 +477,23 @@ async def process_remote_join( # and discover that we do not have it. event.internal_metadata.proactively_send = False - return await self.persist_events_and_notify(room_id, [(event, context)]) + stream_id_after_persist = await self.persist_events_and_notify( + room_id, [(event, context)] + ) + + # If we're joining the room again, check if there is new marker + # state indicating that there is new history imported somewhere in + # the DAG. Multiple markers can exist in the current state with + # unique state_keys. + # + # Do this after the state from the remote join was persisted (via + # `persist_events_and_notify`). Otherwise we can run into a + # situation where the create event doesn't exist yet in the + # `current_state_events` + for e in state: + await self._handle_marker_event(origin, e) + + return stream_id_after_persist async def update_state_for_partial_state_event( self, destination: str, event: EventBase @@ -1230,6 +1246,14 @@ async def _handle_marker_event(self, origin: str, marker_event: EventBase) -> No # Nothing to retrieve then (invalid marker) return + already_seen_insertion_event = await self._store.have_seen_event( + marker_event.room_id, insertion_event_id + ) + if already_seen_insertion_event: + # No need to process a marker again if we have already seen the + # insertion event that it was pointing to + return + logger.debug( "_handle_marker_event: backfilling insertion event %s", insertion_event_id ) From f5b1c09909e81182bacc167e70188a7c43aad813 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 May 2022 11:35:08 +0100 Subject: [PATCH 254/263] Pin poetry.core in Docker images (#12853) --- changelog.d/12853.docker | 1 + docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12853.docker diff --git a/changelog.d/12853.docker b/changelog.d/12853.docker new file mode 100644 index 000000000000..cad10a79cc82 --- /dev/null +++ b/changelog.d/12853.docker @@ -0,0 +1 @@ +Fix the docker file after a dependency update. diff --git a/docker/Dockerfile b/docker/Dockerfile index ccc6a9f77849..7af0e51f97d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -55,7 +55,7 @@ RUN \ # NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also # pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export). RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5 + pip install --user "poetry-core==1.1.0a7" "git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5" WORKDIR /synapse From 0b3423fd51608a8ff9d61d61f4975a1cd877d679 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 24 May 2022 11:48:11 +0100 Subject: [PATCH 255/263] contributing_guide.md: fix link to DCO --- docs/development/contributing_guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index f55a1fbb9002..2b3714df66f9 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -422,8 +422,8 @@ same lightweight approach that the Linux Kernel [submitting patches process]( https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other -projects use: the DCO (Developer Certificate of Origin: -http://developercertificate.org/). This is a simple declaration that you wrote +projects use: the DCO ([Developer Certificate of Origin](http://developercertificate.org/)). +This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix: ``` From a670b5cda24cdabfd4cb5732bda9ed1ccd86a1d1 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 24 May 2022 12:05:33 +0100 Subject: [PATCH 256/263] 1.60.0rc1 --- CHANGES.md | 112 ++++++++++++++++++++++++++++++++++++++ changelog.d/10533.misc | 1 - changelog.d/12477.misc | 1 - changelog.d/12498.misc | 1 - changelog.d/12513.feature | 1 - changelog.d/12567.misc | 1 - changelog.d/12586.misc | 1 - changelog.d/12588.misc | 1 - changelog.d/12611.bugfix | 1 - changelog.d/12618.feature | 1 - changelog.d/12623.feature | 1 - changelog.d/12630.misc | 1 - changelog.d/12672.feature | 1 - changelog.d/12673.feature | 1 - changelog.d/12676.misc | 1 - changelog.d/12677.misc | 1 - changelog.d/12679.misc | 1 - changelog.d/12680.misc | 1 - changelog.d/12683.bugfix | 1 - changelog.d/12687.bugfix | 1 - changelog.d/12689.misc | 1 - changelog.d/12691.misc | 1 - changelog.d/12693.misc | 1 - changelog.d/12694.misc | 1 - changelog.d/12695.misc | 1 - changelog.d/12696.bugfix | 1 - changelog.d/12698.misc | 1 - changelog.d/12699.misc | 1 - changelog.d/12700.misc | 1 - changelog.d/12701.feature | 1 - changelog.d/12703.misc | 1 - changelog.d/12705.misc | 1 - changelog.d/12708.misc | 1 - changelog.d/12709.removal | 1 - changelog.d/12711.misc | 1 - changelog.d/12713.bugfix | 1 - changelog.d/12715.doc | 1 - changelog.d/12716.misc | 1 - changelog.d/12717.misc | 1 - changelog.d/12718.feature | 1 - changelog.d/12720.misc | 1 - changelog.d/12721.bugfix | 1 - changelog.d/12723.misc | 1 - changelog.d/12726.misc | 1 - changelog.d/12727.doc | 1 - changelog.d/12731.misc | 1 - changelog.d/12734.misc | 1 - changelog.d/12742.doc | 1 - changelog.d/12744.feature | 1 - changelog.d/12747.bugfix | 1 - changelog.d/12748.doc | 1 - changelog.d/12749.doc | 1 - changelog.d/12753.misc | 1 - changelog.d/12759.doc | 1 - changelog.d/12761.doc | 1 - changelog.d/12762.misc | 1 - changelog.d/12765.doc | 1 - changelog.d/12769.misc | 1 - changelog.d/12770.bugfix | 1 - changelog.d/12772.misc | 1 - changelog.d/12773.doc | 1 - changelog.d/12774.misc | 1 - changelog.d/12775.misc | 1 - changelog.d/12776.doc | 2 - changelog.d/12777.doc | 2 - changelog.d/12779.bugfix | 1 - changelog.d/12781.misc | 1 - changelog.d/12783.misc | 1 - changelog.d/12785.doc | 1 - changelog.d/12786.feature | 1 - changelog.d/12789.misc | 1 - changelog.d/12790.misc | 1 - changelog.d/12791.misc | 1 - changelog.d/12792.feature | 1 - changelog.d/12794.bugfix | 1 - changelog.d/12803.bugfix | 1 - changelog.d/12808.feature | 1 - changelog.d/12809.feature | 1 - changelog.d/12818.misc | 1 - changelog.d/12819.misc | 1 - changelog.d/12823.bugfix | 1 - changelog.d/12826.misc | 1 - changelog.d/12833.misc | 1 - changelog.d/12842.misc | 1 - changelog.d/12853.docker | 1 - debian/changelog | 6 ++ pyproject.toml | 2 +- 87 files changed, 119 insertions(+), 87 deletions(-) delete mode 100644 changelog.d/10533.misc delete mode 100644 changelog.d/12477.misc delete mode 100644 changelog.d/12498.misc delete mode 100644 changelog.d/12513.feature delete mode 100644 changelog.d/12567.misc delete mode 100644 changelog.d/12586.misc delete mode 100644 changelog.d/12588.misc delete mode 100644 changelog.d/12611.bugfix delete mode 100644 changelog.d/12618.feature delete mode 100644 changelog.d/12623.feature delete mode 100644 changelog.d/12630.misc delete mode 100644 changelog.d/12672.feature delete mode 100644 changelog.d/12673.feature delete mode 100644 changelog.d/12676.misc delete mode 100644 changelog.d/12677.misc delete mode 100644 changelog.d/12679.misc delete mode 100644 changelog.d/12680.misc delete mode 100644 changelog.d/12683.bugfix delete mode 100644 changelog.d/12687.bugfix delete mode 100644 changelog.d/12689.misc delete mode 100644 changelog.d/12691.misc delete mode 100644 changelog.d/12693.misc delete mode 100644 changelog.d/12694.misc delete mode 100644 changelog.d/12695.misc delete mode 100644 changelog.d/12696.bugfix delete mode 100644 changelog.d/12698.misc delete mode 100644 changelog.d/12699.misc delete mode 100644 changelog.d/12700.misc delete mode 100644 changelog.d/12701.feature delete mode 100644 changelog.d/12703.misc delete mode 100644 changelog.d/12705.misc delete mode 100644 changelog.d/12708.misc delete mode 100644 changelog.d/12709.removal delete mode 100644 changelog.d/12711.misc delete mode 100644 changelog.d/12713.bugfix delete mode 100644 changelog.d/12715.doc delete mode 100644 changelog.d/12716.misc delete mode 100644 changelog.d/12717.misc delete mode 100644 changelog.d/12718.feature delete mode 100644 changelog.d/12720.misc delete mode 100644 changelog.d/12721.bugfix delete mode 100644 changelog.d/12723.misc delete mode 100644 changelog.d/12726.misc delete mode 100644 changelog.d/12727.doc delete mode 100644 changelog.d/12731.misc delete mode 100644 changelog.d/12734.misc delete mode 100644 changelog.d/12742.doc delete mode 100644 changelog.d/12744.feature delete mode 100644 changelog.d/12747.bugfix delete mode 100644 changelog.d/12748.doc delete mode 100644 changelog.d/12749.doc delete mode 100644 changelog.d/12753.misc delete mode 100644 changelog.d/12759.doc delete mode 100644 changelog.d/12761.doc delete mode 100644 changelog.d/12762.misc delete mode 100644 changelog.d/12765.doc delete mode 100644 changelog.d/12769.misc delete mode 100644 changelog.d/12770.bugfix delete mode 100644 changelog.d/12772.misc delete mode 100644 changelog.d/12773.doc delete mode 100644 changelog.d/12774.misc delete mode 100644 changelog.d/12775.misc delete mode 100644 changelog.d/12776.doc delete mode 100644 changelog.d/12777.doc delete mode 100644 changelog.d/12779.bugfix delete mode 100644 changelog.d/12781.misc delete mode 100644 changelog.d/12783.misc delete mode 100644 changelog.d/12785.doc delete mode 100644 changelog.d/12786.feature delete mode 100644 changelog.d/12789.misc delete mode 100644 changelog.d/12790.misc delete mode 100644 changelog.d/12791.misc delete mode 100644 changelog.d/12792.feature delete mode 100644 changelog.d/12794.bugfix delete mode 100644 changelog.d/12803.bugfix delete mode 100644 changelog.d/12808.feature delete mode 100644 changelog.d/12809.feature delete mode 100644 changelog.d/12818.misc delete mode 100644 changelog.d/12819.misc delete mode 100644 changelog.d/12823.bugfix delete mode 100644 changelog.d/12826.misc delete mode 100644 changelog.d/12833.misc delete mode 100644 changelog.d/12842.misc delete mode 100644 changelog.d/12853.docker diff --git a/CHANGES.md b/CHANGES.md index e10ac0314abf..f6ca5c472144 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,115 @@ +Synapse 1.60.0rc1 (2022-05-24) +============================== + +Features +-------- + +- Measure the time taken in spam-checking callbacks and expose those measurements as metrics. ([\#12513](https://github.com/matrix-org/synapse/issues/12513)) +- Add a `default_power_level_content_override` config option to set default room power levels per room preset. ([\#12618](https://github.com/matrix-org/synapse/issues/12618)) +- Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787). ([\#12623](https://github.com/matrix-org/synapse/issues/12623)) +- Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. ([\#12672](https://github.com/matrix-org/synapse/issues/12672), [\#12809](https://github.com/matrix-org/synapse/issues/12809)) +- Synapse will now reload [cache config](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#caching) when it receives a [SIGHUP](https://en.wikipedia.org/wiki/SIGHUP) signal. ([\#12673](https://github.com/matrix-org/synapse/issues/12673)) +- Add a config options to allow for auto-tuning of caches. ([\#12701](https://github.com/matrix-org/synapse/issues/12701)) +- Update [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to process marker events from the current state to avoid markers being lost in timeline gaps for federated servers which would cause the imported history to be undiscovered. ([\#12718](https://github.com/matrix-org/synapse/issues/12718)) +- Add a `drop_federated_event` callback to `SpamChecker` to disregard inbound federated events before they take up much processing power, in an emergency. ([\#12744](https://github.com/matrix-org/synapse/issues/12744)) +- Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). ([\#12786](https://github.com/matrix-org/synapse/issues/12786), [\#12792](https://github.com/matrix-org/synapse/issues/12792)) +- Update to `check_event_for_spam`. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). ([\#12808](https://github.com/matrix-org/synapse/issues/12808)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.7.0 that would prevent events from being sent to clients if there's a retention policy in the room when the support for retention policies is disabled. ([\#12611](https://github.com/matrix-org/synapse/issues/12611)) +- Fix a bug introduced in Synapse 1.57.0 where `/messages` would throw a 500 error when querying for a non-existent room. ([\#12683](https://github.com/matrix-org/synapse/issues/12683)) +- Add a unique index to `state_group_edges` to prevent duplicates being accidentally introduced and the consequential impact to performance. ([\#12687](https://github.com/matrix-org/synapse/issues/12687)) +- Fix a long-standing bug where an empty room would be created when a user with an insufficient power level tried to upgrade a room. ([\#12696](https://github.com/matrix-org/synapse/issues/12696)) +- Fix a bug introduced in Synapse 1.30.0 where empty rooms could be automatically created if a monthly active users limit is set. ([\#12713](https://github.com/matrix-org/synapse/issues/12713)) +- Fix push to dismiss notifications when read on another client. Contributed by @SpiritCroc @ Beeper. ([\#12721](https://github.com/matrix-org/synapse/issues/12721)) +- Fix poor database performance when reading the cache invalidation stream for large servers with lots of workers. ([\#12747](https://github.com/matrix-org/synapse/issues/12747)) +- Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API. ([\#12770](https://github.com/matrix-org/synapse/issues/12770)) +- Give a meaningful error message when a client tries to create a room with an invalid alias localpart. ([\#12779](https://github.com/matrix-org/synapse/issues/12779)) +- Fix a bug introduced in 1.43.0 where a file (`providers.json`) was never closed. Contributed by @arkamar. ([\#12794](https://github.com/matrix-org/synapse/issues/12794)) +- Fix a long-standing bug where finished log contexts would be re-started when failing to contact remote homeservers. ([\#12803](https://github.com/matrix-org/synapse/issues/12803)) +- Fix a bug, introduced in Synapse 1.21.0, that led to media thumbnails being unusable before the index has been added in the background. ([\#12823](https://github.com/matrix-org/synapse/issues/12823)) + + +Updates to the Docker image +--------------------------- + +- Fix the docker file after a dependency update. ([\#12853](https://github.com/matrix-org/synapse/issues/12853)) + + +Improved Documentation +---------------------- + +- Fix a typo in the Media Admin API documentation. ([\#12715](https://github.com/matrix-org/synapse/issues/12715)) +- Update the OpenID Connect example for Keycloak to be compatible with newer versions of Keycloak. Contributed by @nhh. ([\#12727](https://github.com/matrix-org/synapse/issues/12727)) +- Fix typo in server listener documentation. ([\#12742](https://github.com/matrix-org/synapse/issues/12742)) +- Link to the configuration manual from the welcome page of the documentation. ([\#12748](https://github.com/matrix-org/synapse/issues/12748)) +- Fix typo in 'run_background_tasks_on' option name in configuration manual documentation. ([\#12749](https://github.com/matrix-org/synapse/issues/12749)) +- Add information regarding the `rc_invites` ratelimiting option to the configuration docs. ([\#12759](https://github.com/matrix-org/synapse/issues/12759)) +- Add documentation for cancellation of request processing. ([\#12761](https://github.com/matrix-org/synapse/issues/12761)) +- Recommend using docker to run tests against postgres. ([\#12765](https://github.com/matrix-org/synapse/issues/12765)) +- Add missing user directory endpoint from the generic worker documentation. Contributed by @olmari. ([\#12773](https://github.com/matrix-org/synapse/issues/12773)) +- Add additional info to documentation of config option `cache_autotuning`. ([\#12776](https://github.com/matrix-org/synapse/issues/12776)) +- Update configuration manual documentation to document size-related suffixes. ([\#12777](https://github.com/matrix-org/synapse/issues/12777)) +- Fix invalid YAML syntax in the example documentation for the `url_preview_accept_language` config option. ([\#12785](https://github.com/matrix-org/synapse/issues/12785)) + + +Deprecations and Removals +------------------------- + +- Require a body in POST requests to `/rooms/{roomId}/receipt/{receiptType}/{eventId}`, as required by the [Matrix specification](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidreceiptreceipttypeeventid). This breaks compatibility with Element Android 1.2.0 and earlier: users of those clients will be unable to send read receipts. ([\#12709](https://github.com/matrix-org/synapse/issues/12709)) + + +Internal Changes +---------------- + +- Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10533](https://github.com/matrix-org/synapse/issues/10533)) +- Add some type hints to datastore. ([\#12477](https://github.com/matrix-org/synapse/issues/12477), [\#12717](https://github.com/matrix-org/synapse/issues/12717), [\#12753](https://github.com/matrix-org/synapse/issues/12753)) +- Preparation for faster-room-join work: return subsets of room state which we already have, immediately. ([\#12498](https://github.com/matrix-org/synapse/issues/12498)) +- Replace string literal instances of stream key types with typed constants. ([\#12567](https://github.com/matrix-org/synapse/issues/12567)) +- Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. ([\#12586](https://github.com/matrix-org/synapse/issues/12586)) +- Add ability to cancel disconnected requests to `SynapseRequest`. ([\#12588](https://github.com/matrix-org/synapse/issues/12588)) +- Add a helper class for testing request cancellation. ([\#12630](https://github.com/matrix-org/synapse/issues/12630)) +- Improve documentation of the `synapse.push` module. ([\#12676](https://github.com/matrix-org/synapse/issues/12676)) +- Refactor functions to on `PushRuleEvaluatorForEvent`. ([\#12677](https://github.com/matrix-org/synapse/issues/12677)) +- Preparation for database schema simplifications: stop writing to `event_reference_hashes`. ([\#12679](https://github.com/matrix-org/synapse/issues/12679)) +- Remove code which updates unused database column `application_services_state.last_txn`. ([\#12680](https://github.com/matrix-org/synapse/issues/12680)) +- Refactor `EventContext` class. ([\#12689](https://github.com/matrix-org/synapse/issues/12689)) +- Remove an unneeded class in the push code. ([\#12691](https://github.com/matrix-org/synapse/issues/12691)) +- Consolidate parsing of relation information from events. ([\#12693](https://github.com/matrix-org/synapse/issues/12693)) +- Capture the `Deferred` for request cancellation in `_AsyncResource`. ([\#12694](https://github.com/matrix-org/synapse/issues/12694)) +- Fixes an incorrect type hint for `Filter._check_event_relations`. ([\#12695](https://github.com/matrix-org/synapse/issues/12695)) +- Respect the `@cancellable` flag for `DirectServe{Html,Json}Resource`s. ([\#12698](https://github.com/matrix-org/synapse/issues/12698)) +- Respect the `@cancellable` flag for `RestServlet`s and `BaseFederationServlet`s. ([\#12699](https://github.com/matrix-org/synapse/issues/12699)) +- Respect the `@cancellable` flag for `ReplicationEndpoint`s. ([\#12700](https://github.com/matrix-org/synapse/issues/12700)) +- Convert namespace class `Codes` into a string enum. ([\#12703](https://github.com/matrix-org/synapse/issues/12703)) +- Complain if a federation endpoint has the `@cancellable` flag, since some of the wrapper code may not handle cancellation correctly yet. ([\#12705](https://github.com/matrix-org/synapse/issues/12705)) +- Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. ([\#12708](https://github.com/matrix-org/synapse/issues/12708)) +- Optimize private read receipt filtering. ([\#12711](https://github.com/matrix-org/synapse/issues/12711)) +- Add type annotations to increase the number of modules passing `disallow-untyped-defs`. ([\#12716](https://github.com/matrix-org/synapse/issues/12716), [\#12726](https://github.com/matrix-org/synapse/issues/12726)) +- Drop the logging level of status messages for the URL preview cache expiry job from INFO to DEBUG. ([\#12720](https://github.com/matrix-org/synapse/issues/12720)) +- Downgrade some OIDC errors to warnings in the logs, to reduce the noise of Sentry reports. ([\#12723](https://github.com/matrix-org/synapse/issues/12723)) +- Update configs used by Complement to allow more invites/3PID validations during tests. ([\#12731](https://github.com/matrix-org/synapse/issues/12731)) +- Tidy up and type-hint the database engine modules. ([\#12734](https://github.com/matrix-org/synapse/issues/12734)) +- Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) +- Tweak the mypy plugin so that `@cached` can accept `on_invalidate=None`. ([\#12769](https://github.com/matrix-org/synapse/issues/12769)) +- Move methods that call `add_push_rule` to the `PushRuleStore` class. ([\#12772](https://github.com/matrix-org/synapse/issues/12772)) +- Make handling of federation Authorization header (more) compliant with RFC7230. ([\#12774](https://github.com/matrix-org/synapse/issues/12774)) +- Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. ([\#12775](https://github.com/matrix-org/synapse/issues/12775)) +- Do not keep going if there are 5 back-to-back background update failures. ([\#12781](https://github.com/matrix-org/synapse/issues/12781)) +- Fix federation when using the demo scripts. ([\#12783](https://github.com/matrix-org/synapse/issues/12783)) +- The `hash_password` script now fails when it is called without specifying a config file. ([\#12789](https://github.com/matrix-org/synapse/issues/12789)) +- Simplify `disallow_untyped_defs` config in `mypy.ini`. ([\#12790](https://github.com/matrix-org/synapse/issues/12790)) +- Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible. ([\#12791](https://github.com/matrix-org/synapse/issues/12791)) +- Remove Caddy from the Synapse workers image used in Complement. ([\#12818](https://github.com/matrix-org/synapse/issues/12818)) +- Add Complement's shared registration secret to the Complement worker image. This fixes tests that depend on it. ([\#12819](https://github.com/matrix-org/synapse/issues/12819)) +- Support registering Application Services when running with workers under Complement. ([\#12826](https://github.com/matrix-org/synapse/issues/12826)) +- Add some type hints to test files. ([\#12833](https://github.com/matrix-org/synapse/issues/12833)) +- Disable 'faster room join' Complement tests when testing against Synapse with workers. ([\#12842](https://github.com/matrix-org/synapse/issues/12842)) + + Synapse 1.59.1 (2022-05-18) =========================== diff --git a/changelog.d/10533.misc b/changelog.d/10533.misc deleted file mode 100644 index f70dc6496fcf..000000000000 --- a/changelog.d/10533.misc +++ /dev/null @@ -1 +0,0 @@ -Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. diff --git a/changelog.d/12477.misc b/changelog.d/12477.misc deleted file mode 100644 index e793d08e5e3f..000000000000 --- a/changelog.d/12477.misc +++ /dev/null @@ -1 +0,0 @@ -Add some type hints to datastore. \ No newline at end of file diff --git a/changelog.d/12498.misc b/changelog.d/12498.misc deleted file mode 100644 index 8a00b94fbeef..000000000000 --- a/changelog.d/12498.misc +++ /dev/null @@ -1 +0,0 @@ -Preparation for faster-room-join work: return subsets of room state which we already have, immediately. diff --git a/changelog.d/12513.feature b/changelog.d/12513.feature deleted file mode 100644 index 01bf1d9d2cf6..000000000000 --- a/changelog.d/12513.feature +++ /dev/null @@ -1 +0,0 @@ -Measure the time taken in spam-checking callbacks and expose those measurements as metrics. diff --git a/changelog.d/12567.misc b/changelog.d/12567.misc deleted file mode 100644 index 35f08569bada..000000000000 --- a/changelog.d/12567.misc +++ /dev/null @@ -1 +0,0 @@ -Replace string literal instances of stream key types with typed constants. \ No newline at end of file diff --git a/changelog.d/12586.misc b/changelog.d/12586.misc deleted file mode 100644 index d26e332305ce..000000000000 --- a/changelog.d/12586.misc +++ /dev/null @@ -1 +0,0 @@ -Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. diff --git a/changelog.d/12588.misc b/changelog.d/12588.misc deleted file mode 100644 index f62d5c8e210c..000000000000 --- a/changelog.d/12588.misc +++ /dev/null @@ -1 +0,0 @@ -Add ability to cancel disconnected requests to `SynapseRequest`. diff --git a/changelog.d/12611.bugfix b/changelog.d/12611.bugfix deleted file mode 100644 index 093c45a20b7f..000000000000 --- a/changelog.d/12611.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.7.0 that would prevent events from being sent to clients if there's a retention policy in the room when the support for retention policies is disabled. diff --git a/changelog.d/12618.feature b/changelog.d/12618.feature deleted file mode 100644 index 37fa03b3cb41..000000000000 --- a/changelog.d/12618.feature +++ /dev/null @@ -1 +0,0 @@ -Add a `default_power_level_content_override` config option to set default room power levels per room preset. diff --git a/changelog.d/12623.feature b/changelog.d/12623.feature deleted file mode 100644 index cdee19fafa36..000000000000 --- a/changelog.d/12623.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787). \ No newline at end of file diff --git a/changelog.d/12630.misc b/changelog.d/12630.misc deleted file mode 100644 index 43e12603e2d8..000000000000 --- a/changelog.d/12630.misc +++ /dev/null @@ -1 +0,0 @@ -Add a helper class for testing request cancellation. diff --git a/changelog.d/12672.feature b/changelog.d/12672.feature deleted file mode 100644 index b989e0d208c4..000000000000 --- a/changelog.d/12672.feature +++ /dev/null @@ -1 +0,0 @@ -Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. \ No newline at end of file diff --git a/changelog.d/12673.feature b/changelog.d/12673.feature deleted file mode 100644 index f2bddd6e1c27..000000000000 --- a/changelog.d/12673.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse will now reload [cache config](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#caching) when it receives a [SIGHUP](https://en.wikipedia.org/wiki/SIGHUP) signal. diff --git a/changelog.d/12676.misc b/changelog.d/12676.misc deleted file mode 100644 index 26490af00dee..000000000000 --- a/changelog.d/12676.misc +++ /dev/null @@ -1 +0,0 @@ -Improve documentation of the `synapse.push` module. diff --git a/changelog.d/12677.misc b/changelog.d/12677.misc deleted file mode 100644 index eed12e69e9ba..000000000000 --- a/changelog.d/12677.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor functions to on `PushRuleEvaluatorForEvent`. diff --git a/changelog.d/12679.misc b/changelog.d/12679.misc deleted file mode 100644 index 6df1116b49ee..000000000000 --- a/changelog.d/12679.misc +++ /dev/null @@ -1 +0,0 @@ -Preparation for database schema simplifications: stop writing to `event_reference_hashes`. diff --git a/changelog.d/12680.misc b/changelog.d/12680.misc deleted file mode 100644 index dfd1f0a6c658..000000000000 --- a/changelog.d/12680.misc +++ /dev/null @@ -1 +0,0 @@ -Remove code which updates unused database column `application_services_state.last_txn`. diff --git a/changelog.d/12683.bugfix b/changelog.d/12683.bugfix deleted file mode 100644 index 2ce84a223a37..000000000000 --- a/changelog.d/12683.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.57.0 where `/messages` would throw a 500 error when querying for a non-existent room. diff --git a/changelog.d/12687.bugfix b/changelog.d/12687.bugfix deleted file mode 100644 index 196d9766707a..000000000000 --- a/changelog.d/12687.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add a unique index to `state_group_edges` to prevent duplicates being accidentally introduced and the consequential impact to performance. \ No newline at end of file diff --git a/changelog.d/12689.misc b/changelog.d/12689.misc deleted file mode 100644 index daa484ea3019..000000000000 --- a/changelog.d/12689.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `EventContext` class. diff --git a/changelog.d/12691.misc b/changelog.d/12691.misc deleted file mode 100644 index c63543421111..000000000000 --- a/changelog.d/12691.misc +++ /dev/null @@ -1 +0,0 @@ -Remove an unneeded class in the push code. diff --git a/changelog.d/12693.misc b/changelog.d/12693.misc deleted file mode 100644 index 8bd1e1cb0cd5..000000000000 --- a/changelog.d/12693.misc +++ /dev/null @@ -1 +0,0 @@ -Consolidate parsing of relation information from events. diff --git a/changelog.d/12694.misc b/changelog.d/12694.misc deleted file mode 100644 index e1e956a51301..000000000000 --- a/changelog.d/12694.misc +++ /dev/null @@ -1 +0,0 @@ -Capture the `Deferred` for request cancellation in `_AsyncResource`. diff --git a/changelog.d/12695.misc b/changelog.d/12695.misc deleted file mode 100644 index 1b39d969a4c5..000000000000 --- a/changelog.d/12695.misc +++ /dev/null @@ -1 +0,0 @@ -Fixes an incorrect type hint for `Filter._check_event_relations`. diff --git a/changelog.d/12696.bugfix b/changelog.d/12696.bugfix deleted file mode 100644 index e410184a22af..000000000000 --- a/changelog.d/12696.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where an empty room would be created when a user with an insufficient power level tried to upgrade a room. diff --git a/changelog.d/12698.misc b/changelog.d/12698.misc deleted file mode 100644 index 5d626352f9c2..000000000000 --- a/changelog.d/12698.misc +++ /dev/null @@ -1 +0,0 @@ -Respect the `@cancellable` flag for `DirectServe{Html,Json}Resource`s. diff --git a/changelog.d/12699.misc b/changelog.d/12699.misc deleted file mode 100644 index d278a956c7a9..000000000000 --- a/changelog.d/12699.misc +++ /dev/null @@ -1 +0,0 @@ -Respect the `@cancellable` flag for `RestServlet`s and `BaseFederationServlet`s. diff --git a/changelog.d/12700.misc b/changelog.d/12700.misc deleted file mode 100644 index d93eb5dada74..000000000000 --- a/changelog.d/12700.misc +++ /dev/null @@ -1 +0,0 @@ -Respect the `@cancellable` flag for `ReplicationEndpoint`s. diff --git a/changelog.d/12701.feature b/changelog.d/12701.feature deleted file mode 100644 index bb2264602c84..000000000000 --- a/changelog.d/12701.feature +++ /dev/null @@ -1 +0,0 @@ -Add a config options to allow for auto-tuning of caches. diff --git a/changelog.d/12703.misc b/changelog.d/12703.misc deleted file mode 100644 index 9aaa1bbaa3d0..000000000000 --- a/changelog.d/12703.misc +++ /dev/null @@ -1 +0,0 @@ -Convert namespace class `Codes` into a string enum. \ No newline at end of file diff --git a/changelog.d/12705.misc b/changelog.d/12705.misc deleted file mode 100644 index a913d8bb85eb..000000000000 --- a/changelog.d/12705.misc +++ /dev/null @@ -1 +0,0 @@ -Complain if a federation endpoint has the `@cancellable` flag, since some of the wrapper code may not handle cancellation correctly yet. diff --git a/changelog.d/12708.misc b/changelog.d/12708.misc deleted file mode 100644 index aa99e7311b97..000000000000 --- a/changelog.d/12708.misc +++ /dev/null @@ -1 +0,0 @@ -Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. diff --git a/changelog.d/12709.removal b/changelog.d/12709.removal deleted file mode 100644 index 6bb03e28941f..000000000000 --- a/changelog.d/12709.removal +++ /dev/null @@ -1 +0,0 @@ -Require a body in POST requests to `/rooms/{roomId}/receipt/{receiptType}/{eventId}`, as required by the [Matrix specification](https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidreceiptreceipttypeeventid). This breaks compatibility with Element Android 1.2.0 and earlier: users of those clients will be unable to send read receipts. diff --git a/changelog.d/12711.misc b/changelog.d/12711.misc deleted file mode 100644 index 0831ce045268..000000000000 --- a/changelog.d/12711.misc +++ /dev/null @@ -1 +0,0 @@ -Optimize private read receipt filtering. diff --git a/changelog.d/12713.bugfix b/changelog.d/12713.bugfix deleted file mode 100644 index 91e70f102c5d..000000000000 --- a/changelog.d/12713.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.30.0 where empty rooms could be automatically created if a monthly active users limit is set. diff --git a/changelog.d/12715.doc b/changelog.d/12715.doc deleted file mode 100644 index 150d78c3f634..000000000000 --- a/changelog.d/12715.doc +++ /dev/null @@ -1 +0,0 @@ -Fix a typo in the Media Admin API documentation. diff --git a/changelog.d/12716.misc b/changelog.d/12716.misc deleted file mode 100644 index b07e1b52ee7c..000000000000 --- a/changelog.d/12716.misc +++ /dev/null @@ -1 +0,0 @@ -Add type annotations to increase the number of modules passing `disallow-untyped-defs`. \ No newline at end of file diff --git a/changelog.d/12717.misc b/changelog.d/12717.misc deleted file mode 100644 index e793d08e5e3f..000000000000 --- a/changelog.d/12717.misc +++ /dev/null @@ -1 +0,0 @@ -Add some type hints to datastore. \ No newline at end of file diff --git a/changelog.d/12718.feature b/changelog.d/12718.feature deleted file mode 100644 index 1056f519a4c1..000000000000 --- a/changelog.d/12718.feature +++ /dev/null @@ -1 +0,0 @@ -Update [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to process marker events from the current state to avoid markers being lost in timeline gaps for federated servers which would cause the imported history to be undiscovered. diff --git a/changelog.d/12720.misc b/changelog.d/12720.misc deleted file mode 100644 index 01b427f200ae..000000000000 --- a/changelog.d/12720.misc +++ /dev/null @@ -1 +0,0 @@ -Drop the logging level of status messages for the URL preview cache expiry job from INFO to DEBUG. \ No newline at end of file diff --git a/changelog.d/12721.bugfix b/changelog.d/12721.bugfix deleted file mode 100644 index 6987f7ab15e1..000000000000 --- a/changelog.d/12721.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix push to dismiss notifications when read on another client. Contributed by @SpiritCroc @ Beeper. diff --git a/changelog.d/12723.misc b/changelog.d/12723.misc deleted file mode 100644 index 4f5bffeda639..000000000000 --- a/changelog.d/12723.misc +++ /dev/null @@ -1 +0,0 @@ -Downgrade some OIDC errors to warnings in the logs, to reduce the noise of Sentry reports. diff --git a/changelog.d/12726.misc b/changelog.d/12726.misc deleted file mode 100644 index b07e1b52ee7c..000000000000 --- a/changelog.d/12726.misc +++ /dev/null @@ -1 +0,0 @@ -Add type annotations to increase the number of modules passing `disallow-untyped-defs`. \ No newline at end of file diff --git a/changelog.d/12727.doc b/changelog.d/12727.doc deleted file mode 100644 index c41e50c85ba0..000000000000 --- a/changelog.d/12727.doc +++ /dev/null @@ -1 +0,0 @@ -Update the OpenID Connect example for Keycloak to be compatible with newer versions of Keycloak. Contributed by @nhh. diff --git a/changelog.d/12731.misc b/changelog.d/12731.misc deleted file mode 100644 index 962100d516c1..000000000000 --- a/changelog.d/12731.misc +++ /dev/null @@ -1 +0,0 @@ -Update configs used by Complement to allow more invites/3PID validations during tests. \ No newline at end of file diff --git a/changelog.d/12734.misc b/changelog.d/12734.misc deleted file mode 100644 index ffbfb0d63233..000000000000 --- a/changelog.d/12734.misc +++ /dev/null @@ -1 +0,0 @@ -Tidy up and type-hint the database engine modules. diff --git a/changelog.d/12742.doc b/changelog.d/12742.doc deleted file mode 100644 index 0084e27a7d03..000000000000 --- a/changelog.d/12742.doc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in server listener documentation. \ No newline at end of file diff --git a/changelog.d/12744.feature b/changelog.d/12744.feature deleted file mode 100644 index 9836d94f8ca6..000000000000 --- a/changelog.d/12744.feature +++ /dev/null @@ -1 +0,0 @@ -Add a `drop_federated_event` callback to `SpamChecker` to disregard inbound federated events before they take up much processing power, in an emergency. diff --git a/changelog.d/12747.bugfix b/changelog.d/12747.bugfix deleted file mode 100644 index 0fb0059237cc..000000000000 --- a/changelog.d/12747.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix poor database performance when reading the cache invalidation stream for large servers with lots of workers. diff --git a/changelog.d/12748.doc b/changelog.d/12748.doc deleted file mode 100644 index 996ad3a1b926..000000000000 --- a/changelog.d/12748.doc +++ /dev/null @@ -1 +0,0 @@ -Link to the configuration manual from the welcome page of the documentation. diff --git a/changelog.d/12749.doc b/changelog.d/12749.doc deleted file mode 100644 index 4560319ee43b..000000000000 --- a/changelog.d/12749.doc +++ /dev/null @@ -1 +0,0 @@ -Fix typo in 'run_background_tasks_on' option name in configuration manual documentation. diff --git a/changelog.d/12753.misc b/changelog.d/12753.misc deleted file mode 100644 index e793d08e5e3f..000000000000 --- a/changelog.d/12753.misc +++ /dev/null @@ -1 +0,0 @@ -Add some type hints to datastore. \ No newline at end of file diff --git a/changelog.d/12759.doc b/changelog.d/12759.doc deleted file mode 100644 index 45d1c9c0ca1a..000000000000 --- a/changelog.d/12759.doc +++ /dev/null @@ -1 +0,0 @@ -Add information regarding the `rc_invites` ratelimiting option to the configuration docs. diff --git a/changelog.d/12761.doc b/changelog.d/12761.doc deleted file mode 100644 index 2eb2c0976f1b..000000000000 --- a/changelog.d/12761.doc +++ /dev/null @@ -1 +0,0 @@ -Add documentation for cancellation of request processing. diff --git a/changelog.d/12762.misc b/changelog.d/12762.misc deleted file mode 100644 index 990fb6fe74eb..000000000000 --- a/changelog.d/12762.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. diff --git a/changelog.d/12765.doc b/changelog.d/12765.doc deleted file mode 100644 index 277b037d6b03..000000000000 --- a/changelog.d/12765.doc +++ /dev/null @@ -1 +0,0 @@ -Recommend using docker to run tests against postgres. diff --git a/changelog.d/12769.misc b/changelog.d/12769.misc deleted file mode 100644 index 27bd53abe376..000000000000 --- a/changelog.d/12769.misc +++ /dev/null @@ -1 +0,0 @@ -Tweak the mypy plugin so that `@cached` can accept `on_invalidate=None`. diff --git a/changelog.d/12770.bugfix b/changelog.d/12770.bugfix deleted file mode 100644 index a958f9a16ba3..000000000000 --- a/changelog.d/12770.bugfix +++ /dev/null @@ -1 +0,0 @@ -Delete events from the `federation_inbound_events_staging` table when a room is purged through the admin API. diff --git a/changelog.d/12772.misc b/changelog.d/12772.misc deleted file mode 100644 index da66f376fe8e..000000000000 --- a/changelog.d/12772.misc +++ /dev/null @@ -1 +0,0 @@ -Move methods that call `add_push_rule` to the `PushRuleStore` class. diff --git a/changelog.d/12773.doc b/changelog.d/12773.doc deleted file mode 100644 index 6de371653427..000000000000 --- a/changelog.d/12773.doc +++ /dev/null @@ -1 +0,0 @@ -Add missing user directory endpoint from the generic worker documentation. Contributed by @olmari. \ No newline at end of file diff --git a/changelog.d/12774.misc b/changelog.d/12774.misc deleted file mode 100644 index 8651f2e0e062..000000000000 --- a/changelog.d/12774.misc +++ /dev/null @@ -1 +0,0 @@ -Make handling of federation Authorization header (more) compliant with RFC7230. diff --git a/changelog.d/12775.misc b/changelog.d/12775.misc deleted file mode 100644 index eac326cde3a7..000000000000 --- a/changelog.d/12775.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. \ No newline at end of file diff --git a/changelog.d/12776.doc b/changelog.d/12776.doc deleted file mode 100644 index c00489a8ce14..000000000000 --- a/changelog.d/12776.doc +++ /dev/null @@ -1,2 +0,0 @@ -Add additional info to documentation of config option `cache_autotuning`. - diff --git a/changelog.d/12777.doc b/changelog.d/12777.doc deleted file mode 100644 index cc9c07704d02..000000000000 --- a/changelog.d/12777.doc +++ /dev/null @@ -1,2 +0,0 @@ -Update configuration manual documentation to document size-related suffixes. - diff --git a/changelog.d/12779.bugfix b/changelog.d/12779.bugfix deleted file mode 100644 index 7cf7a1f65f24..000000000000 --- a/changelog.d/12779.bugfix +++ /dev/null @@ -1 +0,0 @@ -Give a meaningful error message when a client tries to create a room with an invalid alias localpart. \ No newline at end of file diff --git a/changelog.d/12781.misc b/changelog.d/12781.misc deleted file mode 100644 index 8a045716172a..000000000000 --- a/changelog.d/12781.misc +++ /dev/null @@ -1 +0,0 @@ -Do not keep going if there are 5 back-to-back background update failures. \ No newline at end of file diff --git a/changelog.d/12783.misc b/changelog.d/12783.misc deleted file mode 100644 index 97575608bb8b..000000000000 --- a/changelog.d/12783.misc +++ /dev/null @@ -1 +0,0 @@ -Fix federation when using the demo scripts. diff --git a/changelog.d/12785.doc b/changelog.d/12785.doc deleted file mode 100644 index 5209dfeb053e..000000000000 --- a/changelog.d/12785.doc +++ /dev/null @@ -1 +0,0 @@ -Fix invalid YAML syntax in the example documentation for the `url_preview_accept_language` config option. diff --git a/changelog.d/12786.feature b/changelog.d/12786.feature deleted file mode 100644 index c90ddd411ee2..000000000000 --- a/changelog.d/12786.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). diff --git a/changelog.d/12789.misc b/changelog.d/12789.misc deleted file mode 100644 index 3398d00110c8..000000000000 --- a/changelog.d/12789.misc +++ /dev/null @@ -1 +0,0 @@ -The `hash_password` script now fails when it is called without specifying a config file. diff --git a/changelog.d/12790.misc b/changelog.d/12790.misc deleted file mode 100644 index b78156cf4e1d..000000000000 --- a/changelog.d/12790.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify `disallow_untyped_defs` config in `mypy.ini`. diff --git a/changelog.d/12791.misc b/changelog.d/12791.misc deleted file mode 100644 index b6e92b7eafad..000000000000 --- a/changelog.d/12791.misc +++ /dev/null @@ -1 +0,0 @@ -Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible. diff --git a/changelog.d/12792.feature b/changelog.d/12792.feature deleted file mode 100644 index 4778b8a394d4..000000000000 --- a/changelog.d/12792.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). \ No newline at end of file diff --git a/changelog.d/12794.bugfix b/changelog.d/12794.bugfix deleted file mode 100644 index 2d1a2838e128..000000000000 --- a/changelog.d/12794.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in 1.43.0 where a file (`providers.json`) was never closed. Contributed by @arkamar. diff --git a/changelog.d/12803.bugfix b/changelog.d/12803.bugfix deleted file mode 100644 index 6ddd3d24e05f..000000000000 --- a/changelog.d/12803.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where finished log contexts would be re-started when failing to contact remote homeservers. diff --git a/changelog.d/12808.feature b/changelog.d/12808.feature deleted file mode 100644 index 561c8b9d34a4..000000000000 --- a/changelog.d/12808.feature +++ /dev/null @@ -1 +0,0 @@ -Update to `check_event_for_spam`. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). \ No newline at end of file diff --git a/changelog.d/12809.feature b/changelog.d/12809.feature deleted file mode 100644 index b989e0d208c4..000000000000 --- a/changelog.d/12809.feature +++ /dev/null @@ -1 +0,0 @@ -Send `USER_IP` commands on a different Redis channel, in order to reduce traffic to workers that do not process these commands. \ No newline at end of file diff --git a/changelog.d/12818.misc b/changelog.d/12818.misc deleted file mode 100644 index 2f9dacc21dd9..000000000000 --- a/changelog.d/12818.misc +++ /dev/null @@ -1 +0,0 @@ -Remove Caddy from the Synapse workers image used in Complement. \ No newline at end of file diff --git a/changelog.d/12819.misc b/changelog.d/12819.misc deleted file mode 100644 index 7a03102a632d..000000000000 --- a/changelog.d/12819.misc +++ /dev/null @@ -1 +0,0 @@ -Add Complement's shared registration secret to the Complement worker image. This fixes tests that depend on it. \ No newline at end of file diff --git a/changelog.d/12823.bugfix b/changelog.d/12823.bugfix deleted file mode 100644 index 1a1f5957e712..000000000000 --- a/changelog.d/12823.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug, introduced in Synapse 1.21.0, that led to media thumbnails being unusable before the index has been added in the background. diff --git a/changelog.d/12826.misc b/changelog.d/12826.misc deleted file mode 100644 index f5e91f1ed592..000000000000 --- a/changelog.d/12826.misc +++ /dev/null @@ -1 +0,0 @@ -Support registering Application Services when running with workers under Complement. \ No newline at end of file diff --git a/changelog.d/12833.misc b/changelog.d/12833.misc deleted file mode 100644 index fad5df1afa34..000000000000 --- a/changelog.d/12833.misc +++ /dev/null @@ -1 +0,0 @@ -Add some type hints to test files. \ No newline at end of file diff --git a/changelog.d/12842.misc b/changelog.d/12842.misc deleted file mode 100644 index cec3f97d86fd..000000000000 --- a/changelog.d/12842.misc +++ /dev/null @@ -1 +0,0 @@ -Disable 'faster room join' Complement tests when testing against Synapse with workers. \ No newline at end of file diff --git a/changelog.d/12853.docker b/changelog.d/12853.docker deleted file mode 100644 index cad10a79cc82..000000000000 --- a/changelog.d/12853.docker +++ /dev/null @@ -1 +0,0 @@ -Fix the docker file after a dependency update. diff --git a/debian/changelog b/debian/changelog index dda342a630db..6eba9b3a1bbd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.60.0~rc1) stable; urgency=medium + + * New Synapse release 1.60.0rc1. + + -- Synapse Packaging team Tue, 24 May 2022 12:05:01 +0100 + matrix-synapse-py3 (1.59.1) stable; urgency=medium * New Synapse release 1.59.1. diff --git a/pyproject.toml b/pyproject.toml index 5a5a2eaba73d..9359d211f79e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.59.1" +version = "1.60.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 9385cd063375d351c5cb01ffce00d47cdf482bcd Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Tue, 24 May 2022 13:21:15 +0100 Subject: [PATCH 257/263] Update changelog --- CHANGES.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f6ca5c472144..46ac3fce7a71 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,19 @@ Synapse 1.60.0rc1 (2022-05-24) ============================== +This release of Synapse adds a unique index to the `state_group_edges` table, in +order to prevent accidentally introducing duplicate information (for example, +because a database backup was restored multiple times). If your Synapse database +already has duplicate rows in this table, this could fail with an error and +require manual remediation. + +Additionally, the signature of the `check_event_for_spam` module callback has changed. +The previous signature has been deprecated and remains working for now. Module authors +should update their modules to use the new signature where possible. + +See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1600) +for more details. + Features -------- @@ -13,7 +26,7 @@ Features - Update [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to process marker events from the current state to avoid markers being lost in timeline gaps for federated servers which would cause the imported history to be undiscovered. ([\#12718](https://github.com/matrix-org/synapse/issues/12718)) - Add a `drop_federated_event` callback to `SpamChecker` to disregard inbound federated events before they take up much processing power, in an emergency. ([\#12744](https://github.com/matrix-org/synapse/issues/12744)) - Implement [MSC3818: Copy room type on upgrade](https://github.com/matrix-org/matrix-spec-proposals/pull/3818). ([\#12786](https://github.com/matrix-org/synapse/issues/12786), [\#12792](https://github.com/matrix-org/synapse/issues/12792)) -- Update to `check_event_for_spam`. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). ([\#12808](https://github.com/matrix-org/synapse/issues/12808)) +- Update to the `check_event_for_spam` module callback. Deprecate the current callback signature, replace it with a new signature that is both less ambiguous (replacing booleans with explicit allow/block) and more powerful (ability to return explicit error codes). ([\#12808](https://github.com/matrix-org/synapse/issues/12808)) Bugfixes @@ -46,7 +59,7 @@ Improved Documentation - Update the OpenID Connect example for Keycloak to be compatible with newer versions of Keycloak. Contributed by @nhh. ([\#12727](https://github.com/matrix-org/synapse/issues/12727)) - Fix typo in server listener documentation. ([\#12742](https://github.com/matrix-org/synapse/issues/12742)) - Link to the configuration manual from the welcome page of the documentation. ([\#12748](https://github.com/matrix-org/synapse/issues/12748)) -- Fix typo in 'run_background_tasks_on' option name in configuration manual documentation. ([\#12749](https://github.com/matrix-org/synapse/issues/12749)) +- Fix typo in `run_background_tasks_on` option name in configuration manual documentation. ([\#12749](https://github.com/matrix-org/synapse/issues/12749)) - Add information regarding the `rc_invites` ratelimiting option to the configuration docs. ([\#12759](https://github.com/matrix-org/synapse/issues/12759)) - Add documentation for cancellation of request processing. ([\#12761](https://github.com/matrix-org/synapse/issues/12761)) - Recommend using docker to run tests against postgres. ([\#12765](https://github.com/matrix-org/synapse/issues/12765)) @@ -66,12 +79,9 @@ Internal Changes ---------------- - Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10533](https://github.com/matrix-org/synapse/issues/10533)) -- Add some type hints to datastore. ([\#12477](https://github.com/matrix-org/synapse/issues/12477), [\#12717](https://github.com/matrix-org/synapse/issues/12717), [\#12753](https://github.com/matrix-org/synapse/issues/12753)) - Preparation for faster-room-join work: return subsets of room state which we already have, immediately. ([\#12498](https://github.com/matrix-org/synapse/issues/12498)) -- Replace string literal instances of stream key types with typed constants. ([\#12567](https://github.com/matrix-org/synapse/issues/12567)) -- Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. ([\#12586](https://github.com/matrix-org/synapse/issues/12586)) -- Add ability to cancel disconnected requests to `SynapseRequest`. ([\#12588](https://github.com/matrix-org/synapse/issues/12588)) -- Add a helper class for testing request cancellation. ([\#12630](https://github.com/matrix-org/synapse/issues/12630)) +- Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect. ([\#12586](https://github.com/matrix-org/synapse/issues/12586), [\#12588](https://github.com/matrix-org/synapse/issues/12588), [\#12630](https://github.com/matrix-org/synapse/issues/12630), [\#12694](https://github.com/matrix-org/synapse/issues/12694), [\#12698](https://github.com/matrix-org/synapse/issues/12698), [\#12699](https://github.com/matrix-org/synapse/issues/12699), [\#12700](https://github.com/matrix-org/synapse/issues/12700), [\#12705](https://github.com/matrix-org/synapse/issues/12705)) +- Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. ([\#12708](https://github.com/matrix-org/synapse/issues/12708)) - Improve documentation of the `synapse.push` module. ([\#12676](https://github.com/matrix-org/synapse/issues/12676)) - Refactor functions to on `PushRuleEvaluatorForEvent`. ([\#12677](https://github.com/matrix-org/synapse/issues/12677)) - Preparation for database schema simplifications: stop writing to `event_reference_hashes`. ([\#12679](https://github.com/matrix-org/synapse/issues/12679)) @@ -79,20 +89,11 @@ Internal Changes - Refactor `EventContext` class. ([\#12689](https://github.com/matrix-org/synapse/issues/12689)) - Remove an unneeded class in the push code. ([\#12691](https://github.com/matrix-org/synapse/issues/12691)) - Consolidate parsing of relation information from events. ([\#12693](https://github.com/matrix-org/synapse/issues/12693)) -- Capture the `Deferred` for request cancellation in `_AsyncResource`. ([\#12694](https://github.com/matrix-org/synapse/issues/12694)) -- Fixes an incorrect type hint for `Filter._check_event_relations`. ([\#12695](https://github.com/matrix-org/synapse/issues/12695)) -- Respect the `@cancellable` flag for `DirectServe{Html,Json}Resource`s. ([\#12698](https://github.com/matrix-org/synapse/issues/12698)) -- Respect the `@cancellable` flag for `RestServlet`s and `BaseFederationServlet`s. ([\#12699](https://github.com/matrix-org/synapse/issues/12699)) -- Respect the `@cancellable` flag for `ReplicationEndpoint`s. ([\#12700](https://github.com/matrix-org/synapse/issues/12700)) - Convert namespace class `Codes` into a string enum. ([\#12703](https://github.com/matrix-org/synapse/issues/12703)) -- Complain if a federation endpoint has the `@cancellable` flag, since some of the wrapper code may not handle cancellation correctly yet. ([\#12705](https://github.com/matrix-org/synapse/issues/12705)) -- Enable cancellation of `GET /rooms/$room_id/members`, `GET /rooms/$room_id/state` and `GET /rooms/$room_id/state/$event_type/*` requests. ([\#12708](https://github.com/matrix-org/synapse/issues/12708)) - Optimize private read receipt filtering. ([\#12711](https://github.com/matrix-org/synapse/issues/12711)) -- Add type annotations to increase the number of modules passing `disallow-untyped-defs`. ([\#12716](https://github.com/matrix-org/synapse/issues/12716), [\#12726](https://github.com/matrix-org/synapse/issues/12726)) - Drop the logging level of status messages for the URL preview cache expiry job from INFO to DEBUG. ([\#12720](https://github.com/matrix-org/synapse/issues/12720)) - Downgrade some OIDC errors to warnings in the logs, to reduce the noise of Sentry reports. ([\#12723](https://github.com/matrix-org/synapse/issues/12723)) - Update configs used by Complement to allow more invites/3PID validations during tests. ([\#12731](https://github.com/matrix-org/synapse/issues/12731)) -- Tidy up and type-hint the database engine modules. ([\#12734](https://github.com/matrix-org/synapse/issues/12734)) - Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar. ([\#12762](https://github.com/matrix-org/synapse/issues/12762)) - Tweak the mypy plugin so that `@cached` can accept `on_invalidate=None`. ([\#12769](https://github.com/matrix-org/synapse/issues/12769)) - Move methods that call `add_push_rule` to the `PushRuleStore` class. ([\#12772](https://github.com/matrix-org/synapse/issues/12772)) @@ -100,13 +101,12 @@ Internal Changes - Refactor `resolve_state_groups_for_events` to not pull out full state when no state resolution happens. ([\#12775](https://github.com/matrix-org/synapse/issues/12775)) - Do not keep going if there are 5 back-to-back background update failures. ([\#12781](https://github.com/matrix-org/synapse/issues/12781)) - Fix federation when using the demo scripts. ([\#12783](https://github.com/matrix-org/synapse/issues/12783)) -- The `hash_password` script now fails when it is called without specifying a config file. ([\#12789](https://github.com/matrix-org/synapse/issues/12789)) -- Simplify `disallow_untyped_defs` config in `mypy.ini`. ([\#12790](https://github.com/matrix-org/synapse/issues/12790)) +- The `hash_password` script now fails when it is called without specifying a config file. Contributed by @jae1911. ([\#12789](https://github.com/matrix-org/synapse/issues/12789)) +- Improve and fix type hints. ([\#12567](https://github.com/matrix-org/synapse/issues/12567), [\#12477](https://github.com/matrix-org/synapse/issues/12477), [\#12717](https://github.com/matrix-org/synapse/issues/12717), [\#12753](https://github.com/matrix-org/synapse/issues/12753), [\#12695](https://github.com/matrix-org/synapse/issues/12695), [\#12734](https://github.com/matrix-org/synapse/issues/12734), [\#12716](https://github.com/matrix-org/synapse/issues/12716), [\#12726](https://github.com/matrix-org/synapse/issues/12726), [\#12790](https://github.com/matrix-org/synapse/issues/12790), [\#12833](https://github.com/matrix-org/synapse/issues/12833)) - Update EventContext `get_current_event_ids` and `get_prev_event_ids` to accept state filters and update calls where possible. ([\#12791](https://github.com/matrix-org/synapse/issues/12791)) - Remove Caddy from the Synapse workers image used in Complement. ([\#12818](https://github.com/matrix-org/synapse/issues/12818)) - Add Complement's shared registration secret to the Complement worker image. This fixes tests that depend on it. ([\#12819](https://github.com/matrix-org/synapse/issues/12819)) - Support registering Application Services when running with workers under Complement. ([\#12826](https://github.com/matrix-org/synapse/issues/12826)) -- Add some type hints to test files. ([\#12833](https://github.com/matrix-org/synapse/issues/12833)) - Disable 'faster room join' Complement tests when testing against Synapse with workers. ([\#12842](https://github.com/matrix-org/synapse/issues/12842)) From 317248d42cb05ffa39119d6fefb7da286cb46225 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 26 May 2022 16:07:27 +0100 Subject: [PATCH 258/263] Improve URL previews by not including the content of media tags in the generated description. (#12887) --- changelog.d/12887.misc | 1 + synapse/rest/media/v1/preview_html.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/12887.misc diff --git a/changelog.d/12887.misc b/changelog.d/12887.misc new file mode 100644 index 000000000000..7f6f7318329f --- /dev/null +++ b/changelog.d/12887.misc @@ -0,0 +1 @@ +Improve URL previews by not including the content of media tags in the generated description. \ No newline at end of file diff --git a/synapse/rest/media/v1/preview_html.py b/synapse/rest/media/v1/preview_html.py index ca73965fc28f..0358c68a6452 100644 --- a/synapse/rest/media/v1/preview_html.py +++ b/synapse/rest/media/v1/preview_html.py @@ -246,7 +246,9 @@ def parse_html_description(tree: "etree.Element") -> Optional[str]: Grabs any text nodes which are inside the tag, unless they are within an HTML5 semantic markup tag (
,