From 986e15c6e402bc16b8aca3b483378dab45c372bf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 8 Sep 2025 22:53:43 -0500 Subject: [PATCH 1/4] Wrap the Rust HTTP client with `make_deferred_yieldable` So downstream usage doesn't need to use `PreserveLoggingContext()` or `make_deferred_yieldable` Spawning from https://github.com/element-hq/synapse/pull/18870 and https://github.com/element-hq/synapse/pull/18357#discussion_r2294941827 --- synapse/api/auth/mas.py | 16 ++++----- synapse/api/auth/msc3861_delegated.py | 16 ++++----- synapse/synapse_rust/http_client.pyi | 7 ++++ synapse/synapse_rust_wrapper/__init__.py | 11 ++++++ synapse/synapse_rust_wrapper/http_client.py | 39 +++++++++++++++++++++ 5 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 synapse/synapse_rust_wrapper/__init__.py create mode 100644 synapse/synapse_rust_wrapper/http_client.py diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index 40b4a5bd34b..8d66c21faac 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -33,7 +33,6 @@ UnrecognizedRequestError, ) from synapse.http.site import SynapseRequest -from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import ( active_span, force_tracing, @@ -41,7 +40,7 @@ start_active_span, ) from synapse.metrics import SERVER_NAME_LABEL -from synapse.synapse_rust.http_client import HttpClient +from synapse.synapse_rust_wrapper.http_client import HttpClient from synapse.types import JsonDict, Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall @@ -229,13 +228,12 @@ async def _introspect_token( try: with start_active_span("mas-introspect-token"): inject_request_headers(raw_headers) - with PreserveLoggingContext(): - resp_body = await self._rust_http_client.post( - url=self._introspection_endpoint, - response_limit=1 * 1024 * 1024, - headers=raw_headers, - request_body=body, - ) + resp_body = await self._rust_http_client.post( + url=self._introspection_endpoint, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) except HttpResponseException as e: end_time = self._clock.time() introspection_response_timer.labels( diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index c406c683e71..bb84ed2c78a 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -38,7 +38,6 @@ UnrecognizedRequestError, ) from synapse.http.site import SynapseRequest -from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import ( active_span, force_tracing, @@ -46,7 +45,7 @@ start_active_span, ) from synapse.metrics import SERVER_NAME_LABEL -from synapse.synapse_rust.http_client import HttpClient +from synapse.synapse_rust_wrapper.http_client import HttpClient from synapse.types import Requester, UserID, create_requester from synapse.util import json_decoder from synapse.util.caches.cached_call import RetryOnExceptionCachedCall @@ -327,13 +326,12 @@ async def _introspect_token( try: with start_active_span("mas-introspect-token"): inject_request_headers(raw_headers) - with PreserveLoggingContext(): - resp_body = await self._rust_http_client.post( - url=uri, - response_limit=1 * 1024 * 1024, - headers=raw_headers, - request_body=body, - ) + resp_body = await self._rust_http_client.post( + url=uri, + response_limit=1 * 1024 * 1024, + headers=raw_headers, + request_body=body, + ) except HttpResponseException as e: end_time = self._clock.time() introspection_response_timer.labels( diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi index 9fb7831e6b6..1635c9afa12 100644 --- a/synapse/synapse_rust/http_client.pyi +++ b/synapse/synapse_rust/http_client.pyi @@ -17,6 +17,13 @@ from twisted.internet.defer import Deferred from synapse.types import ISynapseReactor class HttpClient: + """ + Since the returned deferreds don't follow Synapse logcontext rules, + this is not meant to be used by Synapse code directly. + + Use `synapse.synapse_rust_wrapper.http_client.HttpClient` instead. + """ + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: ... def get(self, url: str, response_limit: int) -> Deferred[bytes]: ... def post( diff --git a/synapse/synapse_rust_wrapper/__init__.py b/synapse/synapse_rust_wrapper/__init__.py new file mode 100644 index 00000000000..e056679fd55 --- /dev/null +++ b/synapse/synapse_rust_wrapper/__init__.py @@ -0,0 +1,11 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/synapse/synapse_rust_wrapper/http_client.py b/synapse/synapse_rust_wrapper/http_client.py new file mode 100644 index 00000000000..ed004a81a9a --- /dev/null +++ b/synapse/synapse_rust_wrapper/http_client.py @@ -0,0 +1,39 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +from typing import Mapping + +from twisted.internet.defer import Deferred + +from synapse.logging.context import make_deferred_yieldable +from synapse.synapse_rust.http_client import HttpClient as RustHttpClient +from synapse.types import ISynapseReactor + + +class HttpClient: + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: + self._http_client = RustHttpClient(reactor, user_agent) + + def get(self, url: str, response_limit: int) -> Deferred[bytes]: + deferred = self._http_client.get(url, response_limit) + return make_deferred_yieldable(deferred) + + def post( + self, + url: str, + response_limit: int, + headers: Mapping[str, str], + request_body: str, + ) -> Deferred[bytes]: + deferred = self._http_client.post(url, response_limit, headers, request_body) + return make_deferred_yieldable(deferred) From 5e555f3abb6602160a511b9f2ba01b1bb43d92c5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 8 Sep 2025 22:59:01 -0500 Subject: [PATCH 2/4] Add chagnelog --- changelog.d/18903.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/18903.misc diff --git a/changelog.d/18903.misc b/changelog.d/18903.misc new file mode 100644 index 00000000000..bafa7dad5cf --- /dev/null +++ b/changelog.d/18903.misc @@ -0,0 +1 @@ +Wrap the Rust HTTP client with `make_deferred_yieldable` so it follows Synapse logcontext rules. From 4e0085c2213cc74d18fcd5383dc87aaa15e93ee2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 8 Sep 2025 23:00:29 -0500 Subject: [PATCH 3/4] Docstring for why we have a wrapper --- synapse/synapse_rust_wrapper/http_client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/synapse_rust_wrapper/http_client.py b/synapse/synapse_rust_wrapper/http_client.py index ed004a81a9a..4ac5aa9acdf 100644 --- a/synapse/synapse_rust_wrapper/http_client.py +++ b/synapse/synapse_rust_wrapper/http_client.py @@ -21,6 +21,11 @@ class HttpClient: + """ + Wrap `synapse.synapse_rust.http_client.HttpClient` to ensure the returned + deferreds follow Synapse logcontext rules. + """ + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: self._http_client = RustHttpClient(reactor, user_agent) From ce6f16deaa3de2884744faa5ab50535df7570229 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 9 Sep 2025 14:12:32 -0500 Subject: [PATCH 4/4] WIP: Add test --- tests/synapse_rust/__init__.py | 11 ++++ tests/synapse_rust/test_http_client.py | 87 ++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/synapse_rust/__init__.py create mode 100644 tests/synapse_rust/test_http_client.py diff --git a/tests/synapse_rust/__init__.py b/tests/synapse_rust/__init__.py new file mode 100644 index 00000000000..e056679fd55 --- /dev/null +++ b/tests/synapse_rust/__init__.py @@ -0,0 +1,11 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/tests/synapse_rust/test_http_client.py b/tests/synapse_rust/test_http_client.py new file mode 100644 index 00000000000..c6da4d6cbaa --- /dev/null +++ b/tests/synapse_rust/test_http_client.py @@ -0,0 +1,87 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import time +from typing import Any, Coroutine, Generator, TypeVar, Union + +from twisted.internet.defer import Deferred, ensureDeferred +from twisted.internet.testing import MemoryReactor + +from synapse.logging.context import LoggingContext +from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient +from synapse.util import Clock + +from tests.unittest import HomeserverTestCase + +T = TypeVar("T") + + +class HttpClientTestCase(HomeserverTestCase): + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self._http_client = hs.get_proxied_http_client() + self._rust_http_client = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + def till_deferred_has_result( + self, + awaitable: Union[ + "Coroutine[Deferred[Any], Any, T]", + "Generator[Deferred[Any], Any, T]", + "Deferred[T]", + ], + ) -> "Deferred[T]": + """Wait until a deferred has a result. + + This is useful because the Rust HTTP client will resolve the deferred + using reactor.callFromThread, which are only run when we call + reactor.advance. + """ + deferred = ensureDeferred(awaitable) + tries = 0 + while not deferred.called: + time.sleep(0.1) + self.reactor.advance(0) + tries += 1 + if tries > 100: + raise Exception("Timed out waiting for deferred to resolve") + + return deferred + + def test_logging_context(self) -> None: + async def asdf() -> None: + with LoggingContext("test"): + # TODO: Test logging context before/after this call + await self._rust_http_client.get( + url="http://localhost", + response_limit=1 * 1024 * 1024, + ) + + self.get_success(self.till_deferred_has_result(asdf()))