From 202b3635448c48530fe2a83ea476fa1f60098500 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 22 Mar 2023 11:50:57 +0000
Subject: [PATCH 1/2] Have replication clients remove _INT_STREAM_POS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Suppose worker A makes an internal http request from worker B. B may
make changes that A later learns about over replication. We want A's
request to block until it has seen those changes—mainly to ensure A's
caches are invalidated promptly. This helps provide read-after-write
consistency, eliminating entire categories of races and test flakes.

To implement this, B includes a top-level field `_INT_STREAM_POS` in its
response JSON. Roughly speaking, the field's value tells A what to wait
for. But we weren't removing that internal field before A's request
completed!

Introduced in https://github.com/matrix-org/synapse/pull/14820.
Fixes #15308.
---
 synapse/replication/http/_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index c20d9c7e9da7..8c2c54c07a49 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -345,7 +345,7 @@ async def send_request(*, instance_name: str = "master", **kwargs: Any) -> Any:
                 _outgoing_request_counter.labels(cls.NAME, 200).inc()
 
                 # Wait on any streams that the remote may have written to.
-                for stream_name, position in result.get(
+                for stream_name, position in result.pop(
                     _STREAM_POSITION_KEY, {}
                 ).items():
                     await replication.wait_for_stream_position(

From d0985389187261c5723ac73973879c32361e8ba1 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Wed, 22 Mar 2023 12:03:31 +0000
Subject: [PATCH 2/2] Changelog

---
 changelog.d/15309.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/15309.bugfix

diff --git a/changelog.d/15309.bugfix b/changelog.d/15309.bugfix
new file mode 100644
index 000000000000..4d3fe4e4b10b
--- /dev/null
+++ b/changelog.d/15309.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key.