diff --git a/tests/rest/client/v2_alpha/test_relations.py b/tests/rest/client/v2_alpha/test_relations.py
index 9c2022ba8fa7..1eed2be78c55 100644
--- a/tests/rest/client/v2_alpha/test_relations.py
+++ b/tests/rest/client/v2_alpha/test_relations.py
@@ -17,7 +17,6 @@
 import json
 import urllib
 
-from synapse import events
 from synapse.api.constants import EventTypes, RelationTypes
 from synapse.rest import admin
 from synapse.rest.client.v1 import login, room
@@ -37,13 +36,14 @@ class RelationsTestCase(unittest.HomeserverTestCase):
     hijack_auth = False
 
     def make_homeserver(self, reactor, clock):
-        # We enable frozen dicts as relations/edits change event contents, so we
-        # want to test that we don't modify the events in the caches.
-        events.USE_FROZEN_DICTS = True
-
         # We need to enable msc1849 support for aggregations
         config = self.default_config()
         config["experimental_msc1849_support_enabled"] = True
+
+        # We enable frozen dicts as relations/edits change event contents, so we
+        # want to test that we don't modify the events in the caches.
+        config["use_frozen_dicts"] = True
+
         return self.setup_test_homeserver(config=config)
 
     def prepare(self, reactor, clock, hs):
diff --git a/tests/unittest.py b/tests/unittest.py
index ca7031c724b3..195598035727 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -32,6 +32,7 @@
 from twisted.trial import unittest
 from twisted.web.resource import Resource
 
+from synapse import events
 from synapse.api.constants import EventTypes, Membership
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.ratelimiting import FederationRateLimitConfig
@@ -229,6 +230,11 @@ def setUp(self):
         self._hs_args = {"clock": self.clock, "reactor": self.reactor}
         self.hs = self.make_homeserver(self.reactor, self.clock)
 
+        # Honour the `use_frozen_dicts` config option. We have to do this
+        # manually because this is taken care of in the app `start` code, which
+        # we don't run. Plus we want to reset it on tearDown.
+        events.USE_FROZEN_DICTS = self.hs.config.use_frozen_dicts
+
         if self.hs is None:
             raise Exception("No homeserver returned from make_homeserver.")
 
@@ -292,6 +298,10 @@ async def get_user_by_req(request, allow_guest=False, rights="access"):
         if hasattr(self, "prepare"):
             self.prepare(self.reactor, self.clock, self.hs)
 
+    def tearDown(self):
+        # Reset to not use frozen dicts.
+        events.USE_FROZEN_DICTS = False
+
     def wait_on_thread(self, deferred, timeout=10):
         """
         Wait until a Deferred is done, where it's waiting on a real thread.