diff --git a/ee/packages/federation-matrix/src/api/_matrix/invite.ts b/ee/packages/federation-matrix/src/api/_matrix/invite.ts
index e86c6ecfa4e36..f61ace906b70b 100644
--- a/ee/packages/federation-matrix/src/api/_matrix/invite.ts
+++ b/ee/packages/federation-matrix/src/api/_matrix/invite.ts
@@ -328,14 +328,17 @@ export const getMatrixInviteRoutes = (services: HomeserverServices) => {
 
 			const inviteEvent = await invite.processInvite(event, roomId, eventId, roomVersion);
 
-			setTimeout(() => {
-				void startJoiningRoom({
-					inviteEvent,
-					user: ourUser,
-					room,
-					state,
-				});
-			}, 200);
+			setTimeout(
+				() => {
+					void startJoiningRoom({
+						inviteEvent,
+						user: ourUser,
+						room,
+						state,
+					});
+				},
+				inviteEvent.event.content.is_direct ? 2000 : 0,
+			);
 
 			return {
 				body: {