Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks.

changelog.d/19826.bugfix

@@ -0,0 +1 @@
+Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks.

synapse/storage/databases/main/sliding_sync.py

@@ -204,6 +204,32 @@ def persist_per_connection_state_txn(
                 raise SlidingSyncUnknownPosition()
 
             (connection_key,) = row
+
+            if isinstance(self.database_engine, PostgresEngine):
+                # Lock the sliding sync connection row for update upfront,
+                # to prevent deadlocks between concurrent transactions
+                # (which can retry again and again without making progress).
+                #
+                # (We don't need to explicitly lock in the other branch,
+                # where we re-create the connection, as that implies a lock
+                # anyway)
+                #
+                # Specifically, the statements seen to deadlock against
+                # each other were
+                # `INSERT INTO sliding_sync_connection_lazy_members`
+                # with conflicting tuples on
+                #     "sliding_sync_connection_lazy_members_idx" UNIQUE, btree
+                #     (connection_key, room_id, user_id)
+                # https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS
+                txn.execute(
+                    """
+                    SELECT 1
+                    FROM sliding_sync_connections
+                    WHERE connection_key = ?
+                    FOR NO KEY UPDATE
+                    """,
+                    (connection_key,),
+                )
         else:
             # We're restarting the connection, so we clear the previous existing data we
             # used to track it. We do this here to ensure that if we get lots of