diff --git a/matrix_sdk_base/src/client.rs b/matrix_sdk_base/src/client.rs
index 89abdbb8..06e6ade0 100644
--- a/matrix_sdk_base/src/client.rs
+++ b/matrix_sdk_base/src/client.rs
@@ -990,21 +990,6 @@ impl BaseClient {
                 self.get_or_create_joined_room(&room_id).await?.clone()
             };
 
-            #[cfg(feature = "encryption")]
-            {
-                let mut olm = self.olm.lock().await;
-
-                if let Some(o) = &mut *olm {
-                    let room = matrix_room.read().await;
-
-                    // If the room is encrypted, update the tracked users.
-                    if room.is_encrypted() {
-                        o.update_tracked_users(room.joined_members.keys()).await;
-                        o.update_tracked_users(room.invited_members.keys()).await;
-                    }
-                }
-            }
-
             // RoomSummary contains information for calculating room name.
             matrix_room
                 .write()
@@ -1042,6 +1027,21 @@ impl BaseClient {
                 }
             }
 
+            #[cfg(feature = "encryption")]
+            {
+                let mut olm = self.olm.lock().await;
+
+                if let Some(o) = &mut *olm {
+                    let room = matrix_room.read().await;
+
+                    // If the room is encrypted, update the tracked users.
+                    if room.is_encrypted() {
+                        o.update_tracked_users(room.joined_members.keys()).await;
+                        o.update_tracked_users(room.invited_members.keys()).await;
+                    }
+                }
+            }
+
             // look at AccountData to further cut down users by collecting ignored users
             if let Some(account_data) = &joined_room.account_data {
                 for account_data in &account_data.events {