diff --git a/matrix_sdk_crypto/src/verification/requests.rs b/matrix_sdk_crypto/src/verification/requests.rs
index 2c78604c..087728b6 100644
--- a/matrix_sdk_crypto/src/verification/requests.rs
+++ b/matrix_sdk_crypto/src/verification/requests.rs
@@ -579,10 +579,12 @@ impl VerificationRequest {
             let mut inner = self.inner.lock().unwrap();
             inner.cancel(false, content.cancel_code());
 
-            if let Some(request) =
-                self.cancel_for_other_devices(content.cancel_code().to_owned(), None)
-            {
-                self.verification_cache.add_verification_request(request.into());
+            if self.we_started() {
+                if let Some(request) =
+                    self.cancel_for_other_devices(content.cancel_code().to_owned(), None)
+                {
+                    self.verification_cache.add_verification_request(request.into());
+                }
             }
         }
     }