diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go
index 57612908..12a04d4b 100644
--- a/federationsender/queue/destinationqueue.go
+++ b/federationsender/queue/destinationqueue.go
@@ -239,7 +239,7 @@ func (oq *destinationQueue) backgroundSend() {
 			log.Warnf("Blacklisting %q due to exceeding backoff threshold", oq.destination)
 			return
 		}
-		if until != nil {
+		if until != nil && until.After(time.Now()) {
 			// We haven't backed off yet, so wait for the suggested amount of
 			// time.
 			duration := time.Until(*until)