diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go
index 4d10dea6..88eb7e1e 100644
--- a/roomserver/internal/perform/perform_leave.go
+++ b/roomserver/internal/perform/perform_leave.go
@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 )
 
 type Leaver struct {
@@ -171,7 +172,9 @@ func (r *Leaver) performFederatedRejectInvite(
 	}
 	leaveRes := fsAPI.PerformLeaveResponse{}
 	if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil {
-		return nil, err
+		// failures in PerformLeave should NEVER stop us from telling other components like the
+		// sync API that the invite was withdrawn. Otherwise we can end up with stuck invites.
+		util.GetLogger(ctx).WithError(err).Errorf("failed to PerformLeave, still retiring invite event")
 	}
 
 	// Withdraw the invite, so that the sync API etc are
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 10a0dda8..70374c6a 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -2,8 +2,13 @@ package streams
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"strconv"
+	"time"
 
 	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 type InviteStreamProvider struct {
@@ -56,6 +61,17 @@ func (p *InviteStreamProvider) IncrementalSync(
 	for roomID := range retiredInvites {
 		if _, ok := req.Response.Rooms.Join[roomID]; !ok {
 			lr := types.NewLeaveResponse()
+			h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
+			lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
+				// fake event ID which muxes in the to position
+				EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
+				OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+				RoomID:         roomID,
+				Sender:         req.Device.UserID,
+				StateKey:       &req.Device.UserID,
+				Type:           "m.room.member",
+				Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
+			})
 			req.Response.Rooms.Leave[roomID] = *lr
 		}
 	}
diff --git a/sytest-whitelist b/sytest-whitelist
index f6210357..f72cf333 100644
--- a/sytest-whitelist
+++ b/sytest-whitelist
@@ -527,3 +527,6 @@ POST /_synapse/admin/v1/register with shared secret disallows symbols
 Membership event with an invalid displayname in the send_join response should not cause room join to fail
 Inbound federation rejects incorrectly-signed invite rejections
 Inbound federation can receive invite rejections
+Inbound federation can receive invite and reject when remote replies with a 403
+Inbound federation can receive invite and reject when remote replies with a 500
+Inbound federation can receive invite and reject when remote is unreachable