Sync are-we-synapse with dendrite

tests/sytest/are-we-synapse-yet.list

@@ -91,14 +91,15 @@ snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
 get GET /rooms/:room_id/messages returns a message
 get GET /rooms/:room_id/messages lazy loads members correctly 
 typ PUT /rooms/:room_id/typing/:user_id sets typing notification 
+typ Typing notifications don't leak (3 subtests)
 rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels
 rst PUT /rooms/:room_id/state/m.room.power_levels can set levels
 rst PUT power_levels should not explode if the old power levels were empty
 rst Both GET and PUT work
 rct POST /rooms/:room_id/receipt can create receipts 
 red POST /rooms/:room_id/read_markers can create read marker
-med POST /media/v1/upload can create an upload
+med POST /media/r0/upload can create an upload 
-med GET /media/v1/download can fetch the value again
+med GET /media/r0/download can fetch the value again 
 cap GET /capabilities is present and well formed for registered user 
 cap GET /r0/capabilities is not public
 reg Register with a recaptcha 
@@ -296,7 +297,7 @@ fgt Forgotten room messages cannot be paginated
 fgt Forgetting room does not show up in v2 /sync 
 fgt Can forget room you've been kicked from 
 fgt Can't forget room you're still in
-mem Can re-join room if re-invited
+fgt Can re-join room if re-invited
 ath Only original members of the room can see messages from erased users 
 mem /joined_rooms returns only joined rooms
 mem /joined_members return joined members 
@@ -455,6 +456,19 @@ rmv User can invite remote user to room with version 5
 rmv Remote user can backfill in a room with version 5 
 rmv Can reject invites over federation for rooms with version 5 
 rmv Can receive redactions from regular users over federation in room version 5 
+rmv User can create and send/receive messages in a room with version 6 
+rmv User can create and send/receive messages in a room with version 6 (2 subtests)
+rmv local user can join room with version 6 
+rmv User can invite local user to room with version 6 
+rmv remote user can join room with version 6 
+rmv User can invite remote user to room with version 6 
+rmv Remote user can backfill in a room with version 6 
+rmv Can reject invites over federation for rooms with version 6 
+rmv Can receive redactions from regular users over federation in room version 6 
+rmv Inbound federation rejects invites which include invalid JSON for room version 6
+rmv Outbound federation rejects invite response which include invalid JSON for room version 6
+rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6
+rmv Server rejects invalid JSON in a version 6 room
 pre Presence changes are reported to local room members 
 f,pre Presence changes are also reported to remote room members 
 pre Presence changes to UNAVAILABLE are reported to local room members 
@@ -482,24 +496,24 @@ f,dvk If remote user leaves room, changes device and rejoins we see update in /k
 dvk Get left notifs in sync and /keys/changes when other user leaves 
 dvk Get left notifs for other users in sync and /keys/changes when user leaves 
 f,dvk If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes 
-dvk Can create backup version
+dkb Can create backup version 
-dvk Can update backup version
+dkb Can update backup version 
-dvk Responds correctly when backup is empty
+dkb Responds correctly when backup is empty 
-dvk Can backup keys
+dkb Can backup keys 
-dvk Can update keys with better versions
+dkb Can update keys with better versions 
-dvk Will not update keys with worse versions
+dkb Will not update keys with worse versions 
-dvk Will not back up to an old backup version
+dkb Will not back up to an old backup version 
-dvk Can delete backup
+dkb Can delete backup 
-dvk Deleted & recreated backups are empty
+dkb Deleted & recreated backups are empty 
-dvk Can create more than 10 backup versions
+dkb Can create more than 10 backup versions 
-dvk Can upload self-signing keys
+xsk Can upload self-signing keys 
-dvk Fails to upload self-signing keys with no auth
+xsk Fails to upload self-signing keys with no auth 
-dvk Fails to upload self-signing key without master key
+xsk Fails to upload self-signing key without master key 
-dvk Changing master key notifies local users
+xsk Changing master key notifies local users 
-dvk Changing user-signing key notifies local users
+xsk Changing user-signing key notifies local users 
-f,dvk can fetch self-signing keys over federation
+f,xsk can fetch self-signing keys over federation 
-f,dvk uploading self-signing key notifies over federation
+f,xsk uploading self-signing key notifies over federation 
-f,dvk uploading signed devices gets propagated over federation
+f,xsk uploading signed devices gets propagated over federation 
 tag Can add tag
 tag Can remove tag
 tag Can list tags for a room
@@ -536,11 +550,11 @@ std Can recv device messages until they are acknowledged
 std Device messages with the same txn_id are deduplicated 
 std Device messages wake up /sync 
 std Can recv device messages over federation 
-std Device messages over federation wake up /sync
+fsd Device messages over federation wake up /sync 
 std Can send messages with a wildcard device id 
 std Can send messages with a wildcard device id to two devices 
 std Wildcard device messages wake up /sync 
-std Wildcard device messages over federation wake up /sync
+fsd Wildcard device messages over federation wake up /sync 
 adm /whois 
 nsp /purge_history 
 nsp /purge_history by ts 
@@ -578,6 +592,7 @@ frv A pair of servers can establish a join in a v2 room
 fsj Outbound federation rejects send_join responses with no m.room.create event 
 frv Outbound federation rejects m.room.create events with an unknown room version 
 fsj Event with an invalid signature in the send_join response should not cause room join to fail 
+fsj Inbound: send_join rejects invalid JSON for room version 6
 fed Outbound federation can send events
 fed Inbound federation can receive events 
 fed Inbound federation can receive redacted events 
@@ -636,6 +651,7 @@ fst Name/topic keys are correct
 fau Remote servers cannot set power levels in rooms without existing powerlevels 
 fau Remote servers should reject attempts by non-creators to set the power levels 
 fau Inbound federation rejects typing notifications from wrong remote 
+fau Users cannot set notifications powerlevel higher than their own
 fed Forward extremities remain so even after the next events are populated as outliers 
 fau Banned servers cannot send events 
 fau Banned servers cannot /make_join 
@@ -815,6 +831,7 @@ psh Trying to get push rules with unknown scope fails with 400
 psh Trying to get push rules with unknown template fails with 400 
 psh Trying to get push rules with unknown attribute fails with 400 
 psh Trying to get push rules with unknown rule_id fails with 404
+psh Rooms with names are correctly named in pushes
 v1s GET /initialSync with non-numeric 'limit' 
 v1s GET /events with non-numeric 'limit' 
 v1s GET /events with negative 'limit' 
@@ -823,7 +840,7 @@ ath Event size limits
 syn Check creating invalid filters returns 4xx 
 f,pre New federated private chats get full presence information (SYN-115) 
 pre Left room members do not cause problems for presence 
-crm Rooms can be created with an initial invite list (SYN-205)
+crm Rooms can be created with an initial invite list (SYN-205) (1 subtests)
 typ Typing notifications don't leak 
 ban Non-present room members cannot ban others 
 psh Getting push rules doesn't corrupt the cache SYN-390 
@@ -834,3 +851,16 @@ gst Real user can call /events on another world_readable room (SYN-606)
 gst Events come down the correct room
 pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list
 std Can send a to-device message to two users which both receive it using /sync
+fme Outbound federation will ignore a missing event with bad JSON for room version 6
+fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6
+jso Invalid JSON integers
+jso Invalid JSON floats
+jso Invalid JSON special values
+inv Can invite users to invite-only rooms (2 subtests)
+plv setting 'm.room.name' respects room powerlevel (2 subtests)
+psh Messages that notify from another user increment notification_count
+psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count
+dvk Can claim one time key using POST (2 subtests)
+fdk Can query remote device keys using POST (1 subtests)
+fdk Can claim remote one time key using POST (2 subtests)
+fmj Inbound /make_join rejects attempts to join rooms where all users have left

tests/sytest/are-we-synapse-yet.py

@@ -33,6 +33,8 @@ import sys
 
 test_mappings = {
     "nsp": "Non-Spec API",
+    "unk": "Unknown API (no group specified)",
+    "app": "Application Services API",
     "f": "Federation", # flag to mark test involves federation
 
     "federation_apis": {
@@ -50,6 +52,7 @@ test_mappings = {
         "fpb": "Public Room API",
         "fdk": "Device Key APIs",
         "fed": "Federation API",
+		"fsd": "Send-to-Device APIs",
     },
 
     "client_apis": {
@@ -61,6 +64,8 @@ test_mappings = {
         "pro": "Profile",
         "dev": "Devices",
         "dvk": "Device Keys",
+        "dkb": "Device Key Backup",
+        "xsk": "Cross-signing Keys",
         "pre": "Presence",
         "crm": "Create Room",
         "syn": "Sync API",
@@ -98,7 +103,7 @@ test_mappings = {
         "adm": "Server Admin API",
         "ign": "Ignore Users",
         "udr": "User Directory APIs",
-        "app": "Application Services API",
+		"jso": "Enforced canonical JSON",
     },
 }
 
@@ -156,6 +161,8 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     total_tests = 0
     for gid, tests in gid_to_tests.items():
         group_total = len(tests)
+        if group_total == 0:
+            continue
         group_passing = 0
         test_names_and_marks = []
         for name, passing in tests.items():
@@ -186,7 +193,6 @@ def main(results_tap_path, verbose):
     test_name_to_group_id = {}
     fed_tests = set()
     client_tests = set()
-    groupless_tests = set()
     with open("./are-we-synapse-yet.list", "r") as f:
         for line in f.readlines():
             test_name = " ".join(line.split(" ")[1:]).strip()
@@ -212,8 +218,12 @@ def main(results_tap_path, verbose):
             #   test_name: OK
             # }
         },
+        "appservice": {
+            "app": {},
+        },
         "nonspec": {
-            "nsp": {}
+            "nsp": {},
+            "unk": {}
         },
     }
     with open(results_tap_path, "r") as f:
@@ -224,10 +234,11 @@ def main(results_tap_path, verbose):
             name = test_result["name"]
             group_id = test_name_to_group_id.get(name)
             if not group_id:
-                groupless_tests.add(name)
+                summary["nonspec"]["unk"][name] = test_result["ok"]
-                # raise Exception("The test '%s' doesn't have a group" % (name,))
             if group_id == "nsp":
                 summary["nonspec"]["nsp"][name] = test_result["ok"]
+            elif group_id == "app":
+                summary["appservice"]["app"][name] = test_result["ok"]
             elif group_id in test_mappings["federation_apis"]:
                 group = summary["federation"].get(group_id, {})
                 group[name] = test_result["ok"]
@@ -243,12 +254,7 @@ def main(results_tap_path, verbose):
     print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose)
     print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose)
     print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose)
-    if verbose:
+    print_stats("Application Services APIs", summary["appservice"], test_mappings, verbose)
-        print("The following tests don't have a group:")
-        for name in groupless_tests:
-            print("  %s" % (name,))
-    else:
-        print("%d tests don't have a group" % len(groupless_tests))