Commit cbb715e11963
tarball32 files changed+6595-193
@@ -1839,6 +1839,7 @@ dependencies = [
18391839
"mime",18401840
"reqwest",18411841
"ruma",1842
+ "rusqlite",18421843
"serde",18431844
"serde_json",18441845
"sha2",@@ -1849,6 +1850,7 @@ dependencies = [
18491850
"uuid",18501851
"wasm-bindgen",18511852
"wasm-bindgen-futures",1853
+ "wasm-bindgen-test",18521854
"web-sys",18531855
"wiremock",18541856
]@@ -2000,7 +2002,7 @@ dependencies = [
20002002
[[package]]20012003
name = "matrix-sdk"20022004
version = "0.18.0"2003
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2005
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"20042006
dependencies = [20052007
"anymap2",20062008
"aquamarine",@@ -2063,7 +2065,7 @@ dependencies = [
20632065
[[package]]20642066
name = "matrix-sdk-base"20652067
version = "0.18.0"2066
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2068
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"20672069
dependencies = [20682070
"as_variant",20692071
"assert_matches",@@ -2093,7 +2095,7 @@ dependencies = [
20932095
[[package]]20942096
name = "matrix-sdk-common"20952097
version = "0.18.0"2096
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2098
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"20972099
dependencies = [20982100
"eyeball-im",20992101
"futures-core",@@ -2116,7 +2118,7 @@ dependencies = [
21162118
[[package]]21172119
name = "matrix-sdk-crypto"21182120
version = "0.18.0"2119
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2121
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"21202122
dependencies = [21212123
"aes",21222124
"aquamarine",@@ -2157,7 +2159,7 @@ dependencies = [
21572159
[[package]]21582160
name = "matrix-sdk-indexeddb"21592161
version = "0.18.0"2160
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2162
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"21612163
dependencies = [21622164
"async-trait",21632165
"base64",@@ -2189,7 +2191,7 @@ dependencies = [
21892191
[[package]]21902192
name = "matrix-sdk-sqlite"21912193
version = "0.18.0"2192
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2194
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"21932195
dependencies = [21942196
"as_variant",21952197
"async-trait",@@ -2216,7 +2218,7 @@ dependencies = [
22162218
[[package]]22172219
name = "matrix-sdk-store-encryption"22182220
version = "0.18.0"2219
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2221
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"22202222
dependencies = [22212223
"base64",22222224
"blake3",@@ -2237,7 +2239,7 @@ dependencies = [
22372239
[[package]]22382240
name = "matrix-sdk-test"22392241
version = "0.18.0"2240
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2242
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"22412243
dependencies = [22422244
"as_variant",22432245
"ctor",@@ -2260,7 +2262,7 @@ dependencies = [
22602262
[[package]]22612263
name = "matrix-sdk-test-macros"22622264
version = "0.18.0"2263
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2265
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"22642266
dependencies = [22652267
"quote",22662268
"syn 2.0.119",@@ -2269,7 +2271,7 @@ dependencies = [
22692271
[[package]]22702272
name = "matrix-sdk-ui"22712273
version = "0.18.0"2272
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2274
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"22732275
dependencies = [22742276
"as_variant",22752277
"async-rx",@@ -8,7 +8,7 @@ description = "A compact application backend on top of the Matrix Rust SDK"
88
default = []99
mirror = []1010
serde = ["dep:serde", "imbl/serde"]11
-sqlite = ["matrix-sdk/sqlite", "serde", "dep:url", "dep:matrix-sdk-store-encryption"]11
+sqlite = ["matrix-sdk/sqlite", "serde", "dep:url", "dep:matrix-sdk-store-encryption", "dep:rusqlite"]1212
bundled-sqlite = ["sqlite", "matrix-sdk/bundled-sqlite"]1313
indexeddb = [1414
"matrix-sdk/indexeddb",@@ -26,11 +26,11 @@ indexeddb = [
2626
futures-util = { version = "0.3.31", default-features = false, features = ["std"] }2727
http = "1"2828
imbl = { version = "6.1.0", default-features = false }29
-matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["e2e-encryption"] }30
-matrix-sdk-base = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false }31
-matrix-sdk-crypto = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false }32
-matrix-sdk-store-encryption = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", optional = true }33
-matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false }29
+matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["e2e-encryption"] }30
+matrix-sdk-base = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false }31
+matrix-sdk-crypto = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false }32
+matrix-sdk-store-encryption = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", optional = true }33
+matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false }3434
ruma = { version = "0.16.0", default-features = false, features = ["unstable-msc2545", "unstable-msc4095"] }3535
serde = { version = "1.0.219", features = ["derive", "rc"], optional = true }3636
serde_json = "1.0.143"@@ -40,6 +40,7 @@ sha2 = "0.10.9"
4040
tokio = { version = "1.47.1", features = ["macros", "sync"] }4141
tracing = "0.1.41"4242
url = { version = "2.5.7", optional = true }43
+rusqlite = { version = "0.37", optional = true }4344
4445
[target.'cfg(target_family = "wasm")'.dependencies]4546
getrandom = { version = "0.2", features = ["js"] }@@ -47,8 +48,8 @@ getrandom3 = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
4748
getrandom4 = { package = "getrandom", version = "0.4", features = ["wasm_js"] }4849
indexed_db_futures = { package = "matrix_indexed_db_futures", version = "0.7.0", optional = true, features = ["typed-arrays"] }4950
js-sys = { version = "0.3.105", optional = true }50
-matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["js"] }51
-matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["js"] }51
+matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["js"] }52
+matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["js"] }5253
# the sdk's own wasm reqwest leaves out stream, which the bounded media download needs5354
reqwest = { version = "0.13.1", default-features = false, features = ["stream"] }5455
uuid = { version = "1", features = ["js"] }@@ -62,12 +63,16 @@ web-sys = { version = "0.3.105", optional = true, features = [
6263
"WorkerGlobalScope",6364
] }6465
65
-[dev-dependencies]66
-matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["testing"] }67
-matrix-sdk-test = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767" }66
+[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]67
+matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["testing"] }68
+matrix-sdk-test = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48" }6869
tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread", "signal", "time"] }6970
wiremock = "0.6.5"7071
72
+[target.'cfg(target_family = "wasm")'.dev-dependencies]73
+tokio = { version = "1.47.1", features = ["macros", "rt"] }74
+wasm-bindgen-test = "0.3"75
+7176
[[example]]7277
name = "headless"7378
required-features = ["sqlite"]@@ -0,0 +1,341 @@
1
+# Persistent mentions discovery2
+3
+This records the SDK audit and current persistent mentions implementation.4
+Persistent sessions now commit discovery with Sliding Sync responses, retain an5
+encrypted inbox, and expose it through `ClientHandle` and the mirror.6
+Historical coverage remains incomplete because room-list sync and notification7
+history cannot prove that every past mention was observed.8
+9
+The audited dependency is Matrix Rust SDK 0.18.0 at fork revision10
+`cc2b13ea185c0f694523a3a48d8aadfe366dd767`. macltok now pins fork revision11
+`7aace883fbe03d4c3c118b55188fe9519ba8ce48`, which adds the response commit12
+hook described below. SDK paths below are relative to the audited revision.13
+14
+## Findings15
+16
+The SDK can deliver and decrypt events without opening a UI timeline. macltok17
+records observed events and recovers candidates through notification history,18
+but neither source establishes complete historical coverage.19
+20
+The SDK audit found that an awaited event handler can hold back the Sliding21
+Sync position until a macltok transaction commits, while a background22
+broadcast consumer cannot provide that guarantee. Handlers cannot propagate23
+commit errors, and response processing outlives cancellation of the calling24
+sync task. These limits motivated the fallible response-commit hook now25
+implemented in the pinned SDK fork.26
+27
+## Sync coverage28
+29
+macltok constructs `SyncService` in `src/engine/mod.rs` and uses the SDK room30
+list configuration. The list starts with the selective range `0..=19`. After31
+setup it grows by 100 rooms per response, without an SDK-side maximum, until32
+it reaches the server's room count. Recovery and sufficiently long idle33
+periods can return it to the initial selective range before it grows again.34
+The default list timeline limit is one event per room on initial delivery.35
+Subsequent changes for covered rooms continue arriving through sync.36
+37
+Opening a room separately subscribes it with a38
+timeline limit of 20. macltok's `update_room_subscriptions` and39
+`watchers::room_subscriptions` manage those explicit subscriptions for active40
+room actors only. The mentions response hook filters entries to joined rooms.41
+The built-in list still includes invited rooms.42
+43
+Sources are `crates/matrix-sdk-ui/src/room_list_service/mod.rs`,44
+`crates/matrix-sdk-ui/src/room_list_service/state.rs`, and45
+`crates/matrix-sdk/src/sliding_sync/list/request_generator.rs`.46
+47
+The default one-event initial window cannot recover all earlier pings, including48
+messages sent while the client was offline. Raising49
+`SyncServiceBuilder::with_room_list_timeline_limit` increases the initial50
+window but does not establish continuous history. The response hook processes51
+joined-room events and records limited timelines, initial room coverage,52
+checkpoint discontinuities, and recovery gaps.53
+54
+## Observation and durable commit55
+56
+`Client::subscribe_to_all_room_updates` returns a bounded broadcast receiver57
+of `RoomUpdates`. Its `joined` map includes processed timeline events with raw58
+decrypted content when available and optional evaluated push actions. It does59
+not require a UI timeline. Lag is an error that requires recovery, not an60
+invitation to skip to the next update.61
+62
+`Client::observe_events` retains only the latest observation and can miss63
+intermediate events. It is unsuitable for durable discovery.64
+65
+An earlier probe registers `Client::add_event_handler` for66
+`Raw<AnySyncTimelineEvent>` with `Room` context before sync starts. Its awaited67
+handler demonstrates event ordering, but it cannot report commit failures68
+reliably and is not the production ingestion path. Production uses the69
+fallible Sliding Sync response hook described below.70
+71
+For Sliding Sync v5, the response processor awaits event handlers before72
+updating and persisting `pos`. The response hook commits the inbox first, so73
+a crash permits replay without skipping committed candidates. Stable event74
+keys make replay idempotent. The hook acknowledgement does not wait for75
+context capture, a UI subscriber, or the engine event stream to drain.76
+77
+For ordinary event handlers, the SDK logs and discards handler errors, so78
+returning an error after a failed write still permits `pos` to advance. A79
+pending handler alone also does not provide a reliable shutdown path. The SDK80
+runs response handling in a detached task that outlives cancellation of81
+`sync_once` or its caller. Aborting the engine's sync task cannot prevent a82
+suspended handler from later returning and advancing the position. Registering83
+a handler and placing work on a queue without awaiting its commit does not84
+close the crash window.85
+86
+macltok uses shared `pos` through its SDK service and commits each response87
+batch through one serialized mentions actor before the hook acknowledges it.88
+The actor records joined-room events, raw encrypted candidates, limited89
+timeline gaps, and the connection checkpoint in the same inbox revision.90
+Stable room and event keys make replay idempotent. Context capture, mirror91
+publication, and event-stream delivery continue after the commit. The classic92
+sync `next_batch` ordering differs and does not establish this guarantee.93
+Nor does this ordering make arbitrary concurrent sync loops safe.94
+95
+Sources are `crates/matrix-sdk/src/event_handler/mod.rs`,96
+`crates/matrix-sdk/src/event_handler/static_events.rs`,97
+`crates/matrix-sdk/src/sliding_sync/client.rs`,98
+`crates/matrix-sdk/src/sliding_sync/mod.rs`, and99
+`crates/matrix-sdk/src/sliding_sync/cache.rs`.100
+101
+### SDK commit hook implementation102
+103
+The local SDK workspace at `../matrix-rust-sdk-mentions` implements the104
+fallible asynchronous response-commit hook through `ResponseCommitHook` and105
+`ResponseCommitBatch`. `SyncServiceBuilder` forwards it only to its room-list106
+connection, so independent notification lookups are unaffected.107
+108
+The hook receives the processed room events, original room payloads, limited109
+flags, pagination tokens, connection identity, and previous and next position110
+identifiers. It runs before response handlers and observers and before `pos`111
+is updated or saved. SDK room state may already have been saved. A successful112
+acknowledgement follows the macltok inbox transaction. A failed or dropped113
+acknowledgement returns a sync error without advancing the position.114
+115
+The hook runs outside SDK store locks and supports local futures on wasm and116
+sendable futures natively. The default absent hook preserves existing SDK117
+behavior. Detached response processing observes a dropped store118
+acknowledgement after its caller is cancelled. Checkpoint replay works after119
+SDK room state was saved, and stable event keys deduplicate repeated commits.120
+121
+The SDK audit and focused validation covered commit ordering, failed and122
+dropped acknowledgements, caller cancellation, replay after room-state123
+persistence, restoring a previous checkpoint, absent-hook behavior, builder124
+forwarding, and connection isolation. The SDK UI library also received a125
+focused wasm compilation check with its `js` feature. The SDK changes are126
+published to the user's fork under `mentions-commit-hook`, and the manifest127
+and both macltok lockfiles pin revision128
+`7aace883fbe03d4c3c118b55188fe9519ba8ce48`.129
+130
+## Event cache and delayed decryption131
+132
+`EventCache::subscribe` starts global room-update ingestion. The SDK room list133
+service already calls it during construction. Its generic update subscription134
+identifies the affected room, then `Room::event_cache` and135
+`RoomEventCache::find_event` can retrieve an event. This does not require a136
+`matrix-sdk-ui::Timeline` or a retained subscriber per room.137
+138
+`EventCache::subscribe_to_decryption_reports` reports resolved event IDs,139
+lag, and backup availability. `EventCache::request_decryption` accepts room140
+and Megolm session IDs for explicit retries. `Room::decrypt_event` is also141
+public and can decrypt a retained raw event independently of a UI timeline.142
+143
+The late-key probe confirms automatic decryption and retrieval of a structured144
+mention through the global cache without a UI timeline. Cached push actions145
+are optional. Re-evaluate them with `Room::event_push_actions` when needed,146
+and treat missing push context as unresolved rather than a negative result.147
+The room's own membership state is needed to build that push context.148
+149
+The cache is not a durable ingestion journal. Its room broadcasts and generic150
+updates can lag, and cache retention can remove events. SQLite and IndexedDB151
+builders select persistent SDK event-cache stores, while the default152
+store-free client uses memory. Even with persistence, the asynchronous cache153
+consumer is not awaited before Sliding Sync saves its position.154
+155
+The inbox retains unresolved encrypted candidates as encrypted raw events156
+within a configurable bound. It retries due candidates at startup and resets157
+their retry time when decryption reports arrive. Candidate eviction sets the158
+coverage truncation flag. Imported keys and backups cannot reconstruct events159
+already discarded by every cache.160
+161
+Sources are `crates/matrix-sdk/src/event_cache/mod.rs`,162
+`crates/matrix-sdk/src/event_cache/redecryptor.rs`,163
+`crates/matrix-sdk/src/event_cache/caches/room/mod.rs`, and164
+`crates/matrix-sdk/src/room/mod.rs`.165
+166
+## Mention policy167
+168
+Structured `m.mentions.user_ids` and `m.mentions.room` supply semantic169
+candidates. The classifier does not infer pings from rendered bodies or170
+unread counts. A reply relationship alone does not prove that a reply notifies171
+the user, so reply reasons require notification metadata and confirmation that172
+the referenced event belongs to the signed-in user.173
+174
+`Room::event_push_actions` evaluates push rules but does not independently175
+filter ignored senders. The classifier calls `Client::is_user_ignored` and176
+rechecks entries when the ignore list changes.177
+178
+A room mute can suppress evaluated notification actions while structured179
+mention metadata remains present. The classifier retains explicit user180
+mentions in muted rooms and marks notification delivery as suppressed. Reply181
+reasons require both a reply to the user and a notifying highlight. Room-wide182
+reasons require the relevant push conditions and sender power level. Missing183
+authorization context remains unresolved for reevaluation. Keyword-only184
+notifications are outside the inbox. The classifier excludes the signed-in185
+user's own events even when they contain explicit mention metadata.186
+187
+The existing `NotificationClient` combines notification evaluation with188
+ignored-sender filtering. Its public lookup takes known room and event IDs.189
+It is useful for resolving a candidate, but filtered results erase the190
+distinction between mute policy and an absent semantic mention. It is not the191
+sole classifier for this inbox.192
+193
+## Historical candidates and gaps194
+195
+Ruma exposes196
+`api::client::push::get_notifications::v3::Request`, sent with `Client::send`.197
+It accepts opaque `from`, optional `limit`, and optional `only` fields.198
+Responses contain notifications and an optional `next_token`. macltok requests199
+50 notifications per page. Entries include raw events, room IDs, timestamps,200
+actions, and server notification read state, which does not initialize or201
+replace local inbox read state.202
+203
+The service uses this endpoint as a bounded candidate source, validates204
+events after decryption, and applies the live mention policy. Each page's205
+observed candidates and cursor are committed together in the encrypted inbox,206
+including unresolved encrypted candidates. A failed commit leaves the prior207
+cursor available for replay.208
+209
+An exhausted notification list does not mean complete room history. Muted-room210
+pings, encrypted mentions not recognized by the server, expired notifications,211
+and server-specific retention can be absent. Recovery inspects at most 20 pages212
+of 50 notifications per run and resumes from committed cursors later. The213
+older-history cursor is separate from the cursor for scans from the newest214
+notifications. A newest scan records an anchor and runs at startup and every215
+60 seconds. It follows the committed cursor until it reaches that anchor,216
+resuming on later runs when needed to find newly added notifications.217
+218
+SDK cache gaps retain pagination tokens and support room pagination, but219
+running that pagination mutates the live cache. Mention context capture uses220
+independent `/context` and `/messages` requests. Tokens may expire. Context can221
+be retried around a known event while the room remains joined, and known gaps222
+remain visible when continuity cannot be established.223
+224
+Coverage exposes the earliest observed timestamp, history-endpoint support,225
+recovery activity, known room gaps, unresolved candidate count, and retention226
+truncation. The earliest timestamp is a summary, not proof that every room is227
+covered after that time. The inbox keeps its incomplete-history flag even228
+after notification pagination ends.229
+230
+Complete offline mention recovery across arbitrary muted and encrypted rooms231
+is not supplied by this endpoint or by the current sync configuration. It232
+would require a retained, replayable event source or bounded per-room history233
+recovery with proof of continuity. The notification endpoint does not provide234
+that guarantee.235
+236
+## Storage, context, and API237
+238
+`src/mentions/store` implements encrypted SQLite and IndexedDB record stores.239
+Persistent session construction opens the account store, and local logout240
+deletes it alongside the SDK stores. The mentions actor owns the store for241
+inbox commits and discovery checkpoints.242
+243
+The passphrase wraps a random SDK `StoreCipher`. Logical keys are hashed, and244
+authenticated encrypted records include schema, account identity, and logical245
+key. Restore rejects mismatched accounts, unsupported schemas, corrupt records,246
+and inconsistent record counts or byte totals. Atomic transactions compare the247
+previous encrypted header before publishing a new revision, so stale writers248
+cannot overwrite a newer checkpoint.249
+250
+The default budget is 64 MiB of encrypted records, hashed keys, header, and251
+wrapped cipher, excluding database page overhead. The store record cap is252
+35,128. The inbox defaults to 5,000 entries, 25,000 shared context events, and253
+5,000 unresolved candidates within that byte budget. When a size or record254
+limit requires eviction, the store drops bounded data and marks coverage255
+truncated.256
+257
+The storage tests cover atomic restore of records and checkpoints, stale258
+writers, capacity failure, encrypted physical records, wrong credentials or259
+account, corruption, unsupported schemas, and injected SQLite write failure.260
+The inbox tests cover batch failure, notification recovery, live context261
+updates, and capture persistence across session resume.262
+263
+264
+The inbox is serialized into one `inbox` value and encrypted as a single265
+record on every revision. Each update reserializes and re-encrypts the retained266
+inbox, so write cost scales with retained data. Decrypted previews, saved267
+context, raw candidates, pending capture state, media source registrations,268
+read state, coverage, and sync checkpoints stay inside the authenticated269
+encrypted payload. SDK SQLite custom values remain plaintext and are not used270
+for decrypted mentions.271
+272
+Context capture in `src/engine/context.rs` sizes the before and after requests273
+independently and retains the media registrations produced by normalization.274
+Each capture requests up to 10 events on each side. At most two capture jobs275
+run concurrently, with up to 128 pending or failed entries considered for276
+scheduling at a time. Capture uses independent `/context` and `/messages`277
+requests, so it does not paginate the live cache. Media source registrations278
+are encrypted with saved events and registered again when context is read.279
+Media bodies and key material do not enter the mentions public model.280
+281
+One account-scoped mentions actor serializes store commits for entries,282
+unresolved candidates, shared context events, read state, revisions, discovery283
+checkpoints, and retention decisions.284
+285
+Pending capture state is durable with the inbox, so interrupted capture can286
+resume after startup. When a room leaves the joined state, saved context287
+remains readable and its capture state becomes unavailable. Refetch is288
+disabled after departure.289
+290
+Store publication follows commit. `ClientHandle` exposes page, context, retry,291
+read, read-through, and recovery methods. The public `Command`, mention models,292
+mirror state, and updates support serde when the `serde` feature is enabled,293
+so native and browser consumers can transport them. Browser worker integration294
+is a consumer phase after SDK and macltok revision publication. Malice's worker295
+still needs mention variants in its explicit `Request` dispatch and TypeScript296
+methods.297
+298
+Mirror snapshots contain at most 50 entry previews, coverage, unread count,299
+and a committed revision boundary, with no saved context bodies. The watcher300
+coalesces updates and computes upserts and removals relative to the previous301
+bounded snapshot it observed. A missed update therefore leaves the latest302
+snapshot authoritative. A removal can mean an entry fell out of the top 50,303
+and does not imply that it was deleted from the inbox. Pagination uses304
+timestamp and stable event key within a discovery revision boundary.305
+Read-through boundaries exclude mentions first discovered or reactivated after306
+that boundary, including older messages resolved by late decryption.307
+308
+## Executable probes309
+310
+`tests/mentions_discovery.rs` exercises unopened-room update and cache access,311
+push-rule behavior for ignored senders and muted rooms, and typed312
+notification-history pagination.313
+314
+`tests/mentions_decryption.rs` exchanges a real encrypted event and a delayed315
+room key between two mock clients, then retrieves the decrypted mention316
+without opening a UI timeline.317
+318
+`tests/mentions_checkpoint.rs` holds an event handler pending and verifies319
+that the SDK has not persisted its position. It cancels the calling task,320
+then acknowledges the handler and verifies that detached response processing321
+still saves the position. A new Sliding Sync instance restores that position.322
+This is the historical event-handler ordering probe. It does not exercise the323
+production response-commit hook or the durable inbox transaction. It also324
+demonstrates that returning a handler error still advances the position.325
+326
+`tests/mentions_inbox.rs` covers live mention classification, read-through327
+with revision boundaries, independent context limits, and persistence across328
+session resume. `tests/mentions_late_decryption.rs` verifies that a delayed room key329
+updates the production inbox once. `src/mentions/service_tests.rs` covers330
+checkpoint rollback, notification-history recovery, and context updates.331
+`src/mentions/store/tests.rs` exercises encrypted record restore, capacity,332
+stale writers, and corruption handling.333
+334
+These probes establish SDK behavior against mocked responses. They do not335
+establish homeserver notification retention, room-list completeness, or336
+complete history on a deployed server.337
+338
+Focused validation passed 63 native tests and 12 shared inbox and IndexedDB339
+tests in headless Firefox. Compilation checks also pass without storage340
+features and for wasm with IndexedDB and mirror support. The full flake gate341
+was skipped by request.@@ -1733,6 +1733,7 @@ dependencies = [
17331733
"reqwest",17341734
"ruma",17351735
"serde",1736
+ "serde_json",17361737
"sha2",17371738
"thiserror 2.0.20",17381739
"tokio",@@ -1906,7 +1907,7 @@ dependencies = [
19061907
[[package]]19071908
name = "matrix-sdk"19081909
version = "0.18.0"1909
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"1910
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"19101911
dependencies = [19111912
"anymap2",19121913
"aquamarine",@@ -1966,7 +1967,7 @@ dependencies = [
19661967
[[package]]19671968
name = "matrix-sdk-base"19681969
version = "0.18.0"1969
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"1970
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"19701971
dependencies = [19711972
"as_variant",19721973
"async-trait",@@ -1992,7 +1993,7 @@ dependencies = [
19921993
[[package]]19931994
name = "matrix-sdk-common"19941995
version = "0.18.0"1995
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"1996
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"19961997
dependencies = [19971998
"eyeball-im",19981999
"futures-core",@@ -2015,7 +2016,7 @@ dependencies = [
20152016
[[package]]20162017
name = "matrix-sdk-crypto"20172018
version = "0.18.0"2018
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2019
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"20192020
dependencies = [20202021
"aes",20212022
"aquamarine",@@ -2055,7 +2056,7 @@ dependencies = [
20552056
[[package]]20562057
name = "matrix-sdk-indexeddb"20572058
version = "0.18.0"2058
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2059
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"20592060
dependencies = [20602061
"async-trait",20612062
"base64",@@ -2087,7 +2088,7 @@ dependencies = [
20872088
[[package]]20882089
name = "matrix-sdk-sqlite"20892090
version = "0.18.0"2090
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2091
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"20912092
dependencies = [20922093
"as_variant",20932094
"async-trait",@@ -2114,7 +2115,7 @@ dependencies = [
21142115
[[package]]21152116
name = "matrix-sdk-store-encryption"21162117
version = "0.18.0"2117
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2118
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"21182119
dependencies = [21192120
"base64",21202121
"blake3",@@ -2135,7 +2136,7 @@ dependencies = [
21352136
[[package]]21362137
name = "matrix-sdk-ui"21372138
version = "0.18.0"2138
-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"2139
+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"21392140
dependencies = [21402141
"as_variant",21412142
"async-rx",@@ -42,6 +42,7 @@ pub struct Session {
4242
4343
/// taken as soon as the client exists, since the sdk can refresh before there's an engine4444
struct Tracking {45
+ store: Arc<dyn SessionStore>,4546
session_changes: broadcast::Receiver<SessionChange>,4647
persistence: Option<Persistence>,4748
#[cfg(not(target_family = "wasm"))]@@ -312,6 +313,17 @@ impl StoreConfig {
312313
&self.passphrase313314
}314315
316
+ #[cfg(feature = "sqlite")]317
+ pub(crate) fn mentions_location(&self) -> PathBuf {318
+ let StoreLocation::Sqlite(path) = &self.location;319
+ path.join("macltok-mentions.sqlite3")320
+ }321
+322
+ #[cfg(feature = "indexeddb")]323
+ pub(crate) fn mentions_location(&self) -> String {324
+ format!("{}::macltok-mentions", self.indexeddb_name())325
+ }326
+315327
#[cfg(feature = "indexeddb")]316328
pub(crate) fn indexeddb_name(&self) -> &str {317329
let StoreLocation::IndexedDb(name) = &self.location;@@ -834,12 +846,28 @@ impl Session {
834846
self,835847
config: EngineConfig,836848
) -> Result<(Engine, ClientHandle, EventStream), AccountError> {849
+ let account = crate::mentions::store::Account {850
+ homeserver: self.client.homeserver().to_string(),851
+ user_id: self852
+ .client853
+ .user_id()854
+ .ok_or(AccountError::MissingSession)?855
+ .to_owned(),856
+ };857
+ let mentions = crate::mentions::store::MentionsStore::open(858
+ self.tracking.store.store(),859
+ account,860
+ config.max_mentions_bytes,861
+ )862
+ .await863
+ .map_err(|error| account_error("opening the mentions store", error))?;837864
Engine::with_persistence(838865
self.client,839866
config,840867
self.tracking.session_changes,841868
self.tracking.persistence,842869
Some(self.store_lock),870
+ Some(mentions),843871
)844872
.await845873
.map_err(AccountError::from)@@ -872,6 +900,7 @@ impl Tracking {
872900
#[cfg(not(target_family = "wasm"))]873901
let inline = save_inline(client, sessions.clone(), results)?;874902
Ok(Self {903
+ store: sessions.clone(),875904
session_changes,876905
persistence: Some(Persistence {877906
save: Box::new(move |client| {@@ -1,20 +1,23 @@
11
use std::sync::Arc;22
3
-use matrix_sdk::{room::MessagesOptions, Room};4
-use matrix_sdk_ui::timeline::{RoomExt, TimelineEventFocusThreadMode, TimelineFocus};5
-use ruma::{uint, OwnedEventId};3
+use matrix_sdk::{4
+ Room, deserialized_responses::TimelineEvent as SdkTimelineEvent, room::MessagesOptions,5
+};6
+use ruma::{OwnedEventId, OwnedUserId, RoomId, UInt};67
78
use crate::{8
- normalise::{normalise_timeline, MediaScope},99
Error, ErrorKind, EventContext, EventContextCommand, EventContextDirection, EventContextPage,1010
TimelineItem,11
+ engine::MediaService,12
+ normalise::{MediaRegistration, normalise_observed},1113
};1214
1315
const MAX_CONTEXT_LIMIT: u16 = 50;1416
1517
pub(crate) async fn execute(1618
room: Room,17
- current_user_id: Option<ruma::OwnedUserId>,19
+ current_user_id: Option<OwnedUserId>,20
+ media: Arc<MediaService>,1821
command: EventContextCommand,1922
) -> Result<crate::CommandOutput, Error> {2023
if let EventContextCommand::Before { token, limit, .. }@@ -30,6 +33,7 @@ pub(crate) async fn execute(
3033
return execute_page(3134
room,3235
current_user_id,36
+ media,3337
token,3438
*limit,3539
matches!(command, EventContextCommand::Before { .. }),@@ -37,7 +41,7 @@ pub(crate) async fn execute(
3741
.await;3842
}3943
40
- let (event_id, limit) = match &command {44
+ let (event_id, before_limit, after_limit) = match &command {4145
EventContextCommand::Fetch {4246
event_id,4347
before_limit,@@ -46,71 +50,139 @@ pub(crate) async fn execute(
4650
} => {4751
validate_limit(*before_limit)?;4852
validate_limit(*after_limit)?;49
- (event_id.clone(), (*before_limit).max(*after_limit))53
+ (event_id.clone(), *before_limit, *after_limit)5054
}5155
EventContextCommand::Before { .. } | EventContextCommand::After { .. } => unreachable!(),5256
};53
- validate_limit(limit)?;54
-55
- let raw_context = if matches!(command, EventContextCommand::Fetch { .. }) {56
- Some(57
- room.event_with_context(&event_id, false, uint!(0), None)58
- .await59
- .map_err(|source| {60
- let error = Error::from_sdk("fetching event context failed", source);61
- if error62
- .server63
- .as_ref()64
- .is_some_and(|server| server.status == 404)65
- {66
- Error::new(ErrorKind::EventNotFound, "event was not found")67
- } else {68
- error69
- }70
- })?,71
- )72
- } else {73
- None74
- };57
+ let (context, registrations, _) =58
+ capture(room, current_user_id, event_id, before_limit, after_limit).await?;59
+ media.register(registrations).await;60
+ Ok(crate::CommandOutput::EventContext(context))61
+}7562
76
- let focus = TimelineFocus::Event {77
- target: event_id.clone(),78
- num_context_events: limit,79
- thread_mode: TimelineEventFocusThreadMode::Automatic {80
- hide_threaded_events: true,81
- },82
- };83
- let timeline = room84
- .timeline_builder()85
- .with_focus(focus)86
- .build()63
+pub(crate) async fn capture(64
+ room: Room,65
+ current_user_id: Option<OwnedUserId>,66
+ event_id: OwnedEventId,67
+ before_limit: u16,68
+ after_limit: u16,69
+) -> Result<(EventContext, Vec<MediaRegistration>, Vec<serde_json::Value>), Error> {70
+ validate_limit(before_limit)?;71
+ validate_limit(after_limit)?;72
+ let raw = room73
+ .event_with_context(&event_id, false, UInt::from(0_u16), None)8774
.await8875
.map_err(|source| {89
- Error::with_source(ErrorKind::Timeline, "building event context failed", source)76
+ let error = Error::from_sdk("fetching event context failed", source);77
+ if error78
+ .server79
+ .as_ref()80
+ .is_some_and(|server| server.status == 404)81
+ {82
+ Error::new(ErrorKind::EventNotFound, "event was not found")83
+ } else {84
+ error85
+ }9086
})?;91
-92
- if let EventContextCommand::Fetch { .. } = command {93
- let raw = raw_context.expect("fetch context response");94
- let normalized = normalize_items(timeline.items().await, current_user_id.as_deref());95
- let (before, event, after) = split_items(normalized, &event_id);96
- if event.is_none() && raw.event.is_none() {97
- return Err(Error::new(ErrorKind::EventNotFound, "event was not found"));98
- }99
- return Ok(crate::CommandOutput::EventContext(EventContext {100
- events_before: before,101
- event,102
- events_after: after,103
- start_token: raw.prev_batch_token,104
- end_token: raw.next_batch_token,105
- }));87
+ if raw.event.is_none() {88
+ return Err(Error::new(ErrorKind::EventNotFound, "event was not found"));10689
}90
+ let before = fetch_side(91
+ &room,92
+ raw.prev_batch_token.as_deref(),93
+ before_limit,94
+ true,95
+ current_user_id.as_deref(),96
+ );97
+ let after = fetch_side(98
+ &room,99
+ raw.next_batch_token.as_deref(),100
+ after_limit,101
+ false,102
+ current_user_id.as_deref(),103
+ );104
+ let (events_before, mut before_media, start_token, mut observed) = before.await?;105
+ let (events_after, after_media, end_token, following) = after.await?;106
+ observed.extend(following);107
+ if let Some(value) = raw108
+ .event109
+ .as_ref()110
+ .and_then(|event| serde_json::from_str(event.raw().json().get()).ok())111
+ {112
+ observed.push(value);113
+ }114
+ let event = raw.event.as_ref().and_then(|event| {115
+ normalise_sdk_event(event, current_user_id.as_deref(), room.room_id()).map(116
+ |(item, media)| {117
+ before_media.extend(media);118
+ item119
+ },120
+ )121
+ });122
+ before_media.extend(after_media);123
+ Ok((124
+ EventContext {125
+ events_before,126
+ event,127
+ events_after,128
+ start_token,129
+ end_token,130
+ },131
+ before_media,132
+ observed,133
+ ))134
+}107135
108
- unreachable!("fetch context returned above")136
+async fn fetch_side(137
+ room: &Room,138
+ token: Option<&str>,139
+ limit: u16,140
+ backwards: bool,141
+ current_user_id: Option<&ruma::UserId>,142
+) -> Result<143
+ (144
+ Vec<TimelineItem>,145
+ Vec<MediaRegistration>,146
+ Option<String>,147
+ Vec<serde_json::Value>,148
+ ),149
+ Error,150
+> {151
+ if limit == 0 {152
+ return Ok((Vec::new(), Vec::new(), token.map(str::to_owned), Vec::new()));153
+ }154
+ let Some(token) = token else {155
+ return Ok((Vec::new(), Vec::new(), None, Vec::new()));156
+ };157
+ let mut options = if backwards {158
+ MessagesOptions::backward()159
+ } else {160
+ MessagesOptions::forward()161
+ };162
+ options = options.from(token);163
+ options.limit = UInt::from(limit);164
+ let messages = room165
+ .messages(options)166
+ .await167
+ .map_err(|source| Error::from_sdk("fetching event context events failed", source))?;168
+ let mut events = messages.chunk;169
+ events.truncate(usize::from(limit));170
+ if backwards {171
+ events.reverse();172
+ }173
+ let next_token = messages.end;174
+ let observed = events175
+ .iter()176
+ .filter_map(|event| serde_json::from_str(event.raw().json().get()).ok())177
+ .collect();178
+ let (items, media) = normalise_events(events, current_user_id, room.room_id());179
+ Ok((items, media, next_token, observed))109180
}110181
111182
async fn execute_page(112183
room: Room,113
- current_user_id: Option<ruma::OwnedUserId>,184
+ current_user_id: Option<OwnedUserId>,185
+ media: Arc<MediaService>,114186
token: &str,115187
limit: u16,116188
backwards: bool,@@ -121,55 +193,20 @@ async fn execute_page(
121193
MessagesOptions::forward()122194
};123195
options = options.from(token);124
- options.limit = ruma::UInt::from(limit);196
+ options.limit = UInt::from(limit);125197
let messages = room126198
.messages(options)127199
.await128200
.map_err(|source| Error::from_sdk("fetching event context page failed", source))?;129
-130
- let ids: Vec<_> = messages131
- .chunk132
- .iter()133
- .filter_map(|event| event.event_id())134
- .collect();135
- let normalized = if let Some(target) = ids.first() {136
- let focus = TimelineFocus::Event {137
- target: target.clone(),138
- num_context_events: limit,139
- thread_mode: TimelineEventFocusThreadMode::Automatic {140
- hide_threaded_events: false,141
- },142
- };143
- let timeline = room144
- .timeline_builder()145
- .with_focus(focus)146
- .build()147
- .await148
- .map_err(|source| {149
- Error::with_source(150
- ErrorKind::Timeline,151
- "building event context page failed",152
- source,153
- )154
- })?;155
- normalize_items(timeline.items().await, current_user_id.as_deref())156
- } else {157
- Vec::new()158
- };159
-160
- let mut by_id = std::collections::HashMap::new();161
- for item in normalized {162
- if let TimelineItem::Event(event) = &item {163
- if let Some(event_id) = &event.event_id {164
- by_id.insert(event_id.clone(), item);165
- }166
- }167
- }168
- let mut events: Vec<_> = ids.iter().filter_map(|id| by_id.remove(id)).collect();201
+ let next_token = messages.end.clone();202
+ let mut events = messages.chunk;203
+ events.truncate(usize::from(limit));169204
if backwards {170205
events.reverse();171206
}172
- let next_token = messages.end.clone();207
+ let (events, registrations) =208
+ normalise_events(events, current_user_id.as_deref(), room.room_id());209
+ media.register(registrations).await;173210
let end_reached = next_token.is_none();174211
let direction = if backwards {175212
EventContextDirection::Before@@ -184,41 +221,37 @@ async fn execute_page(
184221
}))185222
}186223
187
-fn validate_limit(limit: u16) -> Result<(), Error> {188
- if limit > MAX_CONTEXT_LIMIT {189
- Err(Error::new(ErrorKind::Invalid, "context limit exceeds 50"))190
- } else {191
- Ok(())224
+fn normalise_events(225
+ events: Vec<SdkTimelineEvent>,226
+ current_user_id: Option<&ruma::UserId>,227
+ room_id: &RoomId,228
+) -> (Vec<TimelineItem>, Vec<MediaRegistration>) {229
+ let mut items = Vec::with_capacity(events.len());230
+ let mut media = Vec::new();231
+ for event in events {232
+ if let Some((item, registrations)) = normalise_sdk_event(&event, current_user_id, room_id) {233
+ items.push(item);234
+ media.extend(registrations);235
+ }192236
}237
+ (items, media)193238
}194239
195
-fn normalize_items(196
- items: imbl::Vector<Arc<matrix_sdk_ui::timeline::TimelineItem>>,240
+fn normalise_sdk_event(241
+ event: &SdkTimelineEvent,197242
current_user_id: Option<&ruma::UserId>,198
-) -> Vec<TimelineItem> {199
- let scope = MediaScope::new();200
- normalise_timeline(items.iter(), current_user_id, &scope)201
- .items202
- .into_iter()203
- .collect()243
+ room_id: &RoomId,244
+) -> Option<(TimelineItem, Vec<MediaRegistration>)> {245
+ let value: serde_json::Value = serde_json::from_str(event.raw().json().get()).ok()?;246
+ let event_id = value.get("event_id")?.as_str()?;247
+ let identity = format!("{room_id}\n{event_id}");248
+ normalise_observed(&value, current_user_id, &identity)204249
}205250
206
-fn split_items(207
- items: Vec<TimelineItem>,208
- event_id: &OwnedEventId,209
-) -> (Vec<TimelineItem>, Option<TimelineItem>, Vec<TimelineItem>) {210
- let items: Vec<_> = items211
- .into_iter()212
- .filter(|item| matches!(item, TimelineItem::Event(_)))213
- .collect();214
- let target = items.iter().position(|item| {215
- matches!(item, TimelineItem::Event(event) if event.event_id.as_ref() == Some(event_id))216
- });217
- let Some(index) = target else {218
- return (items, None, Vec::new());219
- };220
- let mut before = items;221
- let after = before.split_off(index + 1);222
- let event = before.pop();223
- (before, event, after)251
+fn validate_limit(limit: u16) -> Result<(), Error> {252
+ if limit > MAX_CONTEXT_LIMIT {253
+ Err(Error::new(ErrorKind::Invalid, "context limit exceeds 50"))254
+ } else {255
+ Ok(())256
+ }224257
}@@ -1,5 +1,5 @@
11
mod commands;2
-mod context;2
+pub(crate) mod context;33
pub(crate) mod executor;44
mod media;55
mod mutations;@@ -33,6 +33,11 @@ pub(crate) use crate::store_lock::StoreLock;
3333
#[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]3434
pub(crate) enum StoreLock {}3535
36
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]37
+pub(crate) use crate::mentions::store::MentionsStore;38
+#[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]39
+pub(crate) enum MentionsStore {}40
+3641
use self::{3742
commands::Job,3843
executor::{EngineBoxFuture, TaskSet},@@ -58,12 +63,40 @@ pub struct EngineConfig {
5863
pub initial_history_limit: u16,5964
pub max_media_bytes: usize,6065
pub max_media_sources: usize,66
+ pub max_mentions_bytes: usize,67
+ pub max_mentions_entries: usize,68
+ pub max_mention_context_events: usize,69
+ pub max_mention_candidates: usize,6170
/// warm a room up for sending when it's opened or someone starts typing in it6271
pub prepare_to_send: bool,6372
}6473
6574
impl EngineConfig {6675
#[must_use]76
+ pub const fn with_max_mentions_bytes(mut self, bytes: usize) -> Self {77
+ self.max_mentions_bytes = bytes;78
+ self79
+ }80
+81
+ #[must_use]82
+ pub const fn with_max_mentions_entries(mut self, entries: usize) -> Self {83
+ self.max_mentions_entries = entries;84
+ self85
+ }86
+87
+ #[must_use]88
+ pub const fn with_max_mention_context_events(mut self, events: usize) -> Self {89
+ self.max_mention_context_events = events;90
+ self91
+ }92
+93
+ #[must_use]94
+ pub const fn with_max_mention_candidates(mut self, candidates: usize) -> Self {95
+ self.max_mention_candidates = candidates;96
+ self97
+ }98
+99
+ #[must_use]67100
pub const fn with_command_capacity(mut self, capacity: usize) -> Self {68101
self.command_capacity = capacity;69102
self@@ -108,6 +141,10 @@ impl Default for EngineConfig {
108141
initial_history_limit: 30,109142
max_media_bytes: 20 * 1024 * 1024,110143
max_media_sources: 2_048,144
+ max_mentions_bytes: 64 * 1024 * 1024,145
+ max_mentions_entries: 5_000,146
+ max_mention_context_events: 25_000,147
+ max_mention_candidates: 5_000,111148
prepare_to_send: true,112149
}113150
}@@ -117,10 +154,12 @@ impl Default for EngineConfig {
117154
pub enum EngineError {118155
#[error("the Matrix client is not authenticated")]119156
Unauthenticated,120
- #[error("command, event, and media source capacities must be nonzero")]157
+ #[error("command, event, media source, and mentions capacities must be nonzero")]121158
ZeroCapacity,122159
#[error("could not build the sync service")]123160
SyncService(#[source] matrix_sdk_ui::sync_service::Error),161
+ #[error("could not open the mentions service")]162
+ MentionsStore(#[source] crate::Error),124163
#[error("the event receiver has closed")]125164
EventReceiverClosed,126165
#[error("engine task {task} panicked")]@@ -133,9 +172,10 @@ impl EngineError {
133172
kind: match self {134173
Self::EventReceiverClosed => EngineErrorKind::EventReceiverClosed,135174
Self::TaskPanicked { .. } => EngineErrorKind::Watcher,136
- Self::Unauthenticated | Self::ZeroCapacity | Self::SyncService(_) => {137
- EngineErrorKind::Other138
- }175
+ Self::Unauthenticated176
+ | Self::ZeroCapacity177
+ | Self::SyncService(_)178
+ | Self::MentionsStore(_) => EngineErrorKind::Other,139179
},140180
message: self.to_string(),141181
}@@ -282,6 +322,9 @@ pub struct Engine {
282322
preparations: prepare::Preparations,283323
/// shared with every `ClientHandle`, so the store stays locked while the client is reachable284324
_store_lock: Option<Arc<StoreLock>>,325
+ mentions: crate::mentions::service::Handle,326
+ mentions_service: Option<crate::mentions::service::Service>,327
+ mentions_updates: Option<watch::Receiver<crate::MentionsPage>>,285328
}286329
287330
impl Engine {@@ -291,7 +334,7 @@ impl Engine {
291334
config: EngineConfig,292335
) -> Result<(Self, ClientHandle, EventStream), EngineError> {293336
let session_changes = client.subscribe_to_session_changes();294
- Self::with_persistence(client, config, session_changes, None, None).await337
+ Self::with_persistence(client, config, session_changes, None, None, None).await295338
}296339
297340
/// `session_changes` is taken when the client is built, since the sdk can refresh@@ -302,6 +345,7 @@ impl Engine {
302345
session_changes: broadcast::Receiver<SessionChange>,303346
persistence: Option<Persistence>,304347
store_lock: Option<StoreLock>,348
+ mentions: Option<MentionsStore>,305349
) -> Result<(Self, ClientHandle, EventStream), EngineError> {306350
if client.user_id().is_none() {307351
return Err(EngineError::Unauthenticated);@@ -309,9 +353,27 @@ impl Engine {
309353
if config.command_capacity == 0310354
|| config.event_capacity == 0311355
|| config.max_media_sources == 0356
+ || config.max_mentions_entries == 0357
+ || config.max_mention_context_events == 0358
+ || config.max_mention_candidates == 0312359
{313360
return Err(EngineError::ZeroCapacity);314361
}362
+ let store_lock = store_lock.map(Arc::new);363
+ let media = Arc::new(MediaService::new(364
+ client.clone(),365
+ config.max_media_bytes,366
+ config.max_media_sources,367
+ ));368
+ let (mentions, mentions_service, mentions_updates) =369
+ crate::mentions::service::Service::new(370
+ client.clone(),371
+ media.clone(),372
+ mentions,373
+ config.clone(),374
+ )375
+ .await376
+ .map_err(EngineError::MentionsStore)?;315377
let sync = Arc::new(316378
SyncService::builder(client.clone())317379
.with_offline_mode()@@ -319,16 +381,11 @@ impl Engine {
319381
(ROOM_PACKS.into(), "*".to_owned()),320382
(ROOM_PREVIEW_URLS.into(), String::new()),321383
])384
+ .with_room_list_response_commit_hook(mentions.hook())322385
.build()323386
.await324387
.map_err(EngineError::SyncService)?,325388
);326
- let store_lock = store_lock.map(Arc::new);327
- let media = Arc::new(MediaService::new(328
- client.clone(),329
- config.max_media_bytes,330
- config.max_media_sources,331
- ));332389
let (handle, port, events) = transport_pair(333390
client.clone(),334391
sync.clone(),@@ -374,6 +431,9 @@ impl Engine {
374431
stops,375432
logouts: Arc::new(watch::Sender::new(Logouts::default())),376433
_store_lock: store_lock,434
+ mentions,435
+ mentions_service: Some(mentions_service),436
+ mentions_updates: Some(mentions_updates),377437
};378438
Ok((engine, handle, events))379439
}@@ -459,6 +519,17 @@ impl Engine {
459519
460520
fn start_tasks(&mut self) {461521
let queues = self.queues.take().expect("engine tasks start once");522
+ if let Some(service) = self.mentions_service.take() {523
+ self.tasks524
+ .spawn(TaskContext::Watcher("mentions service"), service.run());525
+ }526
+ if let Some(updates) = self.mentions_updates.take() {527
+ let events = self.port.event_sender();528
+ self.tasks.spawn(529
+ TaskContext::Watcher("mentions updates"),530
+ crate::mentions::service::watch_updates(updates, events),531
+ );532
+ }462533
let (incoming, incoming_verifications) = mpsc::unbounded_channel();463534
self.verification_handler = Some(self.verification_handler(incoming));464535
let client = &self.client;@@ -754,9 +825,18 @@ impl Engine {
754825
return None;755826
};756827
let current_user_id = client.user_id().map(ToOwned::to_owned);757
- self.tasks.spawn(TaskContext::Command("event context"), async move {758
- reply.send(context::execute(room, current_user_id, command).await);759
- });828
+ let media = self.media.clone();829
+ self.tasks830
+ .spawn(TaskContext::Command("event context"), async move {831
+ reply.send(context::execute(room, current_user_id, media, command).await);832
+ });833
+ }834
+ Command::Mentions(command) => {835
+ let mentions = self.mentions.clone();836
+ self.tasks837
+ .spawn(TaskContext::Command("mentions command"), async move {838
+ reply.send(mentions.execute(command).await);839
+ });760840
}761841
Command::Media(MediaCommand::Upload(upload)) => self.upload(upload, reply),762842
Command::Encryption(command) => enqueue(&self.encryption, Job { command, reply }),@@ -1040,10 +1120,13 @@ impl Engine {
10401120
self.client.remove_event_handler(handler);10411121
}10421122
self.port.close();1123
+ self.sync.stop().await;1124
+ if self.mentions_service.is_none() {1125
+ self.mentions.stop().await;1126
+ }10431127
self.tasks.shutdown().await;10441128
self.rooms.clear();10451129
self.media.clear().await;1046
- self.sync.stop().await;10471130
self.port10481131
.connection_sender()10491132
.send_replace(ConnectionState::Stopped);@@ -3,6 +3,8 @@
33
mod encryption;44
mod engine;55
mod error;6
+mod mentions;7
+mod mentions_model;68
#[cfg(feature = "mirror")]79
mod mirror;810
mod model;@@ -41,6 +43,10 @@ pub use session_store::{EncryptedSessionStore, SessionStore};
4143
pub use engine::{Engine, EngineConfig, EngineError};4244
pub use error::{Error, ErrorKind, ServerError};4345
pub use imbl;46
+pub use mentions_model::{47
+ MentionCaptureState, MentionCoverage, MentionEntry, MentionGap, MentionKey, MentionReason,48
+ MentionsBoundary, MentionsCommand, MentionsPage, MentionsUpdate, SavedMentionContext,49
+};4450
#[cfg(feature = "mirror")]4551
pub use mirror::{4652
Mirror, MirrorHandle, SessionStatus, State, StateChange, Subscription, TimelineChange,@@ -49,7 +55,8 @@ pub use mirror::{
4955
pub use model::{5056
AccountCommand, AttachmentEncryption, AttachmentUpload, Change, Command, CommandOutput,5157
ConnectionState, CreateRoomOptions, EncryptionCommand, EncryptionState, EngineErrorKind,52
- EngineFailure, Event, EventRef, HistoryState, ImagePack, ImagePackSource, ImagePacks,58
+ EngineFailure, Event, EventContext, EventContextCommand, EventContextDirection,59
+ EventContextPage, EventRef, HistoryState, ImagePack, ImagePackSource, ImagePacks,5360
KeyBackupState, LatestEvent, LifecycleCommand, LinkPreview, LinkPreviewImage, Media,5461
MediaCommand, MediaDescriptor, MediaFile, MediaId, MediaKind, MediaLocation, MediaSize,5562
MemberFilter, MemberMembership, MemberSummary, MembershipChange, Message, MessageContent,@@ -62,8 +69,7 @@ pub use model::{
6269
Sender, SpaceChild, Termination, ThreadSummary, TimelineCommand, TimelineContent, TimelineDiff,6370
TimelineDiffError, TimelineEvent, TimelineId, TimelineItem, TimelineItemId, TypingState,6471
UploadDetails, UploadOutcome, UploadThumbnail, UtdCause, VerificationAction,65
- VerificationFailure, VerificationState, VerificationTarget, EventContext, EventContextCommand,66
- EventContextDirection, EventContextPage,72
+ VerificationFailure, VerificationState, VerificationTarget,6773
};6874
pub use ruma;6975
pub use transport::{ClientHandle, EventStream, Pending, SpaceHierarchy};@@ -0,0 +1,6 @@
1
+pub(crate) mod service;2
+mod state;3
+#[cfg(test)]4
+mod state_tests;5
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]6
+pub(crate) mod store;@@ -0,0 +1,1314 @@
1
+use std::{2
+ collections::{BTreeMap, BTreeSet},3
+ sync::Arc,4
+ time::Duration,5
+};6
+7
+use futures_util::{StreamExt, stream::FuturesUnordered};8
+use matrix_sdk::{9
+ Client, RoomState,10
+ sliding_sync::{ResponseCommitBatch, ResponseCommitHook},11
+};12
+use ruma::{13
+ OwnedRoomId, OwnedUserId,14
+ events::{AnySyncTimelineEvent, room::encrypted::OriginalSyncRoomEncryptedEvent},15
+ serde::Raw,16
+};17
+use serde_json::Value;18
+use tokio::sync::{mpsc, oneshot, watch};19
+20
+use super::state::{Candidate, Data, SavedEvent, Window, id, invalid, now};21
+use crate::{22
+ CommandOutput, EngineConfig, Error, ErrorKind, Event, MentionCaptureState, MentionEntry,23
+ MentionGap, MentionKey, MentionReason, MentionsCommand, MentionsPage, MentionsUpdate,24
+ TimelineContent, TimelineItem,25
+ engine::{MediaService, MentionsStore, context, executor::EngineBoxFuture},26
+ normalise::{MediaRegistration, normalise_observed},27
+ transport::EventSender,28
+};29
+30
+#[cfg(all(test, feature = "sqlite"))]31
+#[path = "service_tests.rs"]32
+mod tests;33
+34
+#[derive(Clone)]35
+pub(crate) struct Handle(mpsc::Sender<Request>);36
+37
+enum Request {38
+ Batch(ResponseCommitBatch, oneshot::Sender<Result<(), Error>>),39
+ Command(40
+ MentionsCommand,41
+ oneshot::Sender<Result<CommandOutput, Error>>,42
+ ),43
+ Stop(oneshot::Sender<()>),44
+}45
+46
+pub(crate) struct Service {47
+ client: Client,48
+ account: OwnedUserId,49
+ media: Arc<MediaService>,50
+ store: Option<MentionsStore>,51
+ data: Data,52
+ config: EngineConfig,53
+ requests: mpsc::Receiver<Request>,54
+ updates: watch::Sender<MentionsPage>,55
+ recovery_budget: usize,56
+ latest_scan: bool,57
+ recovery_retry: u64,58
+}59
+60
+enum Work {61
+ Capture(62
+ String,63
+ u64,64
+ Result<(crate::EventContext, Vec<MediaRegistration>, Vec<Value>), Error>,65
+ ),66
+ Candidate(String, Result<Value, Error>),67
+ History(68
+ bool,69
+ Option<String>,70
+ Result<ruma::api::client::push::get_notifications::v3::Response, Error>,71
+ ),72
+}73
+74
+impl Handle {75
+ pub(crate) fn hook(&self) -> ResponseCommitHook {76
+ let handle = self.clone();77
+ ResponseCommitHook::new(move |batch| {78
+ let handle = handle.clone();79
+ async move {80
+ let (reply, response) = oneshot::channel();81
+ handle82
+ .083
+ .send(Request::Batch(batch, reply))84
+ .await85
+ .map_err(|_| "mentions service stopped".to_owned())?;86
+ response87
+ .await88
+ .map_err(|_| "mentions commit acknowledgement dropped".to_owned())?89
+ .map_err(|error| error.to_string())90
+ }91
+ })92
+ }93
+94
+ pub(crate) async fn execute(&self, command: MentionsCommand) -> Result<CommandOutput, Error> {95
+ let (reply, response) = oneshot::channel();96
+ self.097
+ .send(Request::Command(command, reply))98
+ .await99
+ .map_err(|_| Error::interrupted())?;100
+ response.await.map_err(|_| Error::interrupted())?101
+ }102
+103
+ pub(crate) async fn stop(&self) {104
+ let (reply, response) = oneshot::channel();105
+ if self.0.send(Request::Stop(reply)).await.is_ok() {106
+ let _ = response.await;107
+ }108
+ }109
+}110
+111
+impl Service {112
+ pub(crate) async fn new(113
+ client: Client,114
+ media: Arc<MediaService>,115
+ store: Option<MentionsStore>,116
+ config: EngineConfig,117
+ ) -> Result<(Handle, Self, watch::Receiver<MentionsPage>), Error> {118
+ let account = client119
+ .user_id()120
+ .ok_or_else(|| {121
+ Error::new(122
+ ErrorKind::Unauthenticated,123
+ "mentions require an authenticated account",124
+ )125
+ })?126
+ .to_owned();127
+ #[cfg(any(feature = "sqlite", feature = "indexeddb"))]128
+ let data: Data = store129
+ .as_ref()130
+ .map(|store| store.read("inbox"))131
+ .transpose()?132
+ .flatten()133
+ .unwrap_or_default();134
+ #[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]135
+ let data = Data::default();136
+ #[cfg(any(feature = "sqlite", feature = "indexeddb"))]137
+ if store138
+ .as_ref()139
+ .is_some_and(|store| store.revision() != data.revision)140
+ {141
+ return Err(Error::new(142
+ ErrorKind::Store,143
+ "mentions revision does not match its checkpoint",144
+ ));145
+ }146
+ let page = data.page(&account, None, 50)?;147
+ media148
+ .register(149
+ page.entries150
+ .iter()151
+ .filter_map(|entry| data.events.get(&id(&entry.key)))152
+ .flat_map(|event| event.media.clone())153
+ .collect(),154
+ )155
+ .await;156
+ let (updates, receiver) = watch::channel(page);157
+ let (sender, requests) = mpsc::channel(config.command_capacity.max(1));158
+ Ok((159
+ Handle(sender),160
+ Self {161
+ client,162
+ account,163
+ media,164
+ store,165
+ data,166
+ config,167
+ requests,168
+ updates,169
+ recovery_budget: 20,170
+ latest_scan: true,171
+ recovery_retry: 0,172
+ },173
+ receiver,174
+ ))175
+ }176
+177
+ async fn save(&mut self, mut next: Data) -> Result<(), Error> {178
+ next.revision = self179
+ .data180
+ .revision181
+ .checked_add(1)182
+ .ok_or_else(|| invalid("mentions revision exhausted"))?;183
+ next.bound(184
+ self.config.max_mentions_entries,185
+ self.config.max_mention_context_events,186
+ self.config.max_mention_candidates,187
+ );188
+ #[cfg(any(feature = "sqlite", feature = "indexeddb"))]189
+ if let Some(store) = &mut self.store {190
+ loop {191
+ let value = serde_json::to_value(&next).map_err(|error| {192
+ Error::with_source(193
+ ErrorKind::Store,194
+ "encoding the mentions inbox failed",195
+ error,196
+ )197
+ })?;198
+ match store199
+ .commit(200
+ self.data.revision,201
+ vec![super::store::Write::Set("inbox".to_owned(), value)],202
+ )203
+ .await204
+ {205
+ Ok(_) => break,206
+ Err(error)207
+ if error.message == "mentions storage limit exceeded"208
+ && next.evict_one() => {}209
+ Err(error) => return Err(error),210
+ }211
+ }212
+ }213
+ self.data = next;214
+ let page = self.data.page(&self.account, None, 50)?;215
+ self.media216
+ .register(217
+ page.entries218
+ .iter()219
+ .filter_map(|entry| self.data.events.get(&id(&entry.key)))220
+ .flat_map(|event| event.media.clone())221
+ .collect(),222
+ )223
+ .await;224
+ self.updates.send_replace(page);225
+ Ok(())226
+ }227
+228
+ async fn command(&mut self, command: MentionsCommand) -> Result<CommandOutput, Error> {229
+ match command {230
+ MentionsCommand::Page { cursor, limit } => {231
+ let page = self.data.page(&self.account, cursor.as_deref(), limit)?;232
+ self.media233
+ .register(234
+ page.entries235
+ .iter()236
+ .filter_map(|entry| self.data.events.get(&id(&entry.key)))237
+ .flat_map(|event| event.media.clone())238
+ .collect(),239
+ )240
+ .await;241
+ Ok(CommandOutput::MentionsPage(page))242
+ }243
+ MentionsCommand::Context(key) => {244
+ let (context, media) = self.data.context(&key)?;245
+ self.media.register(media).await;246
+ Ok(CommandOutput::MentionContext(context))247
+ }248
+ MentionsCommand::RetryContext(key) => {249
+ let room = self250
+ .client251
+ .get_room(&key.room_id)252
+ .filter(|room| room.state() == RoomState::Joined)253
+ .ok_or_else(|| {254
+ Error::new(255
+ ErrorKind::Forbidden,256
+ "saved context cannot be refetched after leaving the room",257
+ )258
+ })?;259
+ drop(room);260
+ let mut next = self.data.clone();261
+ let entry = next262
+ .entries263
+ .get_mut(&id(&key))264
+ .ok_or_else(|| Error::new(ErrorKind::EventNotFound, "mention was not found"))?;265
+ entry.capture = MentionCaptureState::Pending;266
+ entry.revision = self.data.revision + 1;267
+ next.windows.entry(id(&key)).or_insert(Window {268
+ before: Vec::new(),269
+ target: Some(id(&key)),270
+ after: Vec::new(),271
+ start: None,272
+ end: None,273
+ });274
+ self.save(next).await?;275
+ let (context, media) = self.data.context(&key)?;276
+ self.media.register(media).await;277
+ Ok(CommandOutput::MentionContext(context))278
+ }279
+ MentionsCommand::SetRead { keys, read } => {280
+ let mut next = self.data.clone();281
+ for key in keys {282
+ if let Some(entry) = next.entries.get_mut(&id(&key)) {283
+ entry.read = read;284
+ entry.revision = self.data.revision + 1;285
+ }286
+ }287
+ self.save(next).await?;288
+ Ok(CommandOutput::Done)289
+ }290
+ MentionsCommand::ReadThrough(boundary) => {291
+ if boundary.account != self.account || boundary.revision > self.data.revision {292
+ return Err(invalid("invalid mentions read boundary"));293
+ }294
+ let mut next = self.data.clone();295
+ for entry in next296
+ .entries297
+ .values_mut()298
+ .filter(|entry| entry.discovery_revision <= boundary.revision)299
+ {300
+ entry.read = true;301
+ entry.revision = self.data.revision + 1;302
+ }303
+ self.save(next).await?;304
+ Ok(CommandOutput::Done)305
+ }306
+ MentionsCommand::Recover => {307
+ self.recovery_budget = 20;308
+ self.latest_scan = true;309
+ self.recovery_retry = 0;310
+ let mut next = self.data.clone();311
+ next.coverage.recovering = true;312
+ next.coverage.historical_supported = None;313
+ self.save(next).await?;314
+ Ok(CommandOutput::Done)315
+ }316
+ }317
+ }318
+319
+ fn gap(data: &mut Data, room_id: Option<OwnedRoomId>, reason: &str, token: Option<String>) {320
+ if !data321
+ .coverage322
+ .gaps323
+ .iter()324
+ .any(|gap| gap.room_id == room_id && gap.reason == reason && gap.token == token)325
+ {326
+ data.coverage.gaps.push(MentionGap {327
+ room_id,328
+ reason: reason.to_owned(),329
+ token,330
+ });331
+ if data.coverage.gaps.len() > 128 {332
+ data.coverage.gaps.remove(0);333
+ data.coverage.truncated = true;334
+ }335
+ }336
+ }337
+338
+ async fn batch(&mut self, batch: ResponseCommitBatch) -> Result<(), Error> {339
+ let mut next = self.data.clone();340
+ if next.checkpoints.get(&batch.connection_id) == Some(&batch.next_pos) {341
+ return Ok(());342
+ }343
+ if next.checkpoints.get(&batch.connection_id) != batch.previous_pos.as_ref() {344
+ Self::gap(345
+ &mut next,346
+ None,347
+ "sync checkpoint continuity is unknown",348
+ None,349
+ );350
+ }351
+ for (room_id, raw) in &batch.raw_rooms {352
+ if raw.limited || !next.observed_rooms.contains(room_id) {353
+ Self::gap(354
+ &mut next,355
+ Some(room_id.clone()),356
+ if raw.limited {357
+ "limited sync timeline"358
+ } else {359
+ "initial room history is incomplete"360
+ },361
+ raw.prev_batch.clone(),362
+ );363
+ }364
+ }365
+ let room_ids: BTreeSet<_> = batch366
+ .raw_rooms367
+ .keys()368
+ .chain(batch.response.rooms.joined.keys())369
+ .cloned()370
+ .collect();371
+ for room_id in &room_ids {372
+ if !self373
+ .client374
+ .get_room(room_id)375
+ .is_some_and(|room| room.state() == RoomState::Joined)376
+ {377
+ continue;378
+ }379
+ next.observed_rooms.insert(room_id.clone());380
+ let mut observed = BTreeMap::new();381
+ if let Some(raw) = batch.raw_rooms.get(room_id) {382
+ for event in &raw.timeline {383
+ let value: Value =384
+ serde_json::from_str(event.json().get()).map_err(|error| {385
+ Error::with_source(ErrorKind::Invalid, "invalid observed event", error)386
+ })?;387
+ if let Some(event_id) = value["event_id"].as_str() {388
+ observed.insert(event_id.to_owned(), value);389
+ }390
+ }391
+ }392
+ if let Some(update) = batch.response.rooms.joined.get(room_id) {393
+ for event in &update.timeline.events {394
+ let value: Value =395
+ serde_json::from_str(event.raw().json().get()).map_err(|error| {396
+ Error::with_source(ErrorKind::Invalid, "invalid observed event", error)397
+ })?;398
+ if let Some(event_id) = value["event_id"].as_str() {399
+ observed.insert(event_id.to_owned(), value);400
+ }401
+ }402
+ }403
+ let mut observed: Vec<_> = observed.into_values().collect();404
+ observed.sort_by_key(|value| value["origin_server_ts"].as_u64().unwrap_or(0));405
+ for value in observed {406
+ self.observe(&mut next, room_id, value).await?;407
+ }408
+ }409
+ for room_id in batch.response.rooms.left.keys() {410
+ Self::gap(411
+ &mut next,412
+ Some(room_id.clone()),413
+ "room left, retained context is read only",414
+ None,415
+ );416
+ next.candidates417
+ .retain(|_, candidate| &candidate.room_id != room_id);418
+ for entry in next419
+ .entries420
+ .values_mut()421
+ .filter(|entry| &entry.key.room_id == room_id)422
+ {423
+ entry.capture = MentionCaptureState::Unavailable;424
+ entry.revision = next.revision + 1;425
+ }426
+ }427
+ let reevaluate: Vec<_> = next428
+ .entries429
+ .values()430
+ .filter(|entry| room_ids.contains(&entry.key.room_id))431
+ .filter_map(|entry| {432
+ next.events433
+ .get(&id(&entry.key))434
+ .and_then(|event| event.raw.clone())435
+ .map(|raw| (entry.key.room_id.clone(), raw))436
+ })437
+ .collect();438
+ for (room_id, raw) in reevaluate {439
+ self.observe(&mut next, &room_id, raw).await?;440
+ }441
+ self.filter_ignored(&mut next).await;442
+ next.checkpoints.insert(batch.connection_id, batch.next_pos);443
+ self.save(next).await444
+ }445
+446
+ async fn filter_ignored(&self, next: &mut Data) {447
+ let mut removed = Vec::new();448
+ for (key, entry) in &next.entries {449
+ if let TimelineItem::Event(event) = &entry.event {450
+ if self.client.is_user_ignored(&event.sender.user_id).await {451
+ removed.push(key.clone());452
+ }453
+ }454
+ }455
+ for key in removed {456
+ next.entries.remove(&key);457
+ next.windows.remove(&key);458
+ }459
+ next.collect_events();460
+ }461
+462
+ async fn observe(463
+ &self,464
+ data: &mut Data,465
+ room_id: &OwnedRoomId,466
+ mut value: Value,467
+ ) -> Result<(), Error> {468
+ let Some(event_id) = value["event_id"]469
+ .as_str()470
+ .and_then(|value| value.parse().ok())471
+ else {472
+ return Ok(());473
+ };474
+ let key = MentionKey {475
+ room_id: room_id.clone(),476
+ event_id,477
+ };478
+ let identity = id(&key);479
+ let Some(timestamp) = value["origin_server_ts"].as_u64() else {480
+ return Ok(());481
+ };482
+ data.coverage.earliest_timestamp_ms = Some(483
+ data.coverage484
+ .earliest_timestamp_ms485
+ .map_or(timestamp, |old| old.min(timestamp)),486
+ );487
+ if value["type"] == "m.room.redaction" {488
+ if let Some(target) = value["redacts"]489
+ .as_str()490
+ .or_else(|| value["content"]["redacts"].as_str())491
+ .and_then(|value| value.parse().ok())492
+ {493
+ let target = id(&MentionKey {494
+ room_id: room_id.clone(),495
+ event_id: target,496
+ });497
+ data.redacted.insert(target.clone(), timestamp);498
+ data.entries.remove(&target);499
+ data.windows.remove(&target);500
+ data.candidates.remove(&target);501
+ data.edits.remove(&target);502
+ data.candidates.retain(|_, candidate| {503
+ candidate.room_id != *room_id504
+ || candidate.raw["content"]["m.relates_to"]["event_id"].as_str()505
+ != target.split_once('\n').map(|(_, event)| event)506
+ });507
+ if let Some(saved) = data.events.get_mut(&target) {508
+ redact(saved);509
+ }510
+ data.touch_contexts(&target);511
+ }512
+ return Ok(());513
+ }514
+ if data.redacted.contains_key(&identity)515
+ || value["unsigned"].get("redacted_because").is_some()516
+ {517
+ data.redacted.insert(identity.clone(), timestamp);518
+ data.entries.remove(&identity);519
+ data.windows.remove(&identity);520
+ data.candidates.remove(&identity);521
+ if let Some(saved) = data.events.get_mut(&identity) {522
+ redact(saved);523
+ }524
+ data.touch_contexts(&identity);525
+ return Ok(());526
+ }527
+ if value["content"]["m.relates_to"]["rel_type"] == "m.replace" {528
+ if let Some(target) = value["content"]["m.relates_to"]["event_id"]529
+ .as_str()530
+ .and_then(|value| value.parse().ok())531
+ {532
+ let target = id(&MentionKey {533
+ room_id: room_id.clone(),534
+ event_id: target,535
+ });536
+ if data.redacted.contains_key(&target) {537
+ return Ok(());538
+ }539
+ let newer = data.edits.get(&target).is_none_or(|old| {540
+ (old["origin_server_ts"].as_u64(), old["event_id"].as_str())541
+ < (Some(timestamp), value["event_id"].as_str())542
+ });543
+ if newer {544
+ if let Some(original) =545
+ data.events.get(&target).and_then(|saved| saved.raw.clone())546
+ {547
+ if original["sender"] != value["sender"] {548
+ return Ok(());549
+ }550
+ data.edits.insert(target.clone(), value.clone());551
+ value = edited(original, &value);552
+ return Box::pin(self.observe(data, room_id, value)).await;553
+ }554
+ data.candidates.insert(555
+ identity,556
+ Candidate {557
+ room_id: room_id.clone(),558
+ raw: value,559
+ attempts: 0,560
+ retry_at_ms: 0,561
+ },562
+ );563
+ }564
+ }565
+ return Ok(());566
+ }567
+ if let Some(edit) = value["unsigned"]["m.relations"].get("m.replace").cloned() {568
+ if edit["sender"] == value["sender"]569
+ && edit["content"]["m.new_content"].is_object()570
+ && edit["content"]["m.relates_to"]["event_id"]571
+ .as_str()572
+ .is_none_or(|target| target == key.event_id.as_str())573
+ && edit["origin_server_ts"].as_u64().is_some()574
+ && data.edits.get(&identity).is_none_or(|old| {575
+ (old["origin_server_ts"].as_u64(), old["event_id"].as_str())576
+ < (edit["origin_server_ts"].as_u64(), edit["event_id"].as_str())577
+ })578
+ {579
+ data.edits.insert(identity.clone(), edit);580
+ }581
+ }582
+ if let Some(edit) = data.edits.get(&identity) {583
+ if value["sender"] == edit["sender"] {584
+ value = edited(value, edit);585
+ }586
+ }587
+ if let Some(edit) = value["unsigned"].get("macltok_edit").cloned() {588
+ data.edits.insert(identity.clone(), edit);589
+ }590
+ if value["type"] == "m.room.encrypted" {591
+ data.candidates.entry(identity).or_insert(Candidate {592
+ room_id: room_id.clone(),593
+ raw: value,594
+ attempts: 0,595
+ retry_at_ms: 0,596
+ });597
+ return Ok(());598
+ }599
+ let Some((item, media)) = normalise_observed(&value, Some(&self.account), &identity) else {600
+ return Ok(());601
+ };602
+ self.supplement(data, &key, &item, &media, &value);603
+ let (reasons, suppressed, unresolved) = self.classify(data, room_id, &value).await;604
+ if unresolved {605
+ data.candidates606
+ .entry(identity.clone())607
+ .or_insert(Candidate {608
+ room_id: room_id.clone(),609
+ raw: value.clone(),610
+ attempts: 0,611
+ retry_at_ms: now() + 30_000,612
+ });613
+ } else {614
+ data.candidates.remove(&identity);615
+ }616
+ if reasons.is_empty() {617
+ data.entries.remove(&identity);618
+ data.windows.remove(&identity);619
+ if data.events.contains_key(&identity) || data.edits.contains_key(&identity) {620
+ data.events.insert(621
+ identity,622
+ SavedEvent {623
+ item,624
+ media,625
+ raw: Some(value),626
+ },627
+ );628
+ }629
+ return Ok(());630
+ }631
+ let previous = data.entries.get(&identity);632
+ let changed = previous.is_none_or(|entry| {633
+ entry.event != item634
+ || entry.reasons != reasons635
+ || entry.notification_suppressed != suppressed636
+ });637
+ let entry = MentionEntry {638
+ key,639
+ timestamp_ms: timestamp,640
+ discovered_at_ms: previous.map_or_else(now, |entry| entry.discovered_at_ms),641
+ discovery_revision: previous642
+ .map_or(data.revision + 1, |entry| entry.discovery_revision),643
+ revision: if changed {644
+ data.revision + 1645
+ } else {646
+ previous.unwrap().revision647
+ },648
+ event: item.clone(),649
+ reasons,650
+ read: previous.is_some_and(|entry| entry.read),651
+ capture: previous.map_or(MentionCaptureState::Pending, |entry| {652
+ if changed {653
+ MentionCaptureState::Pending654
+ } else {655
+ entry.capture.clone()656
+ }657
+ }),658
+ room_name: self.client.get_room(room_id).and_then(|room| room.name()),659
+ notification_suppressed: suppressed,660
+ };661
+ data.entries.insert(identity.clone(), entry);662
+ data.windows.entry(identity.clone()).or_insert(Window {663
+ before: Vec::new(),664
+ target: Some(identity.clone()),665
+ after: Vec::new(),666
+ start: None,667
+ end: None,668
+ });669
+ data.events.insert(670
+ identity,671
+ SavedEvent {672
+ item,673
+ media,674
+ raw: Some(value),675
+ },676
+ );677
+ Ok(())678
+ }679
+680
+ fn supplement(681
+ &self,682
+ data: &mut Data,683
+ key: &MentionKey,684
+ item: &TimelineItem,685
+ media: &[MediaRegistration],686
+ raw: &Value,687
+ ) {688
+ let TimelineItem::Event(event) = item else {689
+ return;690
+ };691
+ let identity = id(key);692
+ if data693
+ .events694
+ .get(&identity)695
+ .is_some_and(|saved| saved.item != *item)696
+ {697
+ data.touch_contexts(&identity);698
+ }699
+ let mut retained = data.events.contains_key(&identity);700
+ let saved_events = &data.events;701
+ for (target, window) in &mut data.windows {702
+ let Some(entry) = data.entries.get_mut(target) else {703
+ continue;704
+ };705
+ if entry.key.room_id != key.room_id706
+ || (entry.timestamp_ms, target.as_str()) >= (event.timestamp_ms, identity.as_str())707
+ || window.after.contains(&identity)708
+ {709
+ continue;710
+ }711
+ let previous = window.after.clone();712
+ window.after.push(identity.clone());713
+ window.after.sort_by_key(|key| {714
+ if key == &identity {715
+ (event.timestamp_ms, key.clone())716
+ } else {717
+ (718
+ saved_events719
+ .get(key)720
+ .and_then(|saved| match &saved.item {721
+ TimelineItem::Event(event) => Some(event.timestamp_ms),722
+ _ => None,723
+ })724
+ .unwrap_or(0),725
+ key.clone(),726
+ )727
+ }728
+ });729
+ window.after.truncate(10);730
+ if window.after == previous {731
+ continue;732
+ }733
+ window.end = None;734
+ retained |= window.after.contains(&identity);735
+ if entry.capture == MentionCaptureState::Partial736
+ && window.after.len() == 10737
+ && (window.before.len() == 10 || window.start.is_none())738
+ {739
+ entry.capture = MentionCaptureState::Ready;740
+ }741
+ entry.revision = data.revision + 1;742
+ }743
+ if retained {744
+ data.events.insert(745
+ identity,746
+ SavedEvent {747
+ item: item.clone(),748
+ media: media.to_vec(),749
+ raw: Some(raw.clone()),750
+ },751
+ );752
+ }753
+ }754
+755
+ async fn classify(756
+ &self,757
+ data: &Data,758
+ room_id: &OwnedRoomId,759
+ value: &Value,760
+ ) -> (Vec<MentionReason>, bool, bool) {761
+ let Some(sender) = value["sender"]762
+ .as_str()763
+ .and_then(|sender| sender.parse::<OwnedUserId>().ok())764
+ else {765
+ return (vec![], false, false);766
+ };767
+ if sender == self.account768
+ || self.client.is_user_ignored(&sender).await769
+ || value["type"] != "m.room.message"770
+ {771
+ return (vec![], false, false);772
+ }773
+ let Some(room) = self774
+ .client775
+ .get_room(room_id)776
+ .filter(|room| room.state() == RoomState::Joined)777
+ else {778
+ return (vec![], false, false);779
+ };780
+ let raw: Raw<AnySyncTimelineEvent> =781
+ Raw::from_json_string(value.to_string()).expect("serialized event");782
+ let actions = room.event_push_actions(&raw).await.ok().flatten();783
+ let notify = actions.as_ref().is_some_and(|actions| {784
+ actions785
+ .iter()786
+ .any(|action| matches!(action, ruma::push::Action::Notify))787
+ });788
+ let highlight = actions.as_ref().is_some_and(|actions| actions.iter().any(|action| matches!(action, ruma::push::Action::SetTweak(ruma::push::Tweak::Highlight(value)) if value != &ruma::push::HighlightTweakValue::No)));789
+ let direct = value["content"]["m.mentions"]["user_ids"]790
+ .as_array()791
+ .is_some_and(|users| {792
+ users793
+ .iter()794
+ .any(|user| user.as_str() == Some(self.account.as_str()))795
+ });796
+ let mut reasons = Vec::new();797
+ if direct {798
+ reasons.push(MentionReason::Direct);799
+ }800
+ let mut unresolved = false;801
+ if let Some(reply) = value["content"]["m.relates_to"]["m.in_reply_to"]["event_id"]802
+ .as_str()803
+ .and_then(|value| value.parse().ok())804
+ {805
+ let target = data.events.get(&id(&MentionKey {806
+ room_id: room_id.clone(),807
+ event_id: reply,808
+ }));809
+ let reply_to_self = value["unsigned"]["macltok_reply_sender"].as_str() == Some(self.account.as_str()) || target.is_some_and(|target| matches!(&target.item, TimelineItem::Event(event) if event.sender.user_id == self.account));810
+ if reply_to_self && (direct || (notify && highlight)) {811
+ reasons.push(MentionReason::Reply);812
+ } else if target.is_none()813
+ && value["unsigned"]["macltok_reply_sender"].as_str().is_none()814
+ && (direct || (notify && highlight))815
+ {816
+ unresolved = true;817
+ }818
+ }819
+ if value["content"]["m.mentions"]["room"] == true {820
+ match room.power_levels().await {821
+ Ok(levels) if levels.user_can_trigger_room_notification(&sender) => {822
+ if notify && highlight {823
+ reasons.push(MentionReason::Room);824
+ } else if actions.is_none() {825
+ unresolved = true;826
+ }827
+ }828
+ Err(_) => unresolved = true,829
+ _ => {}830
+ }831
+ }832
+ (reasons, !notify, unresolved)833
+ }834
+835
+ pub(crate) async fn run(mut self) {836
+ let reports_client = self.client.clone();837
+ let reports = reports_client838
+ .event_cache()839
+ .subscribe_to_decryption_reports();840
+ futures_util::pin_mut!(reports);841
+ let mut ignores = self.client.subscribe_to_ignore_user_list_changes();842
+ let mut work: FuturesUnordered<EngineBoxFuture<'static, Work>> = FuturesUnordered::new();843
+ let mut captures = BTreeSet::new();844
+ let mut retries = BTreeSet::new();845
+ let mut history = false;846
+ let mut next_scan = now() + 60_000;847
+ let mut tick = Box::pin(matrix_sdk::sleep::sleep(Duration::from_secs(1)));848
+ loop {849
+ self.schedule(&mut work, &mut captures, &mut retries, &mut history);850
+ tokio::select! {851
+ biased;852
+ request = self.requests.recv() => match request {853
+ Some(Request::Stop(reply)) => { let _ = reply.send(()); return; }854
+ Some(Request::Batch(batch, reply)) => { let result = self.batch(batch).await; let _ = reply.send(result); }855
+ Some(Request::Command(command, reply)) => { let result = self.command(command).await; let _ = reply.send(result); }856
+ None => return,857
+ },858
+ Some(result) = work.next(), if !work.is_empty() => {859
+ let mut next = self.data.clone();860
+ let recovery = (self.latest_scan, self.recovery_budget);861
+ match result {862
+ Work::Capture(key, revision, result) => {863
+ captures.remove(&key);864
+ let observed = result.as_ref().ok().map(|(_, _, observed)| observed.clone()).unwrap_or_default();865
+ let room_id = next.entries.get(&key).filter(|entry| entry.revision == revision)866
+ .map(|entry| entry.key.room_id.clone())867
+ .filter(|room_id| self.client.get_room(room_id).is_some_and(|room| room.state() == RoomState::Joined));868
+ self.captured(&mut next, key, revision, result);869
+ if let Some(room_id) = room_id {870
+ for value in observed {871
+ let identity = value["event_id"].as_str().map(|event_id| format!("{room_id}\n{event_id}"));872
+ if identity.as_ref().is_some_and(|key| next.events.get(key).is_some_and(|event| event.raw.is_some())) { continue; }873
+ self.observe(&mut next, &room_id, value).await.ok();874
+ }875
+ }876
+ }877
+ Work::Candidate(key, result) => {878
+ retries.remove(&key);879
+ if let Some(mut candidate) = next.candidates.remove(&key) {880
+ match result {881
+ Ok(value) => {882
+ if let Err(error) = self.observe(&mut next, &candidate.room_id, value).await { tracing::warn!(%error, "mention retry failed"); }883
+ if let Some(unresolved) = next.candidates.get_mut(&key) { unresolved.attempts = candidate.attempts.saturating_add(1); unresolved.retry_at_ms = now() + (1000u64 << unresolved.attempts.min(8)); }884
+ }885
+ Err(error) => {886
+ if candidate.raw["type"] != "m.room.encrypted" && (matches!(error.kind, ErrorKind::EventNotFound | ErrorKind::Forbidden) || error.server.as_ref().is_some_and(|server| matches!(server.status, 403 | 404))) {887
+ Self::gap(&mut next, Some(candidate.room_id), "mention notification context is unavailable", None);888
+ } else { candidate.attempts = candidate.attempts.saturating_add(1); candidate.retry_at_ms = retry_at(&error, 1000u64 << candidate.attempts.min(8)); next.candidates.insert(key, candidate); }889
+ }890
+ }891
+ }892
+ }893
+ Work::History(latest, from, result) => {894
+ history = false; self.recovery_budget = self.recovery_budget.saturating_sub(1);895
+ match result {896
+ Ok(page) => {897
+ next.coverage.historical_supported = Some(true);898
+ let mut reached_anchor = false;899
+ for notification in page.notifications {900
+ if let Some(event_id) = notification.event.get_field::<String>("event_id").ok().flatten() {901
+ let identity = format!("{}\n{}", notification.room_id, event_id);902
+ if latest && from.is_none() && next.latest_head.is_none() { next.latest_head = Some(identity.clone()); }903
+ if latest && next.latest_anchor.as_ref() == Some(&identity) { reached_anchor = true; break; }904
+ }905
+ if self.client.get_room(¬ification.room_id).is_some_and(|room| room.state() == RoomState::Joined) {906
+ if let Ok(value) = serde_json::from_str(notification.event.json().get()) { let _ = self.observe(&mut next, ¬ification.room_id, value).await; }907
+ }}908
+ let token = page.next_token.filter(|token| Some(token) != from.as_ref() && !reached_anchor);909
+ if latest { next.latest_cursor = token.clone(); if token.is_none() { self.latest_scan = false; next.latest_anchor = next.latest_head.take().or(next.latest_anchor); } }910
+ else { next.history_cursor = token.clone(); next.history_exhausted = token.is_none(); }911
+ }912
+ Err(error) => {913
+ if error.server.as_ref().is_some_and(|server| matches!(server.status, 404 | 405 | 501)) { next.coverage.historical_supported = Some(false); self.recovery_budget = 0; }914
+ if error.server.as_ref().is_some_and(|server| server.status == 400) {915
+ if latest { next.latest_cursor = None; next.latest_head = None; } else { next.history_cursor = None; }916
+ }917
+ Self::gap(&mut next, None, "notification history could not be recovered", from);918
+ self.recovery_retry = retry_at(&error, 60_000);919
+ }920
+ }921
+ next.coverage.recovering = self.recovery_budget > 0 && (self.latest_scan || !next.history_exhausted) && next.coverage.historical_supported != Some(false);922
+ }923
+ }924
+ if let Err(error) = self.save(next).await { tracing::error!(%error, "saving mention background work failed"); self.latest_scan = recovery.0; self.recovery_budget = recovery.1; self.recovery_retry = now() + 60_000; }925
+ }926
+ Some(_) = reports.next() => { for candidate in self.data.candidates.values_mut() { candidate.retry_at_ms = 0; } }927
+ Some(_) = ignores.next() => {928
+ let mut next = self.data.clone(); self.filter_ignored(&mut next).await;929
+ if let Err(error) = self.save(next).await { tracing::error!(%error, "saving ignored mention removals failed"); }930
+ }931
+ () = &mut tick => {932
+ tick = Box::pin(matrix_sdk::sleep::sleep(Duration::from_secs(1)));933
+ if now() >= next_scan && !history { self.recovery_budget = 20; self.latest_scan = true; next_scan = now() + 60_000; }934
+ }935
+ }936
+ }937
+ }938
+939
+ fn schedule(940
+ &mut self,941
+ work: &mut FuturesUnordered<EngineBoxFuture<'static, Work>>,942
+ captures: &mut BTreeSet<String>,943
+ retries: &mut BTreeSet<String>,944
+ history: &mut bool,945
+ ) {946
+ for (key, entry) in self.data.entries.iter().filter(|(_, entry)| matches!(entry.capture, MentionCaptureState::Pending) || matches!(entry.capture, MentionCaptureState::Failed {retry_at_ms, ..} if retry_at_ms <= now())).take(128) {947
+ if captures.len() >= 2 { break; }948
+ if captures.contains(key) { continue; }949
+ let Some(room) = self.client.get_room(&entry.key.room_id).filter(|room| room.state() == RoomState::Joined) else { continue; };950
+ captures.insert(key.clone());951
+ let key = key.clone(); let event_id = entry.key.event_id.clone(); let account = self.account.clone(); let revision = entry.revision;952
+ work.push(Box::pin(async move { Work::Capture(key, revision, context::capture(room, Some(account), event_id, 10, 10).await) }));953
+ }954
+ for (key, candidate) in &self.data.candidates {955
+ if retries.len() >= 2 {956
+ break;957
+ }958
+ if candidate.retry_at_ms > now() || retries.contains(key) {959
+ continue;960
+ }961
+ let Some(room) = self962
+ .client963
+ .get_room(&candidate.room_id)964
+ .filter(|room| room.state() == RoomState::Joined)965
+ else {966
+ continue;967
+ };968
+ let key = key.clone();969
+ let candidate = candidate.clone();970
+ let key_for_work = key.clone();971
+ retries.insert(key);972
+ work.push(Box::pin(async move {973
+ let result = async {974
+ if candidate.raw["type"] == "m.room.encrypted" {975
+ let raw: Raw<OriginalSyncRoomEncryptedEvent> =976
+ Raw::from_json_string(candidate.raw.to_string()).map_err(|error| {977
+ Error::with_source(978
+ ErrorKind::Invalid,979
+ "invalid encrypted mention candidate",980
+ error,981
+ )982
+ })?;983
+ let event = room.decrypt_event(&raw, None).await.map_err(|error| {984
+ Error::from_sdk("decrypting a mention candidate failed", error)985
+ })?;986
+ let value: Value =987
+ serde_json::from_str(event.raw().json().get()).map_err(|error| {988
+ Error::with_source(989
+ ErrorKind::Invalid,990
+ "invalid decrypted mention",991
+ error,992
+ )993
+ })?;994
+ if value["type"] == "m.room.encrypted" {995
+ return Err(Error::new(996
+ ErrorKind::Encryption,997
+ "mention candidate still awaits keys",998
+ ));999
+ }1000
+ return Ok(value);1001
+ }1002
+ let target = candidate.raw["content"]["m.relates_to"]["event_id"]1003
+ .as_str()1004
+ .filter(|_| {1005
+ candidate.raw["content"]["m.relates_to"]["rel_type"] == "m.replace"1006
+ });1007
+ if let Some(target) =1008
+ target.and_then(|target| target.parse::<ruma::OwnedEventId>().ok())1009
+ {1010
+ let event = room.event(&target, None).await.map_err(|error| {1011
+ Error::from_sdk("fetching an edited mention failed", error)1012
+ })?;1013
+ let original: Value = serde_json::from_str(event.raw().json().get())1014
+ .map_err(|error| {1015
+ Error::with_source(1016
+ ErrorKind::Invalid,1017
+ "invalid edited mention",1018
+ error,1019
+ )1020
+ })?;1021
+ if original["sender"] == candidate.raw["sender"] {1022
+ let mut original = edited(original, &candidate.raw);1023
+ original["unsigned"]["macltok_edit"] = candidate.raw;1024
+ return Ok(original);1025
+ }1026
+ return Err(Error::new(1027
+ ErrorKind::Forbidden,1028
+ "an edit belongs to a different sender",1029
+ ));1030
+ }1031
+ let mut value = candidate.raw;1032
+ if let Some(target) =1033
+ value["content"]["m.relates_to"]["m.in_reply_to"]["event_id"]1034
+ .as_str()1035
+ .and_then(|target| target.parse::<ruma::OwnedEventId>().ok())1036
+ {1037
+ let event = room.event(&target, None).await.map_err(|error| {1038
+ Error::from_sdk("fetching a mention reply target failed", error)1039
+ })?;1040
+ let target: Value = serde_json::from_str(event.raw().json().get())1041
+ .map_err(|error| {1042
+ Error::with_source(1043
+ ErrorKind::Invalid,1044
+ "invalid mention reply target",1045
+ error,1046
+ )1047
+ })?;1048
+ value["unsigned"]["macltok_reply_sender"] = target["sender"].clone();1049
+ }1050
+ Ok(value)1051
+ }1052
+ .await;1053
+ Work::Candidate(key_for_work, result)1054
+ }));1055
+ }1056
+ if !*history1057
+ && self.recovery_budget > 01058
+ && now() >= self.recovery_retry1059
+ && (self.latest_scan || !self.data.history_exhausted)1060
+ && self.data.coverage.historical_supported != Some(false)1061
+ {1062
+ *history = true;1063
+ let latest = self.latest_scan;1064
+ let from = if latest {1065
+ self.data.latest_cursor.clone()1066
+ } else {1067
+ self.data.history_cursor.clone()1068
+ };1069
+ let client = self.client.clone();1070
+ work.push(Box::pin(async move {1071
+ let mut request = ruma::api::client::push::get_notifications::v3::Request::new();1072
+ request.from = from.clone();1073
+ request.limit = Some(ruma::uint!(50));1074
+ let result = client.send(request).await.map_err(|error| {1075
+ Error::from_sdk("fetching mention history failed", error.into())1076
+ });1077
+ Work::History(latest, from, result)1078
+ }));1079
+ }1080
+ }1081
+1082
+ fn captured(1083
+ &self,1084
+ data: &mut Data,1085
+ key: String,1086
+ revision: u64,1087
+ result: Result<(crate::EventContext, Vec<MediaRegistration>, Vec<Value>), Error>,1088
+ ) {1089
+ let Some(entry) = data.entries.get(&key) else {1090
+ return;1091
+ };1092
+ if entry.revision != revision {1093
+ return;1094
+ }1095
+ let room_id = entry.key.room_id.clone();1096
+ if !self1097
+ .client1098
+ .get_room(&room_id)1099
+ .is_some_and(|room| room.state() == RoomState::Joined)1100
+ {1101
+ return;1102
+ }1103
+ match result {1104
+ Ok((context, media, _)) => {1105
+ data.capture_attempts.remove(&key);1106
+ let live_after = data1107
+ .windows1108
+ .get(&key)1109
+ .map(|window| window.after.clone())1110
+ .unwrap_or_default();1111
+ let mut insert = |item: TimelineItem| -> Option<String> {1112
+ let TimelineItem::Event(event) = &item else {1113
+ return None;1114
+ };1115
+ let event_id = event.event_id.clone()?;1116
+ let event_key = id(&MentionKey {1117
+ room_id: room_id.clone(),1118
+ event_id,1119
+ });1120
+ let registrations = media_for_item(&item, &media);1121
+ let mut saved = SavedEvent {1122
+ item,1123
+ media: registrations,1124
+ raw: None,1125
+ };1126
+ if let Some(existing) = data.events.get(&event_key) {1127
+ saved = existing.clone();1128
+ }1129
+ if data.redacted.contains_key(&event_key) {1130
+ redact(&mut saved);1131
+ }1132
+ data.events.insert(event_key.clone(), saved);1133
+ Some(event_key)1134
+ };1135
+ let before: Vec<_> = context1136
+ .events_before1137
+ .into_iter()1138
+ .filter_map(&mut insert)1139
+ .collect();1140
+ let target = context.event.and_then(&mut insert);1141
+ let mut after: Vec<_> = context1142
+ .events_after1143
+ .into_iter()1144
+ .filter_map(&mut insert)1145
+ .collect();1146
+ let extended = live_after.iter().any(|key| !after.contains(key));1147
+ after.extend(live_after);1148
+ after.sort_by_key(|key| {1149
+ (1150
+ data.events1151
+ .get(key)1152
+ .and_then(|saved| match &saved.item {1153
+ TimelineItem::Event(event) => Some(event.timestamp_ms),1154
+ _ => None,1155
+ })1156
+ .unwrap_or(0),1157
+ key.clone(),1158
+ )1159
+ });1160
+ after.dedup();1161
+ after.truncate(10);1162
+ let entry = data.entries.get_mut(&key).unwrap();1163
+ entry.capture = if target.is_none() {1164
+ MentionCaptureState::Unavailable1165
+ } else if after.len() < 10 || (before.len() < 10 && context.start_token.is_some()) {1166
+ MentionCaptureState::Partial1167
+ } else {1168
+ MentionCaptureState::Ready1169
+ };1170
+ entry.revision = data.revision + 1;1171
+ data.windows.insert(1172
+ key,1173
+ Window {1174
+ before,1175
+ target,1176
+ after,1177
+ start: context.start_token,1178
+ end: if extended { None } else { context.end_token },1179
+ },1180
+ );1181
+ }1182
+ Err(error) => {1183
+ let attempts = data.capture_attempts.entry(key.clone()).or_default();1184
+ *attempts = attempts.saturating_add(1);1185
+ let delay = (1000u64 << (*attempts).min(11)).min(3_600_000);1186
+ let entry = data.entries.get_mut(&key).unwrap();1187
+ entry.capture = if matches!(1188
+ error.kind,1189
+ ErrorKind::EventNotFound | ErrorKind::Forbidden | ErrorKind::RoomUnknown1190
+ ) || error1191
+ .server1192
+ .as_ref()1193
+ .is_some_and(|server| matches!(server.status, 403 | 404))1194
+ {1195
+ MentionCaptureState::Unavailable1196
+ } else {1197
+ MentionCaptureState::Failed {1198
+ message: error.to_string(),1199
+ retry_at_ms: retry_at(&error, delay),1200
+ }1201
+ };1202
+ entry.revision = data.revision + 1;1203
+ }1204
+ }1205
+ }1206
+}1207
+1208
+fn edited(mut original: Value, edit: &Value) -> Value {1209
+ original["content"] = edit["content"]["m.new_content"].clone();1210
+ original["unsigned"]["macltok_edited"] = Value::Bool(true);1211
+ original1212
+}1213
+1214
+fn retry_at(error: &Error, fallback_ms: u64) -> u64 {1215
+ let mut source = std::error::Error::source(error);1216
+ while let Some(current) = source {1217
+ if let Some(sdk) = current.downcast_ref::<matrix_sdk::Error>() {1218
+ if let Some(ruma::api::error::ErrorKind::LimitExceeded(data)) =1219
+ sdk.client_api_error_kind()1220
+ {1221
+ if let Some(delay) = data.retry_after {1222
+ let delay = match delay {1223
+ ruma::api::error::RetryAfter::Delay(delay) => delay,1224
+ ruma::api::error::RetryAfter::DateTime(time) => time1225
+ .duration_since(ruma::time::SystemTime::now())1226
+ .unwrap_or_default(),1227
+ };1228
+ return now().saturating_add(1229
+ u64::try_from(delay.as_millis())1230
+ .unwrap_or(u64::MAX)1231
+ .max(fallback_ms),1232
+ );1233
+ }1234
+ }1235
+ }1236
+ source = current.source();1237
+ }1238
+ now().saturating_add(fallback_ms)1239
+}1240
+1241
+fn redact(event: &mut SavedEvent) {1242
+ if let TimelineItem::Event(item) = &mut event.item {1243
+ item.content = TimelineContent::Redacted;1244
+ item.reply_to = None;1245
+ item.reactions.clear();1246
+ }1247
+ event.media.clear();1248
+ event.raw = None;1249
+}1250
+1251
+fn media_for_item(1252
+ item: &TimelineItem,1253
+ registrations: &[MediaRegistration],1254
+) -> Vec<MediaRegistration> {1255
+ let TimelineItem::Event(event) = item else {1256
+ return Vec::new();1257
+ };1258
+ let mut ids = Vec::new();1259
+ match &event.content {1260
+ TimelineContent::Message(message) => {1261
+ ids.extend(message.media.iter().map(|media| &media.id));1262
+ ids.extend(1263
+ message1264
+ .link_previews1265
+ .iter()1266
+ .filter_map(|preview| preview.image.as_ref().map(|image| &image.media_id)),1267
+ );1268
+ }1269
+ TimelineContent::Sticker { media, .. } => ids.push(&media.id),1270
+ _ => {}1271
+ }1272
+ registrations1273
+ .iter()1274
+ .filter(|registration| ids.contains(&®istration.id))1275
+ .cloned()1276
+ .collect()1277
+}1278
+1279
+pub(crate) async fn watch_updates(mut updates: watch::Receiver<MentionsPage>, events: EventSender) {1280
+ let mut previous: BTreeMap<String, MentionEntry> = BTreeMap::new();1281
+ loop {1282
+ let snapshot = updates.borrow_and_update().clone();1283
+ let current: BTreeMap<_, _> = snapshot1284
+ .entries1285
+ .iter()1286
+ .map(|entry| (id(&entry.key), entry.clone()))1287
+ .collect();1288
+ let upserts = current1289
+ .iter()1290
+ .filter(|(key, entry)| previous.get(*key) != Some(entry))1291
+ .map(|(_, entry)| entry.clone())1292
+ .collect();1293
+ let removed = previous1294
+ .iter()1295
+ .filter(|(key, _)| !current.contains_key(*key))1296
+ .map(|(_, entry)| entry.key.clone())1297
+ .collect();1298
+ previous = current;1299
+ if events1300
+ .send(Event::MentionsChanged(MentionsUpdate {1301
+ snapshot,1302
+ upserts,1303
+ removed,1304
+ }))1305
+ .await1306
+ .is_err()1307
+ {1308
+ return;1309
+ }1310
+ if updates.changed().await.is_err() {1311
+ return;1312
+ }1313
+ }1314
+}@@ -0,0 +1,454 @@
1
+use std::{2
+ sync::{3
+ Arc,4
+ atomic::{AtomicUsize, Ordering},5
+ },6
+ time::Duration,7
+};8
+9
+use matrix_sdk::ruma::events::room::MediaSource;10
+use matrix_sdk::{11
+ ruma::{event_id, events::Mentions, push::Ruleset, room_id, user_id},12
+ test_utils::mocks::MatrixMockServer,13
+};14
+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};15
+use rusqlite::Connection;16
+use serde_json::json;17
+use wiremock::{18
+ Mock, ResponseTemplate,19
+ matchers::{method, path, query_param, query_param_is_missing},20
+};21
+22
+use super::super::store::{Account, MentionsStore};23
+use super::*;24
+use crate::{25
+ MediaId, MediaKind, StoreConfig, TimelineContent, TimelineItem,26
+ normalise::{MediaRegistration, MediaSourceInfo},27
+};28
+29
+static STORES: AtomicUsize = AtomicUsize::new(0);30
+const MAX_BYTES: usize = 64 * 1024 * 1024;31
+32
+fn config() -> StoreConfig {33
+ let number = STORES.fetch_add(1, Ordering::Relaxed);34
+ StoreConfig::sqlite(35
+ std::env::temp_dir().join(format!(36
+ "macltok-mentions-service-{}-{number}",37
+ std::process::id()38
+ )),39
+ "service commit test passphrase",40
+ )41
+}42
+43
+fn account(client: &Client) -> Account {44
+ Account {45
+ homeserver: client.homeserver().to_string(),46
+ user_id: client.user_id().unwrap().to_owned(),47
+ }48
+}49
+50
+async fn joined_client(51
+ server: &MatrixMockServer,52
+ room_id: &ruma::RoomId,53
+) -> (Client, ruma::OwnedUserId) {54
+ let client = server.client_builder().build().await;55
+ let own_user = client.user_id().unwrap().to_owned();56
+ let room_id = room_id.to_owned();57
+ server58
+ .mock_sync()59
+ .ok_and_run(&client, {60
+ let own_user = own_user.clone();61
+ move |builder| {62
+ let factory = EventFactory::new().room(&room_id);63
+ builder.add_global_account_data(64
+ factory.push_rules(Ruleset::server_default(&own_user)),65
+ );66
+ builder.add_joined_room(67
+ JoinedRoomBuilder::new(&room_id).add_state_event(factory.member(&own_user)),68
+ );69
+ }70
+ })71
+ .await;72
+ (client, own_user)73
+}74
+75
+#[tokio::test]76
+async fn failed_batch_keeps_inbox_and_checkpoint_at_the_same_commit_boundary() {77
+ let server = MatrixMockServer::new().await;78
+ let room_id = room_id!("!mentions:localhost");79
+ let (client, own_user) = joined_client(&server, room_id).await;80
+ let event = EventFactory::new()81
+ .room(room_id)82
+ .text_msg("commit boundary")83
+ .mentions(Mentions::with_user_ids([own_user.clone()]))84
+ .event_id(event_id!("$commit-boundary:localhost"))85
+ .sender(user_id!("@bob:localhost"))86
+ .into_raw_sync();87
+ let raw_room = serde_json::from_value(json!({88
+ "initial": true,89
+ "timeline": [serde_json::to_value(event).unwrap()],90
+ }))91
+ .unwrap();92
+ let batch = ResponseCommitBatch {93
+ connection_id: "mentions-connection".to_owned(),94
+ previous_pos: None,95
+ next_pos: "mentions-position-1".to_owned(),96
+ response: Default::default(),97
+ raw_rooms: [(room_id.to_owned(), raw_room)].into(),98
+ };99
+ let config = config();100
+ let store = MentionsStore::open(&config, account(&client), MAX_BYTES)101
+ .await102
+ .unwrap();103
+ let media = Arc::new(MediaService::new(104
+ client.clone(),105
+ EngineConfig::default().max_media_bytes,106
+ EngineConfig::default().max_media_sources,107
+ ));108
+ let (_handle, mut service, updates) =109
+ Service::new(client, media, Some(store), EngineConfig::default())110
+ .await111
+ .unwrap();112
+113
+ let connection = Connection::open(config.mentions_location()).unwrap();114
+ connection115
+ .execute_batch(116
+ "CREATE TRIGGER fail_mentions_inbox_insert BEFORE INSERT ON records WHEN NEW.key NOT IN ('!cipher', '!head') BEGIN SELECT RAISE(ABORT, 'injected inbox write failure'); END;117
+ CREATE TRIGGER fail_mentions_inbox_update BEFORE UPDATE ON records WHEN OLD.key NOT IN ('!cipher', '!head') BEGIN SELECT RAISE(ABORT, 'injected inbox write failure'); END;",118
+ )119
+ .unwrap();120
+121
+ assert!(service.batch(batch.clone()).await.is_err());122
+ assert_eq!(service.data.revision, 0);123
+ assert!(service.data.entries.is_empty());124
+ assert!(service.data.checkpoints.is_empty());125
+ assert_eq!(updates.borrow().revision, 0);126
+ assert!(updates.borrow().entries.is_empty());127
+128
+ let reopened = MentionsStore::open(&config, account(&service.client), MAX_BYTES)129
+ .await130
+ .unwrap();131
+ assert_eq!(reopened.revision(), 0);132
+ assert!(reopened.read::<Data>("inbox").unwrap().is_none());133
+ drop(reopened);134
+135
+ connection136
+ .execute_batch(137
+ "DROP TRIGGER fail_mentions_inbox_insert;138
+ DROP TRIGGER fail_mentions_inbox_update;",139
+ )140
+ .unwrap();141
+ drop(connection);142
+143
+ service.batch(batch.clone()).await.unwrap();144
+ assert_eq!(service.data.revision, 1);145
+ assert_eq!(service.data.entries.len(), 1);146
+ assert_eq!(147
+ service.data.checkpoints.get("mentions-connection"),148
+ Some(&batch.next_pos)149
+ );150
+ assert_eq!(updates.borrow().revision, 1);151
+ assert_eq!(updates.borrow().entries.len(), 1);152
+153
+ let reopened = MentionsStore::open(&config, account(&service.client), MAX_BYTES)154
+ .await155
+ .unwrap();156
+ let persisted: Data = reopened.read("inbox").unwrap().unwrap();157
+ assert_eq!(reopened.revision(), 1);158
+ assert_eq!(persisted.revision, 1);159
+ assert_eq!(persisted.entries.len(), 1);160
+ assert_eq!(161
+ persisted.checkpoints.get("mentions-connection"),162
+ Some(&batch.next_pos)163
+ );164
+ drop(reopened);165
+166
+ service.batch(batch).await.unwrap();167
+ assert_eq!(service.data.revision, 1);168
+ assert_eq!(service.store.as_ref().unwrap().revision(), 1);169
+170
+ drop(service);171
+ config.delete().await.unwrap();172
+}173
+174
+#[tokio::test]175
+async fn service_recovers_notification_history_into_durable_unread_inbox() {176
+ let server = MatrixMockServer::new().await;177
+ let room_id = room_id!("!mentions:localhost");178
+ let (client, own_user) = joined_client(&server, room_id).await;179
+ let first = EventFactory::new()180
+ .room(room_id)181
+ .text_msg("history mention one")182
+ .mentions(Mentions::with_user_ids([own_user.clone()]))183
+ .event_id(event_id!("$history-one:localhost"))184
+ .sender(user_id!("@bob:localhost"))185
+ .into_raw_sync();186
+ let second = EventFactory::new()187
+ .room(room_id)188
+ .text_msg("history mention two")189
+ .mentions(Mentions::with_user_ids([own_user]))190
+ .event_id(event_id!("$history-two:localhost"))191
+ .sender(user_id!("@bob:localhost"))192
+ .into_raw_sync();193
+ let first = serde_json::from_str::<serde_json::Value>(first.json().get()).unwrap();194
+ let second = serde_json::from_str::<serde_json::Value>(second.json().get()).unwrap();195
+ Mock::given(method("GET"))196
+ .and(path("/_matrix/client/v3/notifications"))197
+ .and(query_param_is_missing("from"))198
+ .and(query_param("limit", "50"))199
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({200
+ "notifications": [{201
+ "actions": ["notify"],202
+ "event": first,203
+ "read": true,204
+ "room_id": room_id.as_str(),205
+ "ts": 1,206
+ }],207
+ "next_token": "history-page-two",208
+ })))209
+ .mount(server.server())210
+ .await;211
+ Mock::given(method("GET"))212
+ .and(path("/_matrix/client/v3/notifications"))213
+ .and(query_param("from", "history-page-two"))214
+ .and(query_param("limit", "50"))215
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({216
+ "notifications": [{217
+ "actions": ["notify"],218
+ "event": second,219
+ "read": true,220
+ "room_id": room_id.as_str(),221
+ "ts": 2,222
+ }],223
+ })))224
+ .mount(server.server())225
+ .await;226
+227
+ let config = config();228
+ let store = MentionsStore::open(&config, account(&client), MAX_BYTES)229
+ .await230
+ .unwrap();231
+ let media = Arc::new(MediaService::new(232
+ client.clone(),233
+ EngineConfig::default().max_media_bytes,234
+ EngineConfig::default().max_media_sources,235
+ ));236
+ let (handle, service, mut updates) =237
+ Service::new(client.clone(), media, Some(store), EngineConfig::default())238
+ .await239
+ .unwrap();240
+ let service_task = tokio::spawn(service.run());241
+242
+ let discovered = tokio::time::timeout(Duration::from_secs(10), async {243
+ loop {244
+ let page = updates.borrow_and_update().clone();245
+ if page.entries.len() == 2 && !page.coverage.recovering {246
+ break page;247
+ }248
+ updates.changed().await.unwrap();249
+ }250
+ })251
+ .await;252
+ handle.stop().await;253
+ service_task.await.unwrap();254
+ let discovered = discovered.expect("notification history should be recovered");255
+ assert_eq!(discovered.entries.len(), 2);256
+ assert!(discovered.entries.iter().all(|entry| !entry.read));257
+258
+ let reopened = MentionsStore::open(&config, account(&client), MAX_BYTES)259
+ .await260
+ .unwrap();261
+ let persisted: Data = reopened.read("inbox").unwrap().unwrap();262
+ assert_eq!(reopened.revision(), persisted.revision);263
+ assert_eq!(persisted.entries.len(), 2);264
+ assert!(persisted.entries.values().all(|entry| !entry.read));265
+ assert!(persisted.latest_cursor.is_none());266
+ assert!(persisted.history_cursor.is_none());267
+ drop(reopened);268
+269
+ config.delete().await.unwrap();270
+}271
+272
+fn message_body(item: &TimelineItem) -> Option<&str> {273
+ match item {274
+ TimelineItem::Event(event) => match &event.content {275
+ TimelineContent::Message(message) => Some(&message.body),276
+ _ => None,277
+ },278
+ _ => None,279
+ }280
+}281
+282
+fn media_registration() -> MediaRegistration {283
+ MediaRegistration {284
+ id: MediaId::new("test-scope#image".to_owned()),285
+ kind: MediaKind::Image,286
+ full: MediaSourceInfo {287
+ source: MediaSource::Plain(ruma::mxc_uri!("mxc://mentions.example/image").to_owned()),288
+ mimetype: Some("image/png".to_owned()),289
+ declared_size: Some(1234),290
+ },291
+ thumbnail: None,292
+ }293
+}294
+295
+#[tokio::test]296
+async fn live_context_survives_capture_then_edit_and_redaction() {297
+ let server = MatrixMockServer::new().await;298
+ let room_id = room_id!("!mentions:localhost");299
+ let (client, own_user) = joined_client(&server, room_id).await;300
+ let mention = EventFactory::new()301
+ .room(room_id)302
+ .text_msg("direct mention")303
+ .mentions(Mentions::with_user_ids([own_user]))304
+ .event_id(event_id!("$live-mention:localhost"))305
+ .sender(user_id!("@bob:localhost"))306
+ .into_raw_sync();307
+ let mut mention: serde_json::Value = serde_json::from_str(mention.json().get()).unwrap();308
+ mention["origin_server_ts"] = json!(10);309
+ let following = EventFactory::new()310
+ .room(room_id)311
+ .text_msg("following")312
+ .event_id(event_id!("$live-following:localhost"))313
+ .sender(user_id!("@bob:localhost"))314
+ .into_raw_sync();315
+ let mut following: serde_json::Value = serde_json::from_str(following.json().get()).unwrap();316
+ following["origin_server_ts"] = json!(11);317
+318
+ let media = Arc::new(MediaService::new(319
+ client.clone(),320
+ EngineConfig::default().max_media_bytes,321
+ EngineConfig::default().max_media_sources,322
+ ));323
+ let (_handle, mut service, _updates) =324
+ Service::new(client, media, None, EngineConfig::default())325
+ .await326
+ .unwrap();327
+ let room_id = room_id.to_owned();328
+ let mention_key = MentionKey {329
+ room_id: room_id.clone(),330
+ event_id: event_id!("$live-mention:localhost").to_owned(),331
+ };332
+ let following_id = format!("{room_id}\n$live-following:localhost");333
+ let mut data = Data::default();334
+335
+ service.observe(&mut data, &room_id, mention).await.unwrap();336
+ service337
+ .observe(&mut data, &room_id, following)338
+ .await339
+ .unwrap();340
+341
+ let mention_identity = id(&mention_key);342
+ let entry = data.entries.get(&mention_identity).unwrap();343
+ let revision = entry.revision;344
+ let target = entry.event.clone();345
+ service.captured(346
+ &mut data,347
+ mention_identity.clone(),348
+ revision,349
+ Ok((350
+ crate::EventContext {351
+ events_before: Vec::new(),352
+ event: Some(target),353
+ events_after: Vec::new(),354
+ start_token: None,355
+ end_token: None,356
+ },357
+ Vec::new(),358
+ Vec::new(),359
+ )),360
+ );361
+362
+ assert_eq!(363
+ data.entries[&mention_identity].capture,364
+ MentionCaptureState::Partial365
+ );366
+ assert!(data.events.contains_key(&following_id));367
+ let (saved, _) = data.context(&mention_key).unwrap();368
+ let context = saved.context.unwrap();369
+ assert_eq!(context.events_after.len(), 1);370
+ assert_eq!(message_body(&context.events_after[0]), Some("following"));371
+372
+ let edit = json!({373
+ "content": {374
+ "body": "* edited following",375
+ "m.new_content": {"body": "edited following", "msgtype": "m.text"},376
+ "m.relates_to": {"event_id": "$live-following:localhost", "rel_type": "m.replace"},377
+ "msgtype": "m.text"378
+ },379
+ "event_id": "$live-following-edit:localhost",380
+ "origin_server_ts": 12,381
+ "room_id": room_id.as_str(),382
+ "sender": "@bob:localhost",383
+ "type": "m.room.message"384
+ });385
+ service.observe(&mut data, &room_id, edit).await.unwrap();386
+ let (saved, _) = data.context(&mention_key).unwrap();387
+ let context = saved.context.unwrap();388
+ assert_eq!(389
+ message_body(&context.events_after[0]),390
+ Some("edited following")391
+ );392
+393
+ let saved_following = data.events.get_mut(&following_id).unwrap();394
+ assert!(saved_following.raw.is_some());395
+ saved_following.media.push(media_registration());396
+ assert_eq!(saved_following.media.len(), 1);397
+ let redaction = json!({398
+ "content": {},399
+ "event_id": "$live-following-redaction:localhost",400
+ "origin_server_ts": 13,401
+ "redacts": "$live-following:localhost",402
+ "room_id": room_id.as_str(),403
+ "sender": "@bob:localhost",404
+ "type": "m.room.redaction"405
+ });406
+ service407
+ .observe(&mut data, &room_id, redaction)408
+ .await409
+ .unwrap();410
+411
+ let saved_following = data.events.get(&following_id).unwrap();412
+ assert!(matches!(413
+ &saved_following.item,414
+ TimelineItem::Event(event) if event.content == TimelineContent::Redacted415
+ ));416
+ assert!(saved_following.raw.is_none());417
+ assert!(saved_following.media.is_empty());418
+ let (saved, _) = data.context(&mention_key).unwrap();419
+ let context = saved.context.unwrap();420
+ assert!(matches!(421
+ &context.events_after[0],422
+ TimelineItem::Event(event) if event.content == TimelineContent::Redacted423
+ ));424
+ data.windows.remove(&mention_identity);425
+ service.data = data;426
+ let CommandOutput::MentionContext(context) = service427
+ .command(MentionsCommand::RetryContext(mention_key))428
+ .await429
+ .unwrap()430
+ else {431
+ panic!("retry should return saved context");432
+ };433
+ assert_eq!(context.state, MentionCaptureState::Pending);434
+ assert!(context.context.unwrap().event.is_some());435
+}436
+437
+#[tokio::test]438
+async fn explicit_recovery_rechecks_server_support() {439
+ let server = MatrixMockServer::new().await;440
+ let (client, _) = joined_client(&server, room_id!("!mentions:localhost")).await;441
+ let config = EngineConfig::default();442
+ let media = Arc::new(MediaService::new(443
+ client.clone(),444
+ config.max_media_bytes,445
+ config.max_media_sources,446
+ ));447
+ let (_, mut service, _) = Service::new(client, media, None, config).await.unwrap();448
+ service.data.coverage.historical_supported = Some(false);449
+ service.recovery_budget = 0;450
+ service.command(MentionsCommand::Recover).await.unwrap();451
+ assert_eq!(service.data.coverage.historical_supported, None);452
+ assert!(service.data.coverage.recovering);453
+ assert_eq!(service.recovery_budget, 20);454
+}@@ -0,0 +1,283 @@
1
+use std::collections::{BTreeMap, BTreeSet};2
+3
+use ruma::OwnedUserId;4
+use serde_json::Value;5
+6
+use crate::{7
+ Error, ErrorKind, EventContext, MentionCaptureState, MentionCoverage, MentionEntry, MentionKey,8
+ MentionsBoundary, MentionsPage, SavedMentionContext, TimelineItem,9
+ normalise::MediaRegistration,10
+};11
+12
+#[derive(Clone)]13
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]14
+pub(super) struct SavedEvent {15
+ pub item: TimelineItem,16
+ pub media: Vec<MediaRegistration>,17
+ pub raw: Option<Value>,18
+}19
+20
+#[derive(Clone)]21
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]22
+pub(super) struct Candidate {23
+ pub room_id: ruma::OwnedRoomId,24
+ pub raw: Value,25
+ pub attempts: u32,26
+ pub retry_at_ms: u64,27
+}28
+29
+#[derive(Clone)]30
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]31
+pub(super) struct Window {32
+ pub before: Vec<String>,33
+ pub target: Option<String>,34
+ pub after: Vec<String>,35
+ pub start: Option<String>,36
+ pub end: Option<String>,37
+}38
+39
+#[derive(Clone, Default)]40
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]41
+pub(super) struct Data {42
+ pub revision: u64,43
+ pub entries: BTreeMap<String, MentionEntry>,44
+ pub events: BTreeMap<String, SavedEvent>,45
+ pub windows: BTreeMap<String, Window>,46
+ pub capture_attempts: BTreeMap<String, u32>,47
+ pub candidates: BTreeMap<String, Candidate>,48
+ pub redacted: BTreeMap<String, u64>,49
+ pub edits: BTreeMap<String, Value>,50
+ pub coverage: MentionCoverage,51
+ pub checkpoints: BTreeMap<String, String>,52
+ pub observed_rooms: BTreeSet<ruma::OwnedRoomId>,53
+ pub history_cursor: Option<String>,54
+ pub history_exhausted: bool,55
+ pub latest_cursor: Option<String>,56
+ #[cfg_attr(feature = "serde", serde(default))]57
+ pub latest_anchor: Option<String>,58
+ #[cfg_attr(feature = "serde", serde(default))]59
+ pub latest_head: Option<String>,60
+}61
+62
+pub(super) fn id(key: &MentionKey) -> String {63
+ format!("{}\n{}", key.room_id, key.event_id)64
+}65
+66
+pub(super) fn now() -> u64 {67
+ matrix_sdk::ruma::MilliSecondsSinceUnixEpoch::now()68
+ .get()69
+ .into()70
+}71
+72
+pub(super) fn invalid(message: &'static str) -> Error {73
+ Error::new(ErrorKind::Invalid, message)74
+}75
+76
+impl Data {77
+ pub fn touch_contexts(&mut self, identity: &str) {78
+ for (key, window) in &self.windows {79
+ if window.target.as_deref() == Some(identity)80
+ || window81
+ .before82
+ .iter()83
+ .chain(&window.after)84
+ .any(|key| key == identity)85
+ {86
+ if let Some(entry) = self.entries.get_mut(key) {87
+ entry.revision = self.revision + 1;88
+ }89
+ }90
+ }91
+ }92
+93
+ pub fn page(94
+ &self,95
+ account: &OwnedUserId,96
+ cursor: Option<&str>,97
+ limit: u16,98
+ ) -> Result<MentionsPage, Error> {99
+ let (boundary, last) = if let Some(cursor) = cursor {100
+ let value: Value =101
+ serde_json::from_str(cursor).map_err(|_| invalid("invalid mentions cursor"))?;102
+ if value["account"].as_str() != Some(account.as_str()) {103
+ return Err(invalid("mentions cursor belongs to a different account"));104
+ }105
+ let boundary = value["revision"]106
+ .as_u64()107
+ .ok_or_else(|| invalid("invalid mentions cursor"))?;108
+ let timestamp = value["timestamp"]109
+ .as_u64()110
+ .ok_or_else(|| invalid("invalid mentions cursor"))?;111
+ let key = value["key"]112
+ .as_str()113
+ .ok_or_else(|| invalid("invalid mentions cursor"))?114
+ .to_owned();115
+ if boundary > self.revision {116
+ return Err(invalid("mentions cursor is newer than this store"));117
+ }118
+ (boundary, Some((timestamp, key)))119
+ } else {120
+ (self.revision, None)121
+ };122
+ let mut entries: Vec<_> = self123
+ .entries124
+ .iter()125
+ .filter(|(key, entry)| {126
+ entry.discovery_revision <= boundary127
+ && last.as_ref().is_none_or(|last| {128
+ (entry.timestamp_ms, key.as_str()) < (last.0, last.1.as_str())129
+ })130
+ })131
+ .map(|(_, entry)| entry.clone())132
+ .collect();133
+ entries.sort_by(|a, b| (b.timestamp_ms, id(&b.key)).cmp(&(a.timestamp_ms, id(&a.key))));134
+ let limit = usize::from(limit.clamp(1, 50));135
+ let end_reached = entries.len() <= limit;136
+ entries.truncate(limit);137
+ let next_cursor = if end_reached {138
+ None139
+ } else {140
+ entries.last().map(|entry| serde_json::json!({141
+ "account": account.as_str(), "revision": boundary, "timestamp": entry.timestamp_ms, "key": id(&entry.key)142
+ }).to_string())143
+ };144
+ let mut coverage = self.coverage.clone();145
+ coverage.unresolved_candidates = self.candidates.len();146
+ Ok(MentionsPage {147
+ entries,148
+ next_cursor,149
+ end_reached,150
+ coverage,151
+ unread_count: self.entries.values().filter(|entry| !entry.read).count(),152
+ revision: self.revision,153
+ boundary: MentionsBoundary {154
+ account: account.clone(),155
+ revision: boundary,156
+ },157
+ })158
+ }159
+160
+ pub fn context(161
+ &self,162
+ key: &MentionKey,163
+ ) -> Result<(SavedMentionContext, Vec<MediaRegistration>), Error> {164
+ let entry = self165
+ .entries166
+ .get(&id(key))167
+ .ok_or_else(|| Error::new(ErrorKind::EventNotFound, "mention was not found"))?;168
+ let mut media = Vec::new();169
+ let mut item = |key: &String| {170
+ self.events.get(key).map(|event| {171
+ media.extend(event.media.clone());172
+ event.item.clone()173
+ })174
+ };175
+ let context = self.windows.get(&id(key)).map(|window| EventContext {176
+ events_before: window.before.iter().filter_map(&mut item).collect(),177
+ event: window.target.as_ref().and_then(&mut item),178
+ events_after: window.after.iter().filter_map(&mut item).collect(),179
+ start_token: window.start.clone(),180
+ end_token: window.end.clone(),181
+ });182
+ Ok((183
+ SavedMentionContext {184
+ key: key.clone(),185
+ state: entry.capture.clone(),186
+ context,187
+ revision: self.revision,188
+ },189
+ media,190
+ ))191
+ }192
+193
+ pub fn collect_events(&mut self) {194
+ let mut used: BTreeSet<String> = self.entries.keys().cloned().collect();195
+ for window in self.windows.values() {196
+ used.extend(window.before.iter().chain(&window.after).cloned());197
+ used.extend(window.target.iter().cloned());198
+ }199
+ self.events.retain(|key, _| used.contains(key));200
+ self.capture_attempts201
+ .retain(|key, _| self.entries.contains_key(key));202
+ }203
+204
+ pub fn evict_one(&mut self) -> bool {205
+ let oldest = |keys: Vec<String>, entries: &BTreeMap<String, MentionEntry>| {206
+ keys.into_iter()207
+ .min_by_key(|key| entries.get(key).map_or(0, |entry| entry.timestamp_ms))208
+ };209
+ if let Some(key) = oldest(210
+ self.windows211
+ .keys()212
+ .filter(|key| self.entries.get(*key).is_some_and(|entry| entry.read))213
+ .cloned()214
+ .collect(),215
+ &self.entries,216
+ )217
+ .or_else(|| oldest(self.windows.keys().cloned().collect(), &self.entries))218
+ {219
+ self.windows.remove(&key);220
+ if let Some(entry) = self.entries.get_mut(&key) {221
+ entry.capture = MentionCaptureState::Unavailable;222
+ entry.revision = self.revision;223
+ }224
+ } else if let Some(key) = self225
+ .entries226
+ .iter()227
+ .min_by_key(|(_, entry)| (!entry.read, entry.timestamp_ms))228
+ .map(|(key, _)| key.clone())229
+ {230
+ self.entries.remove(&key);231
+ } else if let Some(key) = self.candidates.keys().next().cloned() {232
+ self.candidates.remove(&key);233
+ } else if let Some(key) = self.edits.keys().next().cloned() {234
+ self.edits.remove(&key);235
+ } else if let Some(key) = self.redacted.keys().next().cloned() {236
+ self.redacted.remove(&key);237
+ } else {238
+ return false;239
+ }240
+ self.coverage.truncated = true;241
+ self.collect_events();242
+ true243
+ }244
+245
+ pub fn bound(&mut self, entries: usize, events: usize, candidates: usize) {246
+ while self.entries.len() > entries {247
+ if let Some(key) = self248
+ .entries249
+ .iter()250
+ .min_by_key(|(_, entry)| (!entry.read, entry.timestamp_ms))251
+ .map(|(key, _)| key.clone())252
+ {253
+ self.entries.remove(&key);254
+ self.windows.remove(&key);255
+ self.coverage.truncated = true;256
+ }257
+ }258
+ self.collect_events();259
+ while self.events.len() > events && self.evict_one() {}260
+ while self.candidates.len() > candidates {261
+ let key = self.candidates.keys().next().cloned().unwrap();262
+ self.candidates.remove(&key);263
+ self.coverage.truncated = true;264
+ }265
+ for map in [&mut self.redacted] {266
+ while map.len() > entries + candidates {267
+ let key = map268
+ .iter()269
+ .min_by_key(|(_, timestamp)| *timestamp)270
+ .map(|(key, _)| key.clone())271
+ .unwrap();272
+ map.remove(&key);273
+ self.coverage.truncated = true;274
+ }275
+ }276
+ while self.edits.len() > candidates {277
+ let key = self.edits.keys().next().cloned().unwrap();278
+ self.edits.remove(&key);279
+ self.coverage.truncated = true;280
+ }281
+ self.coverage.unresolved_candidates = self.candidates.len();282
+ }283
+}@@ -0,0 +1,458 @@
1
+use ruma::{event_id, user_id};2
+3
+use super::state::{Data, SavedEvent, Window, id};4
+use crate::{5
+ MentionCaptureState, MentionEntry, MentionKey, MentionReason, Message, MessageKind, SendState,6
+ Sender, TimelineContent, TimelineEvent, TimelineItem, TimelineItemId,7
+};8
+9
+fn key(room: &str, event: &str) -> MentionKey {10
+ MentionKey {11
+ room_id: format!("!{room}:mentions.example").parse().unwrap(),12
+ event_id: format!("${event}:mentions.example").parse().unwrap(),13
+ }14
+}15
+16
+fn entry(key: MentionKey, timestamp_ms: u64, discovery_revision: u64, read: bool) -> MentionEntry {17
+ MentionEntry {18
+ key,19
+ timestamp_ms,20
+ discovered_at_ms: timestamp_ms,21
+ discovery_revision,22
+ revision: discovery_revision,23
+ event: TimelineItem::DayDivider { timestamp_ms },24
+ reasons: vec![MentionReason::Direct],25
+ read,26
+ capture: MentionCaptureState::Ready,27
+ room_name: None,28
+ notification_suppressed: false,29
+ }30
+}31
+32
+fn message_event(event_id: &str, timestamp_ms: u64, body: &str) -> TimelineItem {33
+ let event_id: ruma::OwnedEventId = event_id.parse().unwrap();34
+ TimelineItem::Event(Box::new(TimelineEvent {35
+ item_id: TimelineItemId::new(format!("item-{event_id}")),36
+ event_id: Some(event_id),37
+ transaction_id: None,38
+ sender: Sender {39
+ user_id: user_id!("@bob:mentions.example").to_owned(),40
+ display_name: Some("Bob".to_owned()),41
+ avatar: None,42
+ name_ambiguous: false,43
+ },44
+ timestamp_ms,45
+ own: false,46
+ send_state: SendState::Sent,47
+ content: TimelineContent::Message(Message {48
+ kind: MessageKind::Text,49
+ body: body.to_owned(),50
+ geo_uri: None,51
+ formatted_body: None,52
+ mentions: None,53
+ edited: false,54
+ media: None,55
+ link_previews: Vec::new(),56
+ }),57
+ reply_to: None,58
+ thread_root: None,59
+ thread_summary: None,60
+ reactions: Vec::new(),61
+ read_by: Vec::new(),62
+ editable: false,63
+ }))64
+}65
+66
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]67
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]68
+async fn pagination_keeps_its_discovery_boundary_and_stable_ties() {69
+ let account = user_id!("@alice:mentions.example").to_owned();70
+ let ordered = vec![71
+ key("room", "latest"),72
+ key("room", "tie-z"),73
+ key("room", "tie-b"),74
+ key("room", "tie-a"),75
+ key("room", "oldest"),76
+ ];77
+ let mut data = Data {78
+ revision: 1,79
+ ..Data::default()80
+ };81
+ for (mention, timestamp) in [82
+ (ordered[0].clone(), 300),83
+ (ordered[1].clone(), 200),84
+ (ordered[2].clone(), 200),85
+ (ordered[3].clone(), 200),86
+ (ordered[4].clone(), 100),87
+ ] {88
+ data.entries89
+ .insert(id(&mention), entry(mention, timestamp, 1, false));90
+ }91
+92
+ let first = data.page(&account, None, 2).unwrap();93
+ assert_eq!(94
+ first95
+ .entries96
+ .iter()97
+ .map(|entry| entry.key.clone())98
+ .collect::<Vec<_>>(),99
+ ordered[..2].to_vec()100
+ );101
+ let mut cursor = first.next_cursor.unwrap();102
+103
+ data.revision = 2;104
+ let later = key("room", "discovered-later");105
+ data.entries.insert(id(&later), entry(later, 50, 2, false));106
+107
+ let mut paged = first108
+ .entries109
+ .into_iter()110
+ .map(|entry| entry.key)111
+ .collect::<Vec<_>>();112
+ loop {113
+ let page = data.page(&account, Some(&cursor), 2).unwrap();114
+ paged.extend(page.entries.into_iter().map(|entry| entry.key));115
+ match page.next_cursor {116
+ Some(next) => cursor = next,117
+ None => break,118
+ }119
+ }120
+ assert_eq!(paged, ordered);121
+}122
+123
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]124
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]125
+async fn pagination_rejects_a_forged_account_and_a_future_revision() {126
+ let account = user_id!("@alice:mentions.example").to_owned();127
+ let mut data = Data {128
+ revision: 1,129
+ ..Data::default()130
+ };131
+ for event in ["one", "two"] {132
+ let mention = key("room", event);133
+ data.entries134
+ .insert(id(&mention), entry(mention, 10, 1, false));135
+ }136
+ let cursor = data.page(&account, None, 1).unwrap().next_cursor.unwrap();137
+138
+ let mut forged_account: serde_json::Value = serde_json::from_str(&cursor).unwrap();139
+ forged_account["account"] = serde_json::Value::String("@mallory:mentions.example".to_owned());140
+ assert!(141
+ data.page(&account, Some(&forged_account.to_string()), 1)142
+ .is_err()143
+ );144
+145
+ let mut future_revision: serde_json::Value = serde_json::from_str(&cursor).unwrap();146
+ future_revision["revision"] = serde_json::json!(data.revision + 1);147
+ assert!(148
+ data.page(&account, Some(&future_revision.to_string()), 1)149
+ .is_err()150
+ );151
+}152
+153
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]154
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]155
+async fn retention_discards_read_context_first_and_keeps_shared_events_until_unused() {156
+ let read_key = key("room", "read-mention");157
+ let unread_key = key("room", "unread-mention");158
+ let shared_key = key("room", "shared-context");159
+ let read_id = id(&read_key);160
+ let unread_id = id(&unread_key);161
+ let shared_id = id(&shared_key);162
+ let mut data = Data {163
+ revision: 3,164
+ ..Data::default()165
+ };166
+ data.entries167
+ .insert(read_id.clone(), entry(read_key, 10, 1, true));168
+ data.entries169
+ .insert(unread_id.clone(), entry(unread_key, 20, 2, false));170
+ data.events.insert(171
+ shared_id.clone(),172
+ SavedEvent {173
+ item: message_event("$shared-context:mentions.example", 15, "shared"),174
+ media: Vec::new(),175
+ raw: None,176
+ },177
+ );178
+ data.windows.insert(179
+ read_id.clone(),180
+ Window {181
+ before: vec![shared_id.clone()],182
+ target: None,183
+ after: Vec::new(),184
+ start: None,185
+ end: None,186
+ },187
+ );188
+ data.windows.insert(189
+ unread_id.clone(),190
+ Window {191
+ before: vec![shared_id.clone()],192
+ target: None,193
+ after: Vec::new(),194
+ start: None,195
+ end: None,196
+ },197
+ );198
+199
+ assert!(data.evict_one());200
+ assert!(!data.windows.contains_key(&read_id));201
+ assert!(data.windows.contains_key(&unread_id));202
+ assert_eq!(203
+ data.entries[&read_id].capture,204
+ MentionCaptureState::Unavailable205
+ );206
+ assert!(!data.entries[&unread_id].read);207
+ assert!(data.events.contains_key(&shared_id));208
+209
+ assert!(data.evict_one());210
+ assert!(data.windows.is_empty());211
+ assert!(data.entries.contains_key(&read_id));212
+ assert!(data.entries.contains_key(&unread_id));213
+ assert!(!data.events.contains_key(&shared_id));214
+}215
+216
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]217
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]218
+async fn context_uses_the_redacted_saved_event_instead_of_the_entry_preview() {219
+ let mention = key("room", "redacted-target");220
+ let mention_id = id(&mention);221
+ let original = message_event(222
+ "$redacted-target:mentions.example",223
+ 42,224
+ "private original body",225
+ );226
+ let mut redacted = original.clone();227
+ if let TimelineItem::Event(event) = &mut redacted {228
+ event.content = TimelineContent::Redacted;229
+ event.reply_to = None;230
+ event.reactions.clear();231
+ }232
+ let mut data = Data::default();233
+ data.entries.insert(234
+ mention_id.clone(),235
+ MentionEntry {236
+ event: original,237
+ ..entry(mention.clone(), 42, 1, false)238
+ },239
+ );240
+ data.events.insert(241
+ mention_id.clone(),242
+ SavedEvent {243
+ item: redacted,244
+ media: Vec::new(),245
+ raw: None,246
+ },247
+ );248
+ data.windows.insert(249
+ mention_id,250
+ Window {251
+ before: Vec::new(),252
+ target: Some(id(&mention)),253
+ after: Vec::new(),254
+ start: Some("start".to_owned()),255
+ end: Some("end".to_owned()),256
+ },257
+ );258
+259
+ let (context, media) = data.context(&mention).unwrap();260
+ let saved = context.context.unwrap().event.unwrap();261
+ let TimelineItem::Event(saved) = saved else {262
+ panic!("the saved target should remain a timeline event");263
+ };264
+ assert_eq!(saved.content, TimelineContent::Redacted);265
+ assert_eq!(266
+ saved.event_id.as_deref(),267
+ Some(event_id!("$redacted-target:mentions.example"))268
+ );269
+ assert!(media.is_empty());270
+}271
+272
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]273
+mod persistence {274
+ use std::sync::atomic::{AtomicUsize, Ordering};275
+276
+ use ruma::{events::room::MediaSource, user_id};277
+ use serde_json::json;278
+279
+ use super::*;280
+ use crate::{281
+ MediaId, MediaKind, StoreConfig,282
+ mentions::store::{Account, MentionsStore, Write},283
+ normalise::{MediaRegistration, MediaSourceInfo},284
+ };285
+286
+ static STORES: AtomicUsize = AtomicUsize::new(0);287
+ const MAX_BYTES: usize = 64 * 1024 * 1024;288
+289
+ fn account() -> Account {290
+ Account {291
+ homeserver: "https://mentions.example/".to_owned(),292
+ user_id: user_id!("@alice:mentions.example").to_owned(),293
+ }294
+ }295
+296
+ fn config() -> StoreConfig {297
+ let number = STORES.fetch_add(1, Ordering::Relaxed);298
+ #[cfg(feature = "sqlite")]299
+ {300
+ StoreConfig::sqlite(301
+ std::env::temp_dir().join(format!(302
+ "macltok-mentions-state-{}-{number}",303
+ std::process::id()304
+ )),305
+ "state test passphrase",306
+ )307
+ }308
+ #[cfg(feature = "indexeddb")]309
+ {310
+ StoreConfig::indexeddb(311
+ format!("macltok-mentions-state-{}-{number}", js_sys::Date::now()),312
+ "state test passphrase",313
+ )314
+ }315
+ }316
+317
+ fn media_registration() -> MediaRegistration {318
+ MediaRegistration {319
+ id: MediaId::new("test-scope#image".to_owned()),320
+ kind: MediaKind::Image,321
+ full: MediaSourceInfo {322
+ source: MediaSource::Plain(323
+ ruma::mxc_uri!("mxc://mentions.example/image").to_owned(),324
+ ),325
+ mimetype: Some("image/png".to_owned()),326
+ declared_size: Some(1234),327
+ },328
+ thumbnail: None,329
+ }330
+ }331
+332
+ fn saved_data() -> Data {333
+ let mention = key("persisted-room", "persisted-mention");334
+ let mention_id = id(&mention);335
+ let context_item =336
+ message_event("$persisted-mention:mentions.example", 42, "saved message");337
+ let mut data = Data {338
+ revision: 1,339
+ ..Data::default()340
+ };341
+ data.entries.insert(342
+ mention_id.clone(),343
+ MentionEntry {344
+ event: context_item.clone(),345
+ ..entry(mention.clone(), 42, 1, true)346
+ },347
+ );348
+ data.events.insert(349
+ mention_id.clone(),350
+ SavedEvent {351
+ item: context_item,352
+ media: vec![media_registration()],353
+ raw: Some(354
+ json!({"event_id": "$persisted-mention:mentions.example", "body": "saved raw"}),355
+ ),356
+ },357
+ );358
+ data.windows.insert(359
+ mention_id.clone(),360
+ Window {361
+ before: Vec::new(),362
+ target: Some(mention_id),363
+ after: Vec::new(),364
+ start: Some("start-token".to_owned()),365
+ end: Some("end-token".to_owned()),366
+ },367
+ );368
+ data.checkpoints369
+ .insert("sync-connection".to_owned(), "position-1".to_owned());370
+ data.history_cursor = Some("history-position".to_owned());371
+ data372
+ }373
+374
+ async fn open(config: &StoreConfig) -> MentionsStore {375
+ MentionsStore::open(config, account(), MAX_BYTES)376
+ .await377
+ .unwrap()378
+ }379
+380
+ #[cfg_attr(not(target_family = "wasm"), tokio::test)]381
+ #[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]382
+ async fn encrypted_store_roundtrips_inbox_context_read_state_and_checkpoints() {383
+ let config = config();384
+ let data = saved_data();385
+ let mut store = open(&config).await;386
+ assert_eq!(387
+ store388
+ .commit(389
+ 0,390
+ vec![Write::Set(391
+ "inbox".to_owned(),392
+ serde_json::to_value(&data).unwrap(),393
+ )],394
+ )395
+ .await396
+ .unwrap(),397
+ 1398
+ );399
+ drop(store);400
+401
+ let mut restored = open(&config).await;402
+ let mut restored_data: Data = restored.read("inbox").unwrap().unwrap();403
+ assert_eq!(restored.revision(), restored_data.revision);404
+ assert!(restored_data.entries.values().next().unwrap().read);405
+ assert_eq!(406
+ restored_data407
+ .checkpoints408
+ .get("sync-connection")409
+ .map(String::as_str),410
+ Some("position-1")411
+ );412
+ let mention = restored_data.entries.values().next().unwrap().key.clone();413
+ let (context, media) = restored_data.context(&mention).unwrap();414
+ assert_eq!(415
+ context.context.unwrap().start_token.as_deref(),416
+ Some("start-token")417
+ );418
+ assert_eq!(media.len(), 1);419
+ assert_eq!(media[0].id.as_str(), "test-scope#image");420
+ assert_eq!(media[0].full.mimetype.as_deref(), Some("image/png"));421
+ let event = restored_data.events.values().next().unwrap();422
+ assert_eq!(event.raw.as_ref().unwrap()["body"], "saved raw");423
+424
+ restored_data.revision = 2;425
+ let updated_entry = restored_data.entries.values_mut().next().unwrap();426
+ updated_entry.read = false;427
+ updated_entry.revision = 2;428
+ restored_data429
+ .checkpoints430
+ .insert("sync-connection".to_owned(), "position-2".to_owned());431
+ restored432
+ .commit(433
+ 1,434
+ vec![Write::Set(435
+ "inbox".to_owned(),436
+ serde_json::to_value(&restored_data).unwrap(),437
+ )],438
+ )439
+ .await440
+ .unwrap();441
+ drop(restored);442
+443
+ let restored = open(&config).await;444
+ let restored_data: Data = restored.read("inbox").unwrap().unwrap();445
+ assert_eq!(restored.revision(), 2);446
+ assert_eq!(restored_data.revision, 2);447
+ assert!(!restored_data.entries.values().next().unwrap().read);448
+ assert_eq!(449
+ restored_data450
+ .checkpoints451
+ .get("sync-connection")452
+ .map(String::as_str),453
+ Some("position-2")454
+ );455
+ drop(restored);456
+ config.delete().await.unwrap();457
+ }458
+}@@ -0,0 +1,217 @@
1
+use std::{collections::BTreeMap, ops::Deref};2
+3
+use indexed_db_futures::{4
+ Build,5
+ database::Database,6
+ error::{Error as IndexedDbError, OpenDbError},7
+ prelude::*,8
+ transaction::{Transaction, TransactionDurability, TransactionMode, TransactionOptions},9
+ typed_array::{Uint8Array, Uint8ArraySlice},10
+};11
+12
+use super::Snapshot;13
+14
+const RECORDS: &str = "records";15
+const CIPHER: &str = "!cipher";16
+const HEAD: &str = "!head";17
+const INCOMPLETE_METADATA: &str = "mentions database metadata is incomplete";18
+const INVALID_RECORDS: &str = "mentions database records are inconsistent";19
+const RESERVED_ROW_KEY: &str = "mentions row key uses a reserved metadata key";20
+21
+struct DatabaseHandle(Database);22
+23
+impl Deref for DatabaseHandle {24
+ type Target = Database;25
+26
+ fn deref(&self) -> &Self::Target {27
+ &self.028
+ }29
+}30
+31
+impl Drop for DatabaseHandle {32
+ fn drop(&mut self) {33
+ self.0.clone().close();34
+ }35
+}36
+37
+pub(super) struct Backend {38
+ name: String,39
+}40
+41
+impl Backend {42
+ pub(super) async fn open(name: String) -> Result<Self, crate::Error> {43
+ let _database = open_database(&name)44
+ .await45
+ .map_err(|error| super::storage_error("opening the mentions database", error))?;46
+ Ok(Self { name })47
+ }48
+49
+ pub(super) async fn load(&self) -> Result<Option<Snapshot>, crate::Error> {50
+ let database = open_database(&self.name)51
+ .await52
+ .map_err(|error| super::storage_error("opening the mentions database", error))?;53
+ let snapshot = read_snapshot(&database).await;54
+ match snapshot {55
+ Ok(snapshot) => Ok(snapshot),56
+ Err(ReadError::Invalid(message)) => {57
+ Err(crate::Error::new(crate::ErrorKind::Store, message))58
+ }59
+ Err(ReadError::Storage(error)) => {60
+ Err(super::storage_error("reading the mentions database", error))61
+ }62
+ }63
+ }64
+65
+ pub(super) async fn commit(66
+ &self,67
+ expected: Option<Vec<u8>>,68
+ exported: Vec<u8>,69
+ head: Vec<u8>,70
+ writes: BTreeMap<String, Option<Vec<u8>>>,71
+ ) -> Result<(), crate::Error> {72
+ let database = open_database(&self.name)73
+ .await74
+ .map_err(|error| super::storage_error("opening the mentions database", error))?;75
+ let committed = write_snapshot(&database, expected, exported, head, writes).await;76
+ match committed {77
+ Ok(()) => Ok(()),78
+ Err(CommitError::Conflict) => Err(super::conflict()),79
+ Err(CommitError::Invalid(message)) => {80
+ Err(crate::Error::new(crate::ErrorKind::Store, message))81
+ }82
+ Err(CommitError::Storage(error)) => {83
+ Err(super::storage_error("writing the mentions database", error))84
+ }85
+ }86
+ }87
+}88
+89
+async fn open_database(name: &str) -> Result<DatabaseHandle, OpenDbError> {90
+ Database::open(name)91
+ .with_version(1u32)92
+ .with_on_upgrade_needed(93
+ |_, transaction: &Transaction<'_>| -> Result<(), IndexedDbError> {94
+ transaction.db().create_object_store(RECORDS).build()?;95
+ Ok(())96
+ },97
+ )98
+ .build()?99
+ .await100
+ .map(DatabaseHandle)101
+}102
+103
+enum ReadError {104
+ Invalid(&'static str),105
+ Storage(IndexedDbError),106
+}107
+108
+impl From<IndexedDbError> for ReadError {109
+ fn from(error: IndexedDbError) -> Self {110
+ Self::Storage(error)111
+ }112
+}113
+114
+async fn read_snapshot(database: &Database) -> Result<Option<Snapshot>, ReadError> {115
+ let transaction = database116
+ .transaction(RECORDS)117
+ .with_mode(TransactionMode::Readonly)118
+ .build()?;119
+ let store = transaction.object_store(RECORDS)?;120
+ let keys = store.get_all_keys::<String>().build()?;121
+ let values = store.get_all::<Uint8Array>().build()?;122
+ let keys = keys.await?.collect::<Result<Vec<_>, _>>()?;123
+ let values = values.await?.collect::<Result<Vec<_>, _>>()?;124
+ if keys.len() != values.len() {125
+ return Err(ReadError::Invalid(INVALID_RECORDS));126
+ }127
+ let mut records = keys128
+ .into_iter()129
+ .zip(values)130
+ .map(|(key, value)| (key, Vec::<u8>::from(value)))131
+ .collect::<BTreeMap<_, _>>();132
+ if records.is_empty() {133
+ return Ok(None);134
+ }135
+ let exported = records.remove(CIPHER);136
+ let head = records.remove(HEAD);137
+ match (exported, head) {138
+ (Some(exported), Some(head)) => Ok(Some(Snapshot {139
+ exported,140
+ head,141
+ rows: records,142
+ })),143
+ _ => Err(ReadError::Invalid(INCOMPLETE_METADATA)),144
+ }145
+}146
+147
+enum CommitError {148
+ Conflict,149
+ Invalid(&'static str),150
+ Storage(IndexedDbError),151
+}152
+153
+impl From<IndexedDbError> for CommitError {154
+ fn from(error: IndexedDbError) -> Self {155
+ Self::Storage(error)156
+ }157
+}158
+159
+async fn write_snapshot(160
+ database: &Database,161
+ expected: Option<Vec<u8>>,162
+ exported: Vec<u8>,163
+ head: Vec<u8>,164
+ writes: BTreeMap<String, Option<Vec<u8>>>,165
+) -> Result<(), CommitError> {166
+ let mut options = TransactionOptions::new();167
+ options.set_durability(TransactionDurability::Strict);168
+ let transaction = database169
+ .transaction(RECORDS)170
+ .with_mode(TransactionMode::Readwrite)171
+ .with_options(options)172
+ .build()?;173
+ let current = transaction174
+ .object_store(RECORDS)?175
+ .get::<Uint8Array, _, _>(HEAD)176
+ .await?177
+ .map(Vec::<u8>::from);178
+ if current != expected {179
+ let _ = transaction.abort().await;180
+ return Err(CommitError::Conflict);181
+ }182
+ if writes.keys().any(|key| key == CIPHER || key == HEAD) {183
+ let _ = transaction.abort().await;184
+ return Err(CommitError::Invalid(RESERVED_ROW_KEY));185
+ }186
+ let requests = (|| -> Result<(), IndexedDbError> {187
+ let store = transaction.object_store(RECORDS)?;188
+ store189
+ .put(Uint8ArraySlice::new(&exported))190
+ .with_key(CIPHER)191
+ .build()?;192
+ store193
+ .put(Uint8ArraySlice::new(&head))194
+ .with_key(HEAD)195
+ .build()?;196
+ for (key, value) in writes {197
+ match value {198
+ Some(value) => {199
+ store200
+ .put(Uint8ArraySlice::new(&value))201
+ .with_key(key)202
+ .build()?;203
+ }204
+ None => {205
+ store.delete(key).build()?;206
+ }207
+ }208
+ }209
+ Ok(())210
+ })();211
+ if let Err(error) = requests {212
+ let _ = transaction.abort().await;213
+ return Err(CommitError::Storage(error));214
+ }215
+ transaction.commit().await?;216
+ Ok(())217
+}@@ -0,0 +1,336 @@
1
+use std::{collections::BTreeMap, fmt};2
+3
+use matrix_sdk::{SendOutsideWasm, SyncOutsideWasm};4
+use matrix_sdk_store_encryption::StoreCipher;5
+use ruma::OwnedUserId;6
+use serde::{Deserialize, Serialize, de::DeserializeOwned};7
+use serde_json::Value;8
+9
+use crate::{Error, ErrorKind, StoreConfig};10
+11
+#[cfg(feature = "sqlite")]12
+mod sqlite;13
+#[cfg(feature = "sqlite")]14
+use sqlite::Backend;15
+#[cfg(feature = "indexeddb")]16
+mod indexeddb;17
+#[cfg(feature = "indexeddb")]18
+use indexeddb::Backend;19
+20
+const SCHEMA: u32 = 1;21
+const MAX_RECORDS: usize = 35_128;22
+23
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]24
+pub(crate) struct Account {25
+ pub(crate) homeserver: String,26
+ pub(crate) user_id: OwnedUserId,27
+}28
+29
+#[derive(Serialize, Deserialize)]30
+struct Head {31
+ schema: u32,32
+ account: Account,33
+ revision: u64,34
+ records: usize,35
+ bytes: usize,36
+}37
+38
+#[derive(Serialize, Deserialize)]39
+struct Record {40
+ schema: u32,41
+ account: Account,42
+ key: String,43
+ value: Value,44
+}45
+46
+pub(super) struct Snapshot {47
+ pub(super) exported: Vec<u8>,48
+ pub(super) head: Vec<u8>,49
+ pub(super) rows: BTreeMap<String, Vec<u8>>,50
+}51
+52
+pub(crate) enum Write {53
+ Set(String, Value),54
+ Remove(String),55
+}56
+57
+pub(crate) struct MentionsStore {58
+ backend: Backend,59
+ cipher: StoreCipher,60
+ account: Account,61
+ snapshot: Snapshot,62
+ revision: u64,63
+ max_bytes: usize,64
+}65
+66
+impl fmt::Debug for MentionsStore {67
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {68
+ formatter69
+ .debug_struct("MentionsStore")70
+ .field("revision", &self.revision)71
+ .field("records", &self.snapshot.rows.len())72
+ .finish_non_exhaustive()73
+ }74
+}75
+76
+impl MentionsStore {77
+ pub(crate) async fn open(78
+ config: &StoreConfig,79
+ account: Account,80
+ max_bytes: usize,81
+ ) -> Result<Self, Error> {82
+ let backend = Backend::open(config.mentions_location()).await?;83
+ let (cipher, snapshot, revision) = match backend.load().await? {84
+ Some(snapshot) => {85
+ validate_size(&snapshot, max_bytes)?;86
+ let cipher = StoreCipher::import(config.passphrase(), &snapshot.exported)87
+ .map_err(|error| storage_error("opening the mentions cipher", error))?;88
+ let head: Head = cipher89
+ .decrypt_value(&snapshot.head)90
+ .map_err(|error| storage_error("opening the mentions header", error))?;91
+ validate_account(head.schema, &head.account, &account)?;92
+ if head.records != snapshot.rows.len() || head.bytes != row_bytes(&snapshot.rows)? {93
+ return Err(invalid(94
+ "mentions records do not match their committed header",95
+ ));96
+ }97
+ for (key, encrypted) in &snapshot.rows {98
+ let record: Record = cipher99
+ .decrypt_value(encrypted)100
+ .map_err(|error| storage_error("opening a mentions record", error))?;101
+ validate_account(record.schema, &record.account, &account)?;102
+ if storage_key(&cipher, &record.key) != *key {103
+ return Err(invalid("a mentions record belongs to a different key"));104
+ }105
+ }106
+ let revision = head.revision;107
+ (cipher, snapshot, revision)108
+ }109
+ None => {110
+ let cipher = StoreCipher::new()111
+ .map_err(|error| storage_error("creating the mentions cipher", error))?;112
+ let exported = cipher113
+ .export(config.passphrase())114
+ .map_err(|error| storage_error("sealing the mentions cipher", error))?;115
+ let head = cipher116
+ .encrypt_value(&Head {117
+ schema: SCHEMA,118
+ account: account.clone(),119
+ revision: 0,120
+ records: 0,121
+ bytes: 0,122
+ })123
+ .map_err(|error| storage_error("sealing the mentions header", error))?;124
+ let snapshot = Snapshot {125
+ exported,126
+ head,127
+ rows: BTreeMap::new(),128
+ };129
+ validate_size(&snapshot, max_bytes)?;130
+ backend131
+ .commit(132
+ None,133
+ snapshot.exported.clone(),134
+ snapshot.head.clone(),135
+ BTreeMap::new(),136
+ )137
+ .await?;138
+ (cipher, snapshot, 0)139
+ }140
+ };141
+ validate_size(&snapshot, max_bytes)?;142
+ Ok(Self {143
+ backend,144
+ cipher,145
+ account,146
+ snapshot,147
+ revision,148
+ max_bytes,149
+ })150
+ }151
+152
+ pub(crate) fn revision(&self) -> u64 {153
+ self.revision154
+ }155
+156
+ pub(crate) fn read<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>, Error> {157
+ let Some(encrypted) = self.snapshot.rows.get(&storage_key(&self.cipher, key)) else {158
+ return Ok(None);159
+ };160
+ let record: Record = self161
+ .cipher162
+ .decrypt_value(encrypted)163
+ .map_err(|error| storage_error("opening a mentions record", error))?;164
+ validate_account(record.schema, &record.account, &self.account)?;165
+ if record.key != key {166
+ return Err(invalid("a mentions record belongs to a different key"));167
+ }168
+ serde_json::from_value(record.value)169
+ .map(Some)170
+ .map_err(|error| storage_error("decoding a mentions record", error))171
+ }172
+173
+ pub(crate) async fn commit(174
+ &mut self,175
+ expected_revision: u64,176
+ writes: Vec<Write>,177
+ ) -> Result<u64, Error> {178
+ if self.revision != expected_revision {179
+ return Err(conflict());180
+ }181
+ if writes.is_empty() {182
+ return Ok(self.revision);183
+ }184
+ let revision = self185
+ .revision186
+ .checked_add(1)187
+ .ok_or_else(|| invalid("mentions revision exhausted"))?;188
+ let mut changes = BTreeMap::new();189
+ for write in writes {190
+ match write {191
+ Write::Set(key, value) => {192
+ let hashed = storage_key(&self.cipher, &key);193
+ let record = Record {194
+ schema: SCHEMA,195
+ account: self.account.clone(),196
+ key,197
+ value,198
+ };199
+ let encrypted = self200
+ .cipher201
+ .encrypt_value(&record)202
+ .map_err(|error| storage_error("sealing a mentions record", error))?;203
+ changes.insert(hashed, Some(encrypted));204
+ }205
+ Write::Remove(key) => {206
+ changes.insert(storage_key(&self.cipher, &key), None);207
+ }208
+ }209
+ }210
+ let mut records = self.snapshot.rows.len();211
+ let mut bytes = row_bytes(&self.snapshot.rows)?;212
+ for (key, value) in &changes {213
+ if let Some(previous) = self.snapshot.rows.get(key) {214
+ bytes -= key.len() + previous.len();215
+ records -= 1;216
+ }217
+ if let Some(value) = value {218
+ bytes = bytes219
+ .checked_add(key.len())220
+ .and_then(|bytes| bytes.checked_add(value.len()))221
+ .ok_or_else(|| invalid("mentions byte count overflow"))?;222
+ records += 1;223
+ }224
+ }225
+ let head = self226
+ .cipher227
+ .encrypt_value(&Head {228
+ schema: SCHEMA,229
+ account: self.account.clone(),230
+ revision,231
+ records,232
+ bytes,233
+ })234
+ .map_err(|error| storage_error("sealing the mentions header", error))?;235
+ validate_budget(236
+ bytes,237
+ records,238
+ self.snapshot.exported.len(),239
+ head.len(),240
+ self.max_bytes,241
+ )?;242
+ self.backend243
+ .commit(244
+ Some(self.snapshot.head.clone()),245
+ self.snapshot.exported.clone(),246
+ head.clone(),247
+ changes.clone(),248
+ )249
+ .await?;250
+ for (key, value) in changes {251
+ match value {252
+ Some(value) => {253
+ self.snapshot.rows.insert(key, value);254
+ }255
+ None => {256
+ self.snapshot.rows.remove(&key);257
+ }258
+ }259
+ }260
+ self.snapshot.head = head;261
+ self.revision = revision;262
+ Ok(revision)263
+ }264
+}265
+266
+fn validate_account(schema: u32, actual: &Account, expected: &Account) -> Result<(), Error> {267
+ if schema != SCHEMA {268
+ return Err(invalid("unsupported mentions schema"));269
+ }270
+ if actual != expected {271
+ return Err(invalid("mentions store belongs to a different account"));272
+ }273
+ Ok(())274
+}275
+276
+fn validate_size(snapshot: &Snapshot, max_bytes: usize) -> Result<(), Error> {277
+ validate_budget(278
+ row_bytes(&snapshot.rows)?,279
+ snapshot.rows.len(),280
+ snapshot.exported.len(),281
+ snapshot.head.len(),282
+ max_bytes,283
+ )284
+}285
+286
+fn validate_budget(287
+ row_bytes: usize,288
+ records: usize,289
+ exported_len: usize,290
+ head_len: usize,291
+ max_bytes: usize,292
+) -> Result<(), Error> {293
+ let bytes = row_bytes294
+ .checked_add(exported_len)295
+ .and_then(|bytes| bytes.checked_add(head_len))296
+ .ok_or_else(|| invalid("mentions byte count overflow"))?;297
+ if bytes > max_bytes || records > MAX_RECORDS {298
+ return Err(invalid("mentions storage limit exceeded"));299
+ }300
+ Ok(())301
+}302
+303
+fn row_bytes(rows: &BTreeMap<String, Vec<u8>>) -> Result<usize, Error> {304
+ rows.iter().try_fold(0usize, |bytes, (key, value)| {305
+ bytes306
+ .checked_add(key.len())307
+ .and_then(|bytes| bytes.checked_add(value.len()))308
+ .ok_or_else(|| invalid("mentions byte count overflow"))309
+ })310
+}311
+312
+fn storage_key(cipher: &StoreCipher, key: &str) -> String {313
+ cipher314
+ .hash_key("macltok-mentions", key.as_bytes())315
+ .iter()316
+ .map(|byte| format!("{byte:02x}"))317
+ .collect()318
+}319
+320
+fn invalid(message: &'static str) -> Error {321
+ Error::new(ErrorKind::Store, message)322
+}323
+324
+pub(super) fn conflict() -> Error {325
+ invalid("mentions store changed, reopen before retrying")326
+}327
+328
+pub(super) fn storage_error(329
+ context: &'static str,330
+ source: impl std::error::Error + SendOutsideWasm + SyncOutsideWasm + 'static,331
+) -> Error {332
+ Error::with_source(ErrorKind::Store, context, source)333
+}334
+335
+#[cfg(test)]336
+mod tests;@@ -0,0 +1,214 @@
1
+use std::{2
+ collections::BTreeMap,3
+ fs::{self, OpenOptions},4
+ io,5
+ path::{Path, PathBuf},6
+ time::Duration,7
+};8
+9
+use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};10
+11
+use super::{Snapshot, conflict, storage_error};12
+use crate::{Error, ErrorKind};13
+14
+const CIPHER_KEY: &str = "!cipher";15
+const HEAD_KEY: &str = "!head";16
+17
+pub(super) struct Backend {18
+ path: PathBuf,19
+}20
+21
+impl Backend {22
+ pub(super) async fn open(path: PathBuf) -> Result<Self, Error> {23
+ let setup_path = path.clone();24
+ tokio::task::spawn_blocking(move || initialize(&setup_path))25
+ .await26
+ .map_err(|error| storage_error("opening the mentions store", error))??;27
+ Ok(Self { path })28
+ }29
+30
+ pub(super) async fn load(&self) -> Result<Option<Snapshot>, Error> {31
+ let path = self.path.clone();32
+ tokio::task::spawn_blocking(move || load_snapshot(&path))33
+ .await34
+ .map_err(|error| storage_error("loading the mentions store", error))?35
+ }36
+37
+ pub(super) async fn commit(38
+ &self,39
+ expected: Option<Vec<u8>>,40
+ exported: Vec<u8>,41
+ head: Vec<u8>,42
+ writes: BTreeMap<String, Option<Vec<u8>>>,43
+ ) -> Result<(), Error> {44
+ let path = self.path.clone();45
+ tokio::task::spawn_blocking(move || {46
+ commit_snapshot(&path, expected, exported, head, writes)47
+ })48
+ .await49
+ .map_err(|error| storage_error("committing the mentions store", error))?50
+ }51
+}52
+53
+fn initialize(path: &Path) -> Result<(), Error> {54
+ let parent = path55
+ .parent()56
+ .filter(|parent| !parent.as_os_str().is_empty())57
+ .unwrap_or_else(|| Path::new("."));58
+ fs::create_dir_all(parent)59
+ .map_err(|error| storage_error("creating the mentions store directory", error))?;60
+ create_private_file(path)61
+ .map_err(|error| storage_error("creating the mentions database file", error))?;62
+ let connection = connect(path)?;63
+ connection64
+ .execute(65
+ "CREATE TABLE IF NOT EXISTS records (key TEXT PRIMARY KEY, value BLOB NOT NULL)",66
+ [],67
+ )68
+ .map_err(|error| storage_error("initializing the mentions database", error))?;69
+ Ok(())70
+}71
+72
+fn connect(path: &Path) -> Result<Connection, Error> {73
+ let connection = Connection::open(path)74
+ .map_err(|error| storage_error("opening the mentions database", error))?;75
+ connection76
+ .busy_timeout(Duration::from_secs(5))77
+ .map_err(|error| storage_error("configuring the mentions database", error))?;78
+ connection79
+ .execute_batch("PRAGMA synchronous = FULL")80
+ .map_err(|error| storage_error("configuring the mentions database", error))?;81
+ Ok(connection)82
+}83
+84
+fn load_snapshot(path: &Path) -> Result<Option<Snapshot>, Error> {85
+ let mut connection = connect(path)?;86
+ let transaction = connection87
+ .transaction_with_behavior(TransactionBehavior::Deferred)88
+ .map_err(|error| storage_error("starting a mentions snapshot", error))?;89
+ let mut rows = BTreeMap::new();90
+ {91
+ let mut statement = transaction92
+ .prepare("SELECT key, value FROM records")93
+ .map_err(|error| storage_error("reading the mentions snapshot", error))?;94
+ let result = statement95
+ .query_map([], |row| {96
+ Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))97
+ })98
+ .map_err(|error| storage_error("reading the mentions snapshot", error))?;99
+ for row in result {100
+ let (key, value) =101
+ row.map_err(|error| storage_error("reading the mentions snapshot", error))?;102
+ rows.insert(key, value);103
+ }104
+ }105
+ transaction106
+ .commit()107
+ .map_err(|error| storage_error("finishing the mentions snapshot", error))?;108
+ if rows.is_empty() {109
+ return Ok(None);110
+ }111
+ let exported = rows.remove(CIPHER_KEY).ok_or_else(|| {112
+ Error::new(113
+ ErrorKind::Store,114
+ "mentions store is missing cipher metadata",115
+ )116
+ })?;117
+ let head = rows118
+ .remove(HEAD_KEY)119
+ .ok_or_else(|| Error::new(ErrorKind::Store, "mentions store is missing head metadata"))?;120
+ Ok(Some(Snapshot {121
+ exported,122
+ head,123
+ rows,124
+ }))125
+}126
+127
+fn commit_snapshot(128
+ path: &Path,129
+ expected: Option<Vec<u8>>,130
+ exported: Vec<u8>,131
+ head: Vec<u8>,132
+ writes: BTreeMap<String, Option<Vec<u8>>>,133
+) -> Result<(), Error> {134
+ let mut connection = connect(path)?;135
+ let transaction = connection136
+ .transaction_with_behavior(TransactionBehavior::Immediate)137
+ .map_err(|error| storage_error("starting a mentions commit", error))?;138
+ let current: Option<Vec<u8>> = transaction139
+ .query_row(140
+ "SELECT value FROM records WHERE key = ?1",141
+ [HEAD_KEY],142
+ |row| row.get(0),143
+ )144
+ .optional()145
+ .map_err(|error| storage_error("checking the mentions store head", error))?;146
+ if current != expected {147
+ return Err(conflict());148
+ }149
+ if writes.contains_key(CIPHER_KEY) || writes.contains_key(HEAD_KEY) {150
+ return Err(Error::new(151
+ ErrorKind::Store,152
+ "mentions record uses a reserved key",153
+ ));154
+ }155
+ transaction156
+ .execute(157
+ "INSERT INTO records (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",158
+ params![CIPHER_KEY, exported],159
+ )160
+ .map_err(|error| storage_error("writing mentions cipher metadata", error))?;161
+ transaction162
+ .execute(163
+ "INSERT INTO records (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",164
+ params![HEAD_KEY, head],165
+ )166
+ .map_err(|error| storage_error("writing the mentions store head", error))?;167
+ for (key, value) in writes {168
+ match value {169
+ Some(value) => {170
+ transaction171
+ .execute(172
+ "INSERT INTO records (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",173
+ params![key, value],174
+ )175
+ .map_err(|error| storage_error("writing a mentions record", error))?;176
+ }177
+ None => {178
+ transaction179
+ .execute("DELETE FROM records WHERE key = ?1", [key])180
+ .map_err(|error| storage_error("deleting a mentions record", error))?;181
+ }182
+ }183
+ }184
+ transaction185
+ .commit()186
+ .map_err(|error| storage_error("finishing a mentions commit", error))187
+}188
+189
+#[cfg(unix)]190
+fn create_private_file(path: &Path) -> io::Result<()> {191
+ use std::os::unix::fs::OpenOptionsExt;192
+193
+ match OpenOptions::new()194
+ .write(true)195
+ .create_new(true)196
+ .mode(0o600)197
+ .open(path)198
+ {199
+ Ok(file) => drop(file),200
+ Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}201
+ Err(error) => return Err(error),202
+ }203
+ Ok(())204
+}205
+206
+#[cfg(not(unix))]207
+fn create_private_file(path: &Path) -> io::Result<()> {208
+ match OpenOptions::new().write(true).create_new(true).open(path) {209
+ Ok(file) => drop(file),210
+ Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}211
+ Err(error) => return Err(error),212
+ }213
+ Ok(())214
+}@@ -0,0 +1,418 @@
1
+use std::sync::atomic::{AtomicUsize, Ordering};2
+3
+use ruma::user_id;4
+use serde_json::{Value, json};5
+6
+use super::{Account, Head, MentionsStore, Write, row_bytes, storage_key};7
+use crate::StoreConfig;8
+9
+#[cfg(target_family = "wasm")]10
+wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);11
+12
+static STORES: AtomicUsize = AtomicUsize::new(0);13
+const MAX_BYTES: usize = 64 * 1024 * 1024;14
+15
+fn account() -> Account {16
+ Account {17
+ homeserver: "https://mentions.example/".to_owned(),18
+ user_id: user_id!("@alice:mentions.example").to_owned(),19
+ }20
+}21
+22
+fn config() -> StoreConfig {23
+ let number = STORES.fetch_add(1, Ordering::Relaxed);24
+ #[cfg(feature = "sqlite")]25
+ return StoreConfig::sqlite(26
+ std::env::temp_dir().join(format!("macltok-mentions-{}-{number}", std::process::id())),27
+ "test store passphrase",28
+ );29
+ #[cfg(feature = "indexeddb")]30
+ return StoreConfig::indexeddb(31
+ format!("macltok-mentions-{}-{number}", js_sys::Date::now()),32
+ "test store passphrase",33
+ );34
+}35
+36
+fn set(key: &str, value: Value) -> Write {37
+ Write::Set(key.to_owned(), value)38
+}39
+40
+async fn open(config: &StoreConfig) -> MentionsStore {41
+ MentionsStore::open(config, account(), MAX_BYTES)42
+ .await43
+ .unwrap()44
+}45
+46
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]47
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]48
+async fn entries_context_read_state_and_checkpoint_restore_together() {49
+ let config = config();50
+ let mut store = open(&config).await;51
+ assert_eq!(store.revision(), 0);52
+ assert_eq!(53
+ store54
+ .commit(55
+ 0,56
+ vec![57
+ set(58
+ "entry/room/event",59
+ json!({"preview": "secret mention preview"})60
+ ),61
+ set("context/room/event", json!({"body": "secret context body"})),62
+ set("read/room/event", json!(false)),63
+ set("checkpoint/sync", json!("position-1")),64
+ ]65
+ )66
+ .await67
+ .unwrap(),68
+ 169
+ );70
+ drop(store);71
+ let mut restored = open(&config).await;72
+ assert_eq!(restored.revision(), 1);73
+ assert_eq!(74
+ restored.read::<Value>("entry/room/event").unwrap(),75
+ Some(json!({"preview": "secret mention preview"}))76
+ );77
+ assert_eq!(78
+ restored.read::<Value>("context/room/event").unwrap(),79
+ Some(json!({"body": "secret context body"}))80
+ );81
+ assert_eq!(82
+ restored.read::<bool>("read/room/event").unwrap(),83
+ Some(false)84
+ );85
+ assert_eq!(86
+ restored87
+ .read::<String>("checkpoint/sync")88
+ .unwrap()89
+ .as_deref(),90
+ Some("position-1")91
+ );92
+ restored93
+ .commit(94
+ 1,95
+ vec![96
+ Write::Remove("entry/room/event".to_owned()),97
+ set("read/room/event", json!(true)),98
+ set("checkpoint/sync", json!("position-2")),99
+ ],100
+ )101
+ .await102
+ .unwrap();103
+ drop(restored);104
+ let restored = open(&config).await;105
+ assert_eq!(restored.revision(), 2);106
+ assert!(107
+ restored108
+ .read::<Value>("entry/room/event")109
+ .unwrap()110
+ .is_none()111
+ );112
+ assert_eq!(113
+ restored.read::<bool>("read/room/event").unwrap(),114
+ Some(true)115
+ );116
+ assert_eq!(117
+ restored118
+ .read::<String>("checkpoint/sync")119
+ .unwrap()120
+ .as_deref(),121
+ Some("position-2")122
+ );123
+ drop(restored);124
+ config.delete().await.unwrap();125
+}126
+127
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]128
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]129
+async fn stale_writers_cannot_overwrite_a_committed_checkpoint() {130
+ let config = config();131
+ let mut first = open(&config).await;132
+ let mut stale = open(&config).await;133
+ first134
+ .commit(0, vec![set("checkpoint", json!("committed"))])135
+ .await136
+ .unwrap();137
+ assert!(138
+ stale139
+ .commit(140
+ 0,141
+ vec![142
+ set("checkpoint", json!("stale")),143
+ set("entry", json!("uncommitted")),144
+ ]145
+ )146
+ .await147
+ .is_err()148
+ );149
+ assert_eq!(stale.revision(), 0);150
+ assert!(stale.read::<Value>("entry").unwrap().is_none());151
+ assert!(152
+ first153
+ .commit(0, vec![set("checkpoint", json!("wrong revision"))])154
+ .await155
+ .is_err()156
+ );157
+ let restored = open(&config).await;158
+ assert_eq!(restored.revision(), 1);159
+ assert_eq!(160
+ restored.read::<String>("checkpoint").unwrap().as_deref(),161
+ Some("committed")162
+ );163
+ assert!(restored.read::<Value>("entry").unwrap().is_none());164
+ drop((first, stale, restored));165
+ config.delete().await.unwrap();166
+}167
+168
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]169
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]170
+async fn capacity_failure_keeps_the_old_revision_and_checkpoint() {171
+ let config = config();172
+ let mut store = MentionsStore::open(&config, account(), 4096).await.unwrap();173
+ store174
+ .commit(0, vec![set("checkpoint", json!("old"))])175
+ .await176
+ .unwrap();177
+ assert!(178
+ store179
+ .commit(180
+ 1,181
+ vec![182
+ set("checkpoint", json!("new")),183
+ set("entry", json!("x".repeat(16384))),184
+ ]185
+ )186
+ .await187
+ .is_err()188
+ );189
+ assert_eq!(store.revision(), 1);190
+ assert_eq!(191
+ store.read::<String>("checkpoint").unwrap().as_deref(),192
+ Some("old")193
+ );194
+ assert!(store.read::<Value>("entry").unwrap().is_none());195
+ drop(store);196
+ let restored = open(&config).await;197
+ assert_eq!(restored.revision(), 1);198
+ assert_eq!(199
+ restored.read::<String>("checkpoint").unwrap().as_deref(),200
+ Some("old")201
+ );202
+ drop(restored);203
+ config.delete().await.unwrap();204
+}205
+206
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]207
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]208
+async fn records_hide_content_keys_and_account_metadata() {209
+ let config = config();210
+ let mut store = open(&config).await;211
+ store212
+ .commit(213
+ 0,214
+ vec![set(215
+ "private/room/event",216
+ json!({"body": "a secret saved message"}),217
+ )],218
+ )219
+ .await220
+ .unwrap();221
+ let snapshot = store.backend.load().await.unwrap().unwrap();222
+ let mut bytes = snapshot.exported;223
+ bytes.extend(snapshot.head);224
+ for (key, value) in snapshot.rows {225
+ bytes.extend(key.as_bytes());226
+ bytes.extend(value);227
+ }228
+ #[cfg(feature = "sqlite")]229
+ bytes.extend(std::fs::read(config.mentions_location()).unwrap());230
+ for secret in [231
+ "private/room/event",232
+ "a secret saved message",233
+ "mentions.example",234
+ ] {235
+ assert!(236
+ !bytes237
+ .windows(secret.len())238
+ .any(|window| window == secret.as_bytes())239
+ );240
+ }241
+ drop(store);242
+ config.delete().await.unwrap();243
+}244
+245
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]246
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]247
+async fn wrong_account_server_and_passphrase_cannot_open_the_store() {248
+ let config = config();249
+ let store = open(&config).await;250
+ drop(store);251
+ let other = Account {252
+ user_id: user_id!("@bob:mentions.example").to_owned(),253
+ ..account()254
+ };255
+ assert!(256
+ MentionsStore::open(&config, other, MAX_BYTES)257
+ .await258
+ .is_err()259
+ );260
+ let other_server = Account {261
+ homeserver: "https://different.example/".to_owned(),262
+ ..account()263
+ };264
+ assert!(265
+ MentionsStore::open(&config, other_server, MAX_BYTES)266
+ .await267
+ .is_err()268
+ );269
+ #[cfg(feature = "sqlite")]270
+ let wrong_passphrase =271
+ StoreConfig::sqlite(config.mentions_location().parent().unwrap(), "incorrect");272
+ #[cfg(feature = "indexeddb")]273
+ let wrong_passphrase = StoreConfig::indexeddb(config.indexeddb_name(), "incorrect");274
+ assert!(275
+ MentionsStore::open(&wrong_passphrase, account(), MAX_BYTES)276
+ .await277
+ .is_err()278
+ );279
+ drop(open(&config).await);280
+ config.delete().await.unwrap();281
+}282
+283
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]284
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]285
+async fn ciphertext_corruption_and_record_swaps_are_rejected() {286
+ let config = config();287
+ let mut store = open(&config).await;288
+ store289
+ .commit(290
+ 0,291
+ vec![set("one", json!("first")), set("two", json!("other"))],292
+ )293
+ .await294
+ .unwrap();295
+ let one = storage_key(&store.cipher, "one");296
+ let two = storage_key(&store.cipher, "two");297
+ let mut swapped = store.snapshot.rows.clone();298
+ let first = swapped[&one].clone();299
+ swapped.insert(one.clone(), swapped[&two].clone());300
+ swapped.insert(two, first);301
+ let swapped = swapped302
+ .into_iter()303
+ .map(|(key, value)| (key, Some(value)))304
+ .collect();305
+ store306
+ .backend307
+ .commit(308
+ Some(store.snapshot.head.clone()),309
+ store.snapshot.exported.clone(),310
+ store.snapshot.head.clone(),311
+ swapped,312
+ )313
+ .await314
+ .unwrap();315
+ assert!(316
+ MentionsStore::open(&config, account(), MAX_BYTES)317
+ .await318
+ .is_err()319
+ );320
+ let mut corrupt = store.snapshot.rows[&one].clone();321
+ let middle = corrupt.len() / 2;322
+ corrupt[middle] ^= 1;323
+ let mut restored = store.snapshot.rows.clone();324
+ restored.insert(one, corrupt);325
+ let restored = restored326
+ .into_iter()327
+ .map(|(key, value)| (key, Some(value)))328
+ .collect();329
+ store330
+ .backend331
+ .commit(332
+ Some(store.snapshot.head.clone()),333
+ store.snapshot.exported.clone(),334
+ store.snapshot.head.clone(),335
+ restored,336
+ )337
+ .await338
+ .unwrap();339
+ assert!(340
+ MentionsStore::open(&config, account(), MAX_BYTES)341
+ .await342
+ .is_err()343
+ );344
+ drop(store);345
+ config.delete().await.unwrap();346
+}347
+348
+#[cfg_attr(not(target_family = "wasm"), tokio::test)]349
+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]350
+async fn unsupported_schema_fails_without_resetting_data() {351
+ let config = config();352
+ let mut store = open(&config).await;353
+ store354
+ .commit(0, vec![set("checkpoint", json!("retained"))])355
+ .await356
+ .unwrap();357
+ let header = store358
+ .cipher359
+ .encrypt_value(&Head {360
+ schema: 2,361
+ account: account(),362
+ revision: 1,363
+ records: store.snapshot.rows.len(),364
+ bytes: row_bytes(&store.snapshot.rows).unwrap(),365
+ })366
+ .unwrap();367
+ store368
+ .backend369
+ .commit(370
+ Some(store.snapshot.head.clone()),371
+ store.snapshot.exported.clone(),372
+ header.clone(),373
+ Default::default(),374
+ )375
+ .await376
+ .unwrap();377
+ assert!(378
+ MentionsStore::open(&config, account(), MAX_BYTES)379
+ .await380
+ .is_err()381
+ );382
+ assert_eq!(store.backend.load().await.unwrap().unwrap().head, header);383
+ drop(store);384
+ config.delete().await.unwrap();385
+}386
+387
+#[cfg(feature = "sqlite")]388
+#[tokio::test]389
+async fn sql_failure_rolls_back_records_and_header_together() {390
+ let config = config();391
+ let mut store = open(&config).await;392
+ let path = config.mentions_location();393
+ tokio::task::spawn_blocking(move || {394
+ rusqlite::Connection::open(path).unwrap().execute_batch(395
+ "CREATE TRIGGER reject_record BEFORE INSERT ON records WHEN NEW.key NOT LIKE '!%' BEGIN SELECT RAISE(ABORT, 'injected write failure'); END;"396
+ ).unwrap();397
+ }).await.unwrap();398
+ assert!(399
+ store400
+ .commit(401
+ 0,402
+ vec![403
+ set("entry", json!("pending")),404
+ set("checkpoint", json!("new"))405
+ ]406
+ )407
+ .await408
+ .is_err()409
+ );410
+ assert_eq!(store.revision(), 0);411
+ drop(store);412
+ let restored = open(&config).await;413
+ assert_eq!(restored.revision(), 0);414
+ assert!(restored.read::<Value>("entry").unwrap().is_none());415
+ assert!(restored.read::<Value>("checkpoint").unwrap().is_none());416
+ drop(restored);417
+ config.delete().await.unwrap();418
+}@@ -0,0 +1,127 @@
1
+use ruma::{OwnedEventId, OwnedRoomId, OwnedUserId};2
+#[cfg(feature = "serde")]3
+use serde::{Deserialize, Serialize};4
+5
+use crate::{EventContext, TimelineItem};6
+7
+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]8
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]9
+pub struct MentionKey {10
+ pub room_id: OwnedRoomId,11
+ pub event_id: OwnedEventId,12
+}13
+14
+#[derive(Clone, Debug, Eq, PartialEq)]15
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]16
+pub enum MentionReason {17
+ Direct,18
+ Reply,19
+ Room,20
+}21
+22
+#[derive(Clone, Debug, Eq, PartialEq)]23
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]24
+pub enum MentionCaptureState {25
+ Pending,26
+ Partial,27
+ Ready,28
+ Unavailable,29
+ Failed { message: String, retry_at_ms: u64 },30
+}31
+32
+#[derive(Clone, Debug, Eq, PartialEq)]33
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]34
+pub struct MentionEntry {35
+ pub key: MentionKey,36
+ pub timestamp_ms: u64,37
+ pub discovered_at_ms: u64,38
+ pub discovery_revision: u64,39
+ pub revision: u64,40
+ pub event: TimelineItem,41
+ pub reasons: Vec<MentionReason>,42
+ pub read: bool,43
+ pub capture: MentionCaptureState,44
+ pub room_name: Option<String>,45
+ pub notification_suppressed: bool,46
+}47
+48
+#[derive(Clone, Debug, Eq, PartialEq)]49
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]50
+pub struct MentionCoverage {51
+ pub earliest_timestamp_ms: Option<u64>,52
+ pub historical_supported: Option<bool>,53
+ pub recovering: bool,54
+ pub incomplete_history: bool,55
+ pub gaps: Vec<MentionGap>,56
+ pub truncated: bool,57
+ pub unresolved_candidates: usize,58
+}59
+60
+impl Default for MentionCoverage {61
+ fn default() -> Self {62
+ Self {63
+ earliest_timestamp_ms: None,64
+ historical_supported: None,65
+ recovering: false,66
+ incomplete_history: true,67
+ gaps: Vec::new(),68
+ truncated: false,69
+ unresolved_candidates: 0,70
+ }71
+ }72
+}73
+74
+#[derive(Clone, Debug, Eq, PartialEq)]75
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]76
+pub struct MentionGap {77
+ pub room_id: Option<OwnedRoomId>,78
+ pub reason: String,79
+ pub token: Option<String>,80
+}81
+82
+#[derive(Clone, Debug, Eq, PartialEq)]83
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]84
+pub struct MentionsBoundary {85
+ pub account: OwnedUserId,86
+ pub revision: u64,87
+}88
+89
+#[derive(Clone, Debug, Eq, PartialEq)]90
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]91
+pub struct MentionsPage {92
+ pub entries: Vec<MentionEntry>,93
+ pub next_cursor: Option<String>,94
+ pub end_reached: bool,95
+ pub coverage: MentionCoverage,96
+ pub unread_count: usize,97
+ pub revision: u64,98
+ pub boundary: MentionsBoundary,99
+}100
+101
+#[derive(Clone, Debug, Eq, PartialEq)]102
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]103
+pub struct SavedMentionContext {104
+ pub key: MentionKey,105
+ pub state: MentionCaptureState,106
+ pub context: Option<EventContext>,107
+ pub revision: u64,108
+}109
+110
+#[derive(Clone, Debug, Eq, PartialEq)]111
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]112
+pub struct MentionsUpdate {113
+ pub snapshot: MentionsPage,114
+ pub upserts: Vec<MentionEntry>,115
+ pub removed: Vec<MentionKey>,116
+}117
+118
+#[derive(Clone, Debug, Eq, PartialEq)]119
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]120
+pub enum MentionsCommand {121
+ Page { cursor: Option<String>, limit: u16 },122
+ Context(MentionKey),123
+ RetryContext(MentionKey),124
+ SetRead { keys: Vec<MentionKey>, read: bool },125
+ ReadThrough(MentionsBoundary),126
+ Recover,127
+}@@ -45,6 +45,7 @@ pub struct State {
4545
pub encryption: Option<EncryptionState>,4646
pub verification: Option<VerificationState>,4747
pub verification_target: Option<VerificationTarget>,48
+ pub mentions: Option<crate::MentionsPage>,4849
pub rooms: Vector<RoomSummary>,4950
pub image_packs: ImagePacks,5051
pub timelines: imbl::HashMap<TimelineId, TimelineState>,@@ -121,6 +122,7 @@ pub enum StateChange {
121122
Encryption(EncryptionState),122123
Verification(VerificationState),123124
VerificationTarget(VerificationTarget),125
+ Mentions(crate::MentionsUpdate),124126
/// room modes arrive with the `Rooms` that follows125127
NotificationSettings,126128
Rooms(Vector<RoomSummary>),@@ -200,6 +202,7 @@ impl Mirror {
200202
encryption: handle.encryption_state(),201203
verification: None,202204
verification_target: None,205
+ mentions: None,203206
rooms: Vector::new(),204207
image_packs: ImagePacks::default(),205208
timelines: imbl::HashMap::new(),@@ -262,6 +265,7 @@ impl Mirror {
262265
Some(StateChange::SessionSaved)263266
}),264267
Event::SessionAccessRejected { soft_logout } => self.publish(|state| {268
+ state.mentions = None;265269
state.session = match soft_logout {266270
Some(device_id) => SessionStatus::SoftLoggedOut { device_id },267271
None => SessionStatus::AccessRevoked,@@ -269,6 +273,7 @@ impl Mirror {
269273
Some(StateChange::Session(state.session.clone()))270274
}),271275
Event::LoggedOut => self.publish(|state| {276
+ state.mentions = None;272277
state.session = SessionStatus::LoggedOut;273278
Some(StateChange::Session(SessionStatus::LoggedOut))274279
}),@@ -290,6 +295,25 @@ impl Mirror {
290295
self.publish(|_| Some(StateChange::NotificationSettings));291296
}292297
Event::RoomsChanged => self.refetch_rooms(),298
+ Event::MentionsChanged(update) => self.publish(|state| {299
+ if state.session != SessionStatus::Active300
+ || state301
+ .user_id302
+ .as_ref()303
+ .is_some_and(|account| account != &update.snapshot.boundary.account)304
+ {305
+ return None;306
+ }307
+ if state308
+ .mentions309
+ .as_ref()310
+ .map_or(false, |current| update.snapshot.revision < current.revision)311
+ {312
+ return None;313
+ }314
+ state.mentions = Some(update.snapshot.clone());315
+ Some(StateChange::Mentions(update))316
+ }),293317
Event::ImagePacksChanged => self.refetch_image_packs(),294318
Event::EncryptionChanged(encryption) => self.publish(|state| {295319
state.encryption = Some(encryption.clone());@@ -24,6 +24,7 @@ pub enum Command {
2424
Room(RoomCommand),2525
Timeline(TimelineCommand),2626
EventContext(EventContextCommand),27
+ Mentions(crate::MentionsCommand),2728
Media(MediaCommand),2829
Encryption(EncryptionCommand),2930
Notification(NotificationCommand),@@ -665,6 +666,8 @@ pub enum CommandOutput {
665666
},666667
EventContext(EventContext),667668
EventContextPage(EventContextPage),669
+ MentionsPage(crate::MentionsPage),670
+ MentionContext(crate::SavedMentionContext),668671
}669672
670673
#[derive(Clone, Debug, Eq, PartialEq)]@@ -885,6 +888,7 @@ pub enum Event {
885888
NotificationSettingsChanged,886889
VerificationTargetChanged(VerificationTarget),887890
RoomsChanged,891
+ MentionsChanged(crate::MentionsUpdate),888892
/// an image pack came, went or changed somewhere. ask `image_packs` again889893
ImagePacksChanged,890894
/// replaces whatever you had for this timeline. every diff after it applies to it@@ -44,6 +44,7 @@ impl MediaScope {
4444
}4545
4646
#[derive(Clone)]47
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]4748
pub(crate) struct MediaRegistration {4849
pub(crate) id: MediaId,4950
pub(crate) kind: MediaKind,@@ -52,6 +53,7 @@ pub(crate) struct MediaRegistration {
5253
}5354
5455
#[derive(Clone)]56
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]5557
pub(crate) struct MediaSourceInfo {5658
pub(crate) source: MediaSource,5759
pub(crate) mimetype: Option<String>,@@ -208,7 +210,7 @@ pub(super) fn media_id(identity: &str, suffix: &str) -> MediaId {
208210
MediaId::new(format!("{identity}#{suffix}"))209211
}210212
211
-fn media_sticker_registration(213
+pub(super) fn media_sticker_registration(212214
identity: &str,213215
content: &StickerEventContent,214216
) -> Option<MediaRegistration> {@@ -227,7 +229,10 @@ fn media_sticker_registration(
227229
})228230
}229231
230
-fn media_registration(identity: &str, message: &MessageType) -> Option<MediaRegistration> {232
+pub(super) fn media_registration(233
+ identity: &str,234
+ message: &MessageType,235
+) -> Option<MediaRegistration> {231236
match message {232237
MessageType::Image(content) => media_registration_with_info(233238
identity,@@ -2,6 +2,7 @@ mod image_pack;
22
mod link_preview;33
mod media;44
mod member;5
+mod observed;56
mod room;67
mod timeline;78
@@ -11,6 +12,7 @@ pub(crate) use image_pack::{
1112
pub(crate) use link_preview::{ROOM_PREVIEW_URLS, server_preview, url_previews_allowed};1213
pub(crate) use media::{MediaRegistration, MediaScope, MediaSourceInfo};1314
pub(crate) use member::member_summary;15
+pub(crate) use observed::normalise_observed;1416
pub(crate) use room::notification_mode;1517
pub(crate) use room::{room_summaries, space_child, stored_space_order};1618
pub(crate) use timeline::{message_kind, normalise_timeline, normalise_timeline_diffs};@@ -0,0 +1,270 @@
1
+use ruma::{2
+ OwnedEventId, UserId,3
+ events::{4
+ poll::{start::PollStartEventContent, unstable_start::UnstablePollStartEventContent},5
+ room::message::{6
+ MessageType, Relation, RoomMessageEventContent, RoomMessageEventContentWithoutRelation,7
+ },8
+ sticker::StickerEventContent,9
+ },10
+};11
+use serde_json::Value;12
+13
+use super::{14
+ link_preview::sender_previews,15
+ media::{16
+ MediaRegistration, media_descriptor_for_sticker, media_registration,17
+ media_sticker_registration,18
+ },19
+ timeline::formatted_body,20
+};21
+use crate::{22
+ Message, MessageMentions, ReplyDetails, ReplyReference, SendState, Sender, TimelineContent,23
+ TimelineEvent, TimelineItem, TimelineItemId, UtdCause,24
+};25
+26
+pub(crate) fn normalise_observed(27
+ value: &Value,28
+ current_user_id: Option<&UserId>,29
+ identity: &str,30
+) -> Option<(TimelineItem, Vec<MediaRegistration>)> {31
+ let event_id: OwnedEventId = serde_json::from_value(value.get("event_id")?.clone()).ok()?;32
+ let sender_id: ruma::OwnedUserId = serde_json::from_value(value.get("sender")?.clone()).ok()?;33
+ let timestamp_ms = value.get("origin_server_ts")?.as_u64()?;34
+ let event_type = value.get("type")?.as_str()?.to_owned();35
+ let own = current_user_id.is_some_and(|user_id| user_id.as_str() == sender_id.as_str());36
+ let redacted = value37
+ .get("unsigned")38
+ .and_then(|unsigned| unsigned.get("redacted_because"))39
+ .is_some();40
+ let mut media = Vec::new();41
+ let (content, reply_to, thread_root) = if redacted {42
+ (TimelineContent::Redacted, None, None)43
+ } else if event_type == "m.room.message" {44
+ match value45
+ .get("content")46
+ .cloned()47
+ .and_then(|content| serde_json::from_value::<RoomMessageEventContent>(content).ok())48
+ {49
+ Some(mut message) => {50
+ let (reply_to, thread_root) = relation_refs(message.relates_to.as_ref());51
+ let relation_replacement = match &message.relates_to {52
+ Some(Relation::Replacement(replacement)) => {53
+ Some(replacement.new_content.clone())54
+ }55
+ _ => None,56
+ };57
+ let bundled_replacement = value58
+ .get("unsigned")59
+ .and_then(|unsigned| unsigned.get("m.relations"))60
+ .and_then(|relations| relations.get("m.replace"))61
+ .and_then(|edit| {62
+ let edit_sender = edit.get("sender")?.as_str()?;63
+ if edit_sender != sender_id.as_str() {64
+ return None;65
+ }66
+ edit.get("content")?.get("m.new_content").cloned()67
+ })68
+ .and_then(|content| {69
+ serde_json::from_value::<RoomMessageEventContentWithoutRelation>(content)70
+ .ok()71
+ });72
+ let edited = relation_replacement.is_some()73
+ || bundled_replacement.is_some()74
+ || value75
+ .get("unsigned")76
+ .and_then(|unsigned| unsigned.get("macltok_edited"))77
+ .and_then(Value::as_bool)78
+ .unwrap_or(false);79
+ if let Some(replacement) = relation_replacement.or(bundled_replacement) {80
+ message.apply_replacement(replacement);81
+ }82
+ let message_type = &message.msgtype;83
+ media.extend(media_registration(identity, message_type));84
+ let previews = sender_previews(Some(identity), message_type);85
+ media.extend(86
+ previews87
+ .iter()88
+ .filter_map(|(_, registration)| registration.clone()),89
+ );90
+ (91
+ TimelineContent::Message(Message {92
+ kind: super::message_kind(message_type),93
+ body: message.body().to_owned(),94
+ geo_uri: match message_type {95
+ MessageType::Location(location) => Some(location.geo_uri.clone()),96
+ _ => None,97
+ },98
+ formatted_body: formatted_body(message_type),99
+ mentions: message.mentions.as_ref().map(|mentions| MessageMentions {100
+ user_ids: mentions.user_ids.iter().cloned().collect(),101
+ room: mentions.room,102
+ }),103
+ edited,104
+ media: super::media::media_from_message(identity, message_type),105
+ link_previews: previews.into_iter().map(|(preview, _)| preview).collect(),106
+ }),107
+ reply_to,108
+ thread_root,109
+ )110
+ }111
+ None => (TimelineContent::Other { event_type }, None, None),112
+ }113
+ } else if event_type == "m.room.encrypted" {114
+ (115
+ TimelineContent::UnableToDecrypt(UtdCause::Unknown),116
+ None,117
+ None,118
+ )119
+ } else if event_type == "m.sticker" {120
+ match value121
+ .get("content")122
+ .cloned()123
+ .and_then(|content| serde_json::from_value::<StickerEventContent>(content).ok())124
+ {125
+ Some(sticker) => {126
+ let (reply_to, thread_root) = relation_refs(sticker.relates_to.as_ref());127
+ media.extend(media_sticker_registration(identity, &sticker));128
+ (129
+ TimelineContent::Sticker {130
+ body: sticker.body.clone(),131
+ media: media_descriptor_for_sticker(identity, &sticker),132
+ },133
+ reply_to,134
+ thread_root,135
+ )136
+ }137
+ None => (TimelineContent::Other { event_type }, None, None),138
+ }139
+ } else if event_type == "m.poll.start" || event_type == "org.matrix.msc3381.poll.start" {140
+ let poll_content = value.get("content");141
+ let question = poll_content.and_then(|content| poll_question(content, &event_type));142
+ match question {143
+ Some(question) => {144
+ let (reply_to, thread_root) =145
+ poll_content.map(raw_relation_refs).unwrap_or_default();146
+ (TimelineContent::Poll { question }, reply_to, thread_root)147
+ }148
+ None => (TimelineContent::Other { event_type }, None, None),149
+ }150
+ } else if let Some(state_key) = value.get("state_key").and_then(Value::as_str) {151
+ (152
+ TimelineContent::OtherState {153
+ event_type,154
+ state_key: state_key.to_owned(),155
+ },156
+ None,157
+ None,158
+ )159
+ } else {160
+ (TimelineContent::Other { event_type }, None, None)161
+ };162
+ let event = TimelineEvent {163
+ item_id: TimelineItemId::new(identity.to_owned()),164
+ event_id: Some(event_id),165
+ transaction_id: None,166
+ sender: Sender {167
+ user_id: sender_id,168
+ display_name: None,169
+ avatar: None,170
+ name_ambiguous: false,171
+ },172
+ timestamp_ms,173
+ own,174
+ send_state: SendState::Sent,175
+ content,176
+ reply_to: reply_to.map(|event_id| ReplyReference {177
+ event_id,178
+ details: ReplyDetails::Pending,179
+ }),180
+ thread_root,181
+ thread_summary: None,182
+ reactions: Vec::new(),183
+ read_by: Vec::new(),184
+ editable: false,185
+ };186
+ Some((TimelineItem::Event(Box::new(event)), media))187
+}188
+189
+fn relation_refs<C>(190
+ relation: Option<&Relation<C>>,191
+) -> (Option<OwnedEventId>, Option<OwnedEventId>) {192
+ match relation {193
+ Some(Relation::Reply(reply)) => (Some(reply.in_reply_to.event_id.clone()), None),194
+ Some(Relation::Thread(thread)) => (195
+ thread196
+ .in_reply_to197
+ .as_ref()198
+ .map(|reply| reply.event_id.clone()),199
+ Some(thread.event_id.clone()),200
+ ),201
+ _ => (None, None),202
+ }203
+}204
+205
+fn raw_relation_refs(content: &Value) -> (Option<OwnedEventId>, Option<OwnedEventId>) {206
+ let Some(relation) = content.get("m.relates_to") else {207
+ return (None, None);208
+ };209
+ let reply_to = relation210
+ .get("m.in_reply_to")211
+ .and_then(|reply| reply.get("event_id"))212
+ .and_then(Value::as_str)213
+ .and_then(|event_id| event_id.parse().ok());214
+ let thread_root = (relation.get("rel_type").and_then(Value::as_str) == Some("m.thread"))215
+ .then(|| relation.get("event_id").and_then(Value::as_str))216
+ .flatten()217
+ .and_then(|event_id| event_id.parse().ok());218
+ (reply_to, thread_root)219
+}220
+221
+fn poll_question(content: &Value, event_type: &str) -> Option<String> {222
+ let typed_question = match event_type {223
+ "m.poll.start" => serde_json::from_value::<PollStartEventContent>(content.clone())224
+ .ok()225
+ .and_then(|poll| poll.poll.question.text.find_plain().map(str::to_owned)),226
+ "org.matrix.msc3381.poll.start" => {227
+ serde_json::from_value::<UnstablePollStartEventContent>(content.clone())228
+ .ok()229
+ .map(|poll| poll.poll_start().question.text.clone())230
+ }231
+ _ => None,232
+ };233
+ typed_question234
+ .or_else(|| {235
+ content236
+ .get("m.poll")237
+ .and_then(|poll| poll.get("question"))238
+ .and_then(|question| question.get("m.text"))239
+ .and_then(generic_text)240
+ })241
+ .or_else(|| {242
+ content243
+ .get("org.matrix.msc3381.poll.start")244
+ .and_then(|poll| poll.get("question"))245
+ .and_then(|question| question.get("org.matrix.msc1767.text"))246
+ .and_then(generic_text)247
+ })248
+ .or_else(|| {249
+ ["org.matrix.msc1767.text", "m.text", "question", "body"]250
+ .iter()251
+ .find_map(|key| content.get(*key).and_then(generic_text))252
+ })253
+}254
+255
+fn generic_text(value: &Value) -> Option<String> {256
+ match value {257
+ Value::String(text) => Some(text.clone()),258
+ Value::Array(representations) => representations259
+ .iter()260
+ .find(|representation| {261
+ representation.get("mimetype").and_then(Value::as_str) == Some("text/plain")262
+ })263
+ .or_else(|| representations.first())264
+ .and_then(|representation| representation.get("body"))265
+ .and_then(Value::as_str)266
+ .map(str::to_owned),267
+ Value::Object(_) => value.get("body").and_then(Value::as_str).map(str::to_owned),268
+ _ => None,269
+ }270
+}@@ -436,7 +436,7 @@ fn reactions(event: &EventTimelineItem, current_user_id: Option<&UserId>) -> Vec
436436
.collect()437437
}438438
439
-fn formatted_body(message: &MessageType) -> Option<String> {439
+pub(super) fn formatted_body(message: &MessageType) -> Option<String> {440440
let formatted = match message {441441
MessageType::Text(content) => content.formatted.as_ref(),442442
MessageType::Emote(content) => content.formatted.as_ref(),@@ -473,11 +473,12 @@ pub(crate) fn delete_sqlite_store(path: &Path) -> std::io::Result<()> {
473473
use std::{fs, io::ErrorKind};474474
475475
// matrix-sdk-sqlite 0.18's database names, which it keeps private476
- const DATABASES: [&str; 4] = [476
+ const DATABASES: [&str; 5] = [477477
"matrix-sdk-state.sqlite3",478478
"matrix-sdk-crypto.sqlite3",479479
"matrix-sdk-event-cache.sqlite3",480480
"matrix-sdk-media.sqlite3",481
+ "macltok-mentions.sqlite3",481482
];482483
for database in DATABASES {483484
for suffix in ["", "-wal", "-shm", "-journal"] {@@ -507,6 +508,7 @@ pub(crate) async fn delete_indexeddb_store(name: &str) -> Result<(), DeleteError
507508
"::matrix-sdk-crypto-meta",508509
"::event_cache",509510
"::media",511
+ "::macltok-mentions",510512
] {511513
delete_database(&format!("{name}{suffix}")).await?;512514
}@@ -29,14 +29,15 @@ use tokio::sync::{OnceCell, mpsc, oneshot, watch};
2929
use crate::{3030
AccountCommand, AttachmentUpload, Command, CommandOutput, ConnectionState, CreateRoomOptions,3131
EncryptionCommand, EncryptionState, Error, ErrorKind, Event, EventContext, EventContextCommand,32
- EventContextPage, EventRef, ImagePackSource,33
- ImagePacks, InteractiveAuth, InteractiveAuthOutcome, LifecycleCommand, LinkPreview, Media,34
- MediaCommand, MediaId, MediaLocation, MediaSize, MemberFilter, MemberSummary, MessageContent,35
- MessageEditContent, NotificationCommand, OwnDevice, PackCommand, PackImageInfo, PackUsage,36
- Patch, PowerLevelChanges, Profile, PushNotification, PushNotificationEvent,37
- PushNotificationStatus, RoomCommand, RoomListFilter, RoomNotificationMode,38
- RoomNotificationSettings, RoomPowerLevels, RoomSummary, SpaceChild, TimelineCommand,39
- TimelineId, VerificationAction, VerificationTarget,32
+ EventContextPage, EventRef, ImagePackSource, ImagePacks, InteractiveAuth,33
+ InteractiveAuthOutcome, LifecycleCommand, LinkPreview, Media, MediaCommand, MediaId,34
+ MediaLocation, MediaSize, MemberFilter, MemberSummary, MentionKey, MentionsBoundary,35
+ MentionsCommand, MentionsPage, MessageContent, MessageEditContent, NotificationCommand,36
+ OwnDevice, PackCommand, PackImageInfo, PackUsage, Patch, PowerLevelChanges, Profile,37
+ PushNotification, PushNotificationEvent, PushNotificationStatus, RoomCommand, RoomListFilter,38
+ RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels, RoomSummary,39
+ SavedMentionContext, SpaceChild, TimelineCommand, TimelineId, VerificationAction,40
+ VerificationTarget,4041
encryption::encryption_state,4142
engine::{MediaService, StoreLock},4243
model::{PushNotificationRoom, PushNotificationSender},@@ -385,6 +386,74 @@ impl ClientHandle {
385386
}386387
}387388
389
+ pub async fn mentions_page(390
+ &self,391
+ cursor: Option<String>,392
+ limit: u32,393
+ ) -> Result<MentionsPage, Error> {394
+ match self395
+ .execute(Command::Mentions(MentionsCommand::Page {396
+ cursor,397
+ limit: limit.min(50) as u16,398
+ }))399
+ .await?400
+ {401
+ CommandOutput::MentionsPage(page) => Ok(page),402
+ output => Err(unexpected("mentions page", &output)),403
+ }404
+ }405
+406
+ pub async fn mention_context(407
+ &self,408
+ room_id: &RoomId,409
+ event_id: &EventId,410
+ ) -> Result<SavedMentionContext, Error> {411
+ self.mention_context_command(MentionsCommand::Context(MentionKey {412
+ room_id: room_id.to_owned(),413
+ event_id: event_id.to_owned(),414
+ }))415
+ .await416
+ }417
+418
+ pub async fn retry_mention_context(419
+ &self,420
+ room_id: &RoomId,421
+ event_id: &EventId,422
+ ) -> Result<SavedMentionContext, Error> {423
+ self.mention_context_command(MentionsCommand::RetryContext(MentionKey {424
+ room_id: room_id.to_owned(),425
+ event_id: event_id.to_owned(),426
+ }))427
+ .await428
+ }429
+430
+ async fn mention_context_command(431
+ &self,432
+ command: MentionsCommand,433
+ ) -> Result<SavedMentionContext, Error> {434
+ match self.execute(Command::Mentions(command)).await? {435
+ CommandOutput::MentionContext(context) => Ok(context),436
+ output => Err(unexpected("mention context", &output)),437
+ }438
+ }439
+440
+ pub async fn set_mentions_read(&self, keys: Vec<MentionKey>, read: bool) -> Result<(), Error> {441
+ self.run(Command::Mentions(MentionsCommand::SetRead { keys, read }))442
+ .await443
+ }444
+445
+ pub async fn mark_mentions_read_through(446
+ &self,447
+ boundary: MentionsBoundary,448
+ ) -> Result<(), Error> {449
+ self.run(Command::Mentions(MentionsCommand::ReadThrough(boundary)))450
+ .await451
+ }452
+453
+ pub async fn recover_mentions(&self) -> Result<(), Error> {454
+ self.run(Command::Mentions(MentionsCommand::Recover)).await455
+ }456
+388457
pub async fn send_message(389458
&self,390459
timeline_id: TimelineId,@@ -0,0 +1,155 @@
1
+use std::{sync::Arc, time::Duration};2
+3
+use matrix_sdk::{4
+ Client, Room,5
+ ruma::{api::client::sync::sync_events::v5, events::AnySyncTimelineEvent, room_id, serde::Raw},6
+ sliding_sync::{SlidingSyncList, SlidingSyncMode},7
+ test_utils::mocks::MatrixMockServer,8
+};9
+use serde_json::{Value, json};10
+use tokio::sync::{Notify, Semaphore};11
+12
+async fn saved_position(client: &Client) -> Option<String> {13
+ let machine = client.olm_machine_for_testing().await;14
+ let key = format!(15
+ "sliding_sync_store::mentions-spike::{}::instance",16
+ client.user_id().unwrap()17
+ );18
+ let saved = machine19
+ .as_ref()20
+ .unwrap()21
+ .store()22
+ .get_custom_value(&key)23
+ .await24
+ .unwrap()?;25
+ serde_json::from_slice::<Value>(&saved).unwrap()["pos"]26
+ .as_str()27
+ .map(ToOwned::to_owned)28
+}29
+30
+async fn server_with_mention() -> (MatrixMockServer, Client) {31
+ let server = MatrixMockServer::new().await;32
+ let client = server.client_builder().build().await;33
+ let mut response = v5::Response::new("position-1".to_owned());34
+ response.rooms.insert(35
+ room_id!("!checkpoint:localhost").to_owned(),36
+ serde_json::from_value(json!({37
+ "initial": true,38
+ "required_state": [],39
+ "timeline": [{40
+ "type": "m.room.message",41
+ "event_id": "$checkpoint-mention",42
+ "sender": "@sender:localhost",43
+ "origin_server_ts": 1,44
+ "content": {45
+ "msgtype": "m.text",46
+ "body": "checkpoint candidate",47
+ "m.mentions": { "user_ids": [client.user_id().unwrap()] }48
+ }49
+ }]50
+ }))51
+ .unwrap(),52
+ );53
+ server.mock_sliding_sync().ok(response).mount().await;54
+ (server, client)55
+}56
+57
+async fn new_sync(client: &Client) -> matrix_sdk::sliding_sync::SlidingSync {58
+ client59
+ .sliding_sync("mentions-spike")60
+ .unwrap()61
+ .share_pos()62
+ .add_list(63
+ SlidingSyncList::builder("mentions")64
+ .sync_mode(SlidingSyncMode::new_selective().add_range(0..=0)),65
+ )66
+ .build()67
+ .await68
+ .unwrap()69
+}70
+71
+#[tokio::test]72
+async fn awaited_event_handler_fences_position_even_after_caller_cancellation() {73
+ let (server, client) = server_with_mention().await;74
+75
+ let entered = Arc::new(Notify::new());76
+ let committed = Arc::new(Semaphore::new(0));77
+ let handler = client.add_event_handler({78
+ let entered = entered.clone();79
+ let committed = committed.clone();80
+ move |event: Raw<AnySyncTimelineEvent>, room: Room| {81
+ let entered = entered.clone();82
+ let committed = committed.clone();83
+ async move {84
+ assert_eq!(room.room_id().as_str(), "!checkpoint:localhost");85
+ assert_eq!(86
+ event.get_field::<String>("event_id").unwrap().as_deref(),87
+ Some("$checkpoint-mention")88
+ );89
+ entered.notify_one();90
+ committed.acquire().await.unwrap().forget();91
+ }92
+ }93
+ });94
+ let sync = new_sync(&client).await;95
+ let processing = tokio::spawn(async move { sync.sync_once().await });96
+ tokio::time::timeout(Duration::from_secs(5), entered.notified())97
+ .await98
+ .unwrap();99
+ assert!(!processing.is_finished());100
+ assert!(saved_position(&client).await.is_none());101
+ processing.abort();102
+ assert!(processing.await.unwrap_err().is_cancelled());103
+ assert!(saved_position(&client).await.is_none());104
+105
+ committed.add_permits(1);106
+ tokio::time::timeout(Duration::from_secs(5), async {107
+ while saved_position(&client).await.as_deref() != Some("position-1") {108
+ tokio::task::yield_now().await;109
+ }110
+ })111
+ .await112
+ .unwrap();113
+ client.remove_event_handler(handler);114
+115
+ let restored = new_sync(&client).await;116
+ tokio::time::timeout(Duration::from_secs(5), restored.sync_once())117
+ .await118
+ .unwrap()119
+ .unwrap();120
+ let requests: Vec<_> = server121
+ .received_requests()122
+ .await123
+ .unwrap()124
+ .into_iter()125
+ .filter_map(|request| {126
+ let body = serde_json::from_slice::<Value>(&request.body).ok()?;127
+ let pos = request128
+ .url129
+ .query_pairs()130
+ .find(|(key, _)| key == "pos")131
+ .map(|(_, value)| value.into_owned());132
+ Some((body, pos))133
+ })134
+ .filter(|(body, _)| body["conn_id"] == "mentions-spike")135
+ .collect();136
+ assert_eq!(requests.len(), 2);137
+ assert!(requests[0].1.is_none());138
+ assert_eq!(requests[1].1.as_deref(), Some("position-1"));139
+}140
+141
+#[tokio::test]142
+async fn event_handler_error_does_not_prevent_position_persistence() {143
+ let (_server, client) = server_with_mention().await;144
+ let handler =145
+ client.add_event_handler(|_event: Raw<AnySyncTimelineEvent>, _room: Room| async {146
+ Err::<(), _>("the candidate transaction failed")147
+ });148
+ let sync = new_sync(&client).await;149
+ tokio::time::timeout(Duration::from_secs(5), sync.sync_once())150
+ .await151
+ .unwrap()152
+ .unwrap();153
+ assert_eq!(saved_position(&client).await.as_deref(), Some("position-1"));154
+ client.remove_event_handler(handler);155
+}@@ -0,0 +1,128 @@
1
+use std::time::Duration;2
+3
+use futures_util::StreamExt;4
+use matrix_sdk::{5
+ deserialized_responses::TimelineEventKind, event_cache::RedecryptorReport,6
+ test_utils::mocks::MatrixMockServer,7
+};8
+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};9
+use ruma::{RoomVersionId, device_id, event_id, room_id, user_id};10
+use serde_json::json;11
+12
+#[tokio::test]13
+async fn late_key_exposes_a_mention_without_a_ui_timeline() {14
+ let server = MatrixMockServer::new().await;15
+ server.mock_crypto_endpoints_preset().await;16
+ let alice_id = user_id!("@alice:localhost");17
+ let bob_id = user_id!("@bob:localhost");18
+ let room_id = room_id!("!mentions-encrypted:localhost");19
+ let event_id = event_id!("$late-mention");20
+ let alice = server21
+ .client_builder_for_crypto_end_to_end(alice_id, device_id!("ALICEDEVICE"))22
+ .build()23
+ .await;24
+ let bob = server25
+ .client_builder_for_crypto_end_to_end(bob_id, device_id!("BOBDEVICE"))26
+ .build()27
+ .await;28
+ bob.event_cache().subscribe().unwrap();29
+ server.exchange_e2ee_identities(&alice, &bob).await;30
+31
+ let factory = EventFactory::new().room(room_id).sender(alice_id);32
+ let joined = JoinedRoomBuilder::new(room_id)33
+ .add_state_event(factory.create(alice_id, RoomVersionId::V1))34
+ .add_state_event(factory.room_encryption())35
+ .add_state_event(factory.member(alice_id))36
+ .add_state_event(factory.member(bob_id));37
+ server.sync_room(&alice, joined.clone()).await;38
+ server.sync_room(&bob, joined).await;39
+40
+ let room = alice.get_room(room_id).unwrap();41
+ let (encrypted_event, send) = server.mock_room_send().ok_with_capture(event_id, alice_id);42
+ let _send = send.mock_once().mount_as_scoped().await;43
+ let (_keys, room_key) = server.mock_capture_put_to_device(alice_id).await;44
+ server45
+ .mock_get_members()46
+ .ok(vec![47
+ factory.member(alice_id).into_raw(),48
+ factory.member(bob_id).into_raw(),49
+ ])50
+ .mock_once()51
+ .mount()52
+ .await;53
+ room.send_raw(54
+ "m.room.message",55
+ json!({56
+ "msgtype": "m.text",57
+ "body": "a private mention",58
+ "m.mentions": { "user_ids": [bob_id] }59
+ }),60
+ )61
+ .await62
+ .unwrap();63
+ let encrypted_event = encrypted_event.await.unwrap();64
+ let room_key = room_key.await;65
+66
+ let mut sync_updates = bob.subscribe_to_all_room_updates();67
+ let mut cache_updates = bob.event_cache().subscribe_to_room_generic_updates();68
+ let mut reports = bob.event_cache().subscribe_to_decryption_reports();69
+ server70
+ .sync_room(71
+ &bob,72
+ JoinedRoomBuilder::new(room_id).add_timeline_event(encrypted_event),73
+ )74
+ .await;75
+ let update = sync_updates.recv().await.unwrap();76
+ let event = &update.joined[room_id].timeline.events[0];77
+ assert!(matches!(78
+ event.kind,79
+ TimelineEventKind::UnableToDecrypt { .. }80
+ ));81
+ tokio::time::timeout(Duration::from_secs(5), async {82
+ while cache_updates.recv().await.unwrap().room_id != room_id {}83
+ })84
+ .await85
+ .unwrap();86
+87
+ server88
+ .mock_sync()89
+ .ok_and_run(&bob, |sync| {90
+ sync.add_to_device_event(room_key.deserialize_as().unwrap());91
+ })92
+ .await;93
+ tokio::time::timeout(Duration::from_secs(5), async {94
+ while let Some(report) = reports.next().await {95
+ if let Ok(RedecryptorReport::ResolvedUtds {96
+ room_id: resolved,97
+ events,98
+ }) = report99
+ {100
+ if resolved == room_id && events.contains(event_id) {101
+ return;102
+ }103
+ }104
+ }105
+ panic!("decryption reports ended before the mention was decrypted");106
+ })107
+ .await108
+ .unwrap();109
+110
+ let room = bob.get_room(room_id).unwrap();111
+ let (cache, _cache_handles) = room.event_cache().await.unwrap();112
+ let decrypted = cache.find_event(event_id).await.unwrap().unwrap();113
+ assert!(matches!(decrypted.kind, TimelineEventKind::Decrypted(_)));114
+ let content = decrypted115
+ .raw()116
+ .get_field::<serde_json::Value>("content")117
+ .unwrap()118
+ .unwrap();119
+ assert_eq!(content["body"], "a private mention");120
+ assert_eq!(content["m.mentions"]["user_ids"], json!([bob_id]));121
+ let push_actions = room122
+ .event_push_actions(decrypted.raw())123
+ .await124
+ .unwrap()125
+ .unwrap();126
+ assert!(push_actions.iter().any(|action| action.should_notify()));127
+ assert_eq!(cache.events().await.unwrap().len(), 1);128
+}@@ -0,0 +1,232 @@
1
+use matrix_sdk::{2
+ Room,3
+ ruma::{4
+ UInt,5
+ api::client::push::get_notifications::v3,6
+ event_id,7
+ events::{AnySyncTimelineEvent, Mentions},8
+ owned_user_id,9
+ push::{Action, EventMatchConditionData, NewConditionalPushRule, PushCondition, Ruleset},10
+ room_id,11
+ serde::Raw,12
+ },13
+ test_utils::mocks::MatrixMockServer,14
+};15
+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};16
+use serde_json::json;17
+use std::time::Duration;18
+use wiremock::{19
+ Mock, ResponseTemplate,20
+ matchers::{method, path, query_param},21
+};22
+23
+fn explicit_mention(24
+ room_id: &matrix_sdk::ruma::RoomId,25
+ event_id: &matrix_sdk::ruma::EventId,26
+ sender: &matrix_sdk::ruma::UserId,27
+ mentioned: &matrix_sdk::ruma::UserId,28
+) -> Raw<AnySyncTimelineEvent> {29
+ EventFactory::new()30
+ .room(room_id)31
+ .text_msg("hello")32
+ .mentions(Mentions::with_user_ids([mentioned.to_owned()]))33
+ .event_id(event_id)34
+ .sender(sender)35
+ .into_raw_sync()36
+}37
+38
+fn assert_mention_actions(actions: &[Action]) {39
+ assert!(40
+ actions41
+ .iter()42
+ .any(|action| matches!(action, Action::Notify))43
+ );44
+ assert!(actions.iter().any(Action::is_highlight));45
+}46
+47
+async fn sync_room_with_mention(48
+ server: &MatrixMockServer,49
+ client: &matrix_sdk::Client,50
+ room_id: &matrix_sdk::ruma::RoomId,51
+ event_id: &matrix_sdk::ruma::EventId,52
+) {53
+ let user_id = client.user_id().unwrap();54
+ let factory = EventFactory::new().room(room_id);55
+ let sender = owned_user_id!("@alice:localhost");56
+ let event = explicit_mention(room_id, event_id, &sender, user_id);57
+ let rules = Ruleset::server_default(user_id);58
+59
+ server60
+ .mock_sync()61
+ .ok_and_run(client, |builder| {62
+ builder.add_global_account_data(factory.push_rules(rules));63
+ builder.add_joined_room(64
+ JoinedRoomBuilder::new(room_id)65
+ .add_state_event(factory.member(user_id))66
+ .add_timeline_event(event),67
+ );68
+ })69
+ .await;70
+}71
+72
+async fn actions_for(room: &Room, event: &Raw<AnySyncTimelineEvent>) -> Option<Vec<Action>> {73
+ room.event_push_actions(event).await.unwrap()74
+}75
+76
+#[tokio::test]77
+async fn joined_room_update_exposes_mention_actions_without_opening_a_ui_timeline() {78
+ let server = MatrixMockServer::new().await;79
+ let client = server.client_builder().build().await;80
+ let room_id = room_id!("!mentions:localhost");81
+ let event_id = event_id!("$mention:localhost");82
+ let mut updates = client.subscribe_to_all_room_updates();83
+84
+ sync_room_with_mention(&server, &client, room_id, event_id).await;85
+86
+ let room_updates = tokio::time::timeout(Duration::from_secs(2), updates.recv())87
+ .await88
+ .unwrap()89
+ .unwrap();90
+ let joined = room_updates91
+ .joined92
+ .get(room_id)93
+ .expect("the joined room update was absent");94
+ let event = joined95
+ .timeline96
+ .events97
+ .iter()98
+ .find(|event| event.event_id().as_deref().map(|id| id.as_str()) == Some(event_id.as_str()))99
+ .expect("the explicit mention was absent from the room update");100
+ let event_json: serde_json::Value =101
+ serde_json::from_str(event.kind.raw().json().get()).unwrap();102
+103
+ assert_eq!(104
+ event_json["content"]["m.mentions"]["user_ids"],105
+ json!([client.user_id().unwrap().as_str()])106
+ );107
+ assert_mention_actions(108
+ event109
+ .push_actions()110
+ .expect("push actions were not computed"),111
+ );112
+}113
+114
+#[tokio::test]115
+async fn event_cache_generic_update_finds_the_mention_without_opening_a_ui_timeline() {116
+ let server = MatrixMockServer::new().await;117
+ let client = server.client_builder().build().await;118
+ let room_id = room_id!("!cached-mentions:localhost");119
+ let event_id = event_id!("$cached-mention:localhost");120
+ client.event_cache().subscribe().unwrap();121
+ let mut updates = client.event_cache().subscribe_to_room_generic_updates();122
+123
+ sync_room_with_mention(&server, &client, room_id, event_id).await;124
+125
+ let update = tokio::time::timeout(Duration::from_secs(2), updates.recv())126
+ .await127
+ .unwrap()128
+ .unwrap();129
+ assert_eq!(update.room_id, room_id);130
+ let room = client.get_room(room_id).unwrap();131
+ let (event_cache, _drop_handles) = room.event_cache().await.unwrap();132
+ let event = event_cache.find_event(event_id).await.unwrap().unwrap();133
+ assert_eq!(134
+ event.event_id().as_deref().map(|id| id.as_str()),135
+ Some(event_id.as_str())136
+ );137
+ assert_mention_actions(138
+ event139
+ .push_actions()140
+ .expect("cached push actions were absent"),141
+ );142
+}143
+144
+#[tokio::test]145
+async fn ignored_sender_and_muted_room_actions_remain_sdk_push_rule_results() {146
+ let server = MatrixMockServer::new().await;147
+ let client = server.client_builder().build().await;148
+ let ignored_room_id = room_id!("!ignored-sender:localhost");149
+ let muted_room_id = room_id!("!muted-room:localhost");150
+ let ignored_sender = owned_user_id!("@ignored:localhost");151
+ let own_user = client.user_id().unwrap();152
+ let factory = EventFactory::new();153
+ let ignored_event_id = event_id!("$ignored-mention:localhost");154
+ let muted_event_id = event_id!("$muted-message:localhost");155
+ let ignored_event =156
+ explicit_mention(ignored_room_id, ignored_event_id, &ignored_sender, own_user);157
+ let sender = owned_user_id!("@alice:localhost");158
+ let muted_event = explicit_mention(muted_room_id, muted_event_id, &sender, own_user);159
+ let mut rules = Ruleset::server_default(own_user);160
+ rules.override_.shift_insert(161
+ 1,162
+ NewConditionalPushRule::new(163
+ "mute-room".to_owned(),164
+ vec![PushCondition::EventMatch(EventMatchConditionData::new(165
+ "room_id".to_owned(),166
+ muted_room_id.to_string(),167
+ ))],168
+ Vec::new(),169
+ )170
+ .into(),171
+ );172
+173
+ server174
+ .mock_sync()175
+ .ok_and_run(&client, |builder| {176
+ builder.add_global_account_data(factory.ignored_user_list([ignored_sender.clone()]));177
+ builder.add_global_account_data(factory.push_rules(rules));178
+ builder.add_joined_room(179
+ JoinedRoomBuilder::new(ignored_room_id)180
+ .add_state_event(EventFactory::new().room(ignored_room_id).member(own_user))181
+ .add_timeline_event(ignored_event.clone()),182
+ );183
+ builder.add_joined_room(184
+ JoinedRoomBuilder::new(muted_room_id)185
+ .add_state_event(EventFactory::new().room(muted_room_id).member(own_user))186
+ .add_timeline_event(muted_event.clone()),187
+ );188
+ })189
+ .await;190
+191
+ assert!(client.is_user_ignored(&ignored_sender).await);192
+ let ignored_room = client.get_room(ignored_room_id).unwrap();193
+ let ignored_actions = actions_for(&ignored_room, &ignored_event)194
+ .await195
+ .expect("the ignored sender's push actions could not be computed");196
+ assert_mention_actions(&ignored_actions);197
+198
+ let muted_room = client.get_room(muted_room_id).unwrap();199
+ let muted_actions = actions_for(&muted_room, &muted_event)200
+ .await201
+ .expect("the muted room's push actions could not be computed");202
+ assert!(muted_actions.is_empty());203
+}204
+205
+#[tokio::test]206
+async fn notifications_v3_request_round_trips_opaque_pagination() {207
+ let server = MatrixMockServer::new().await;208
+ let client = server.client_builder().build().await;209
+ let cursor = "opaque / cursor?x=1&y=2";210
+ let next_cursor = "next / cursor?x=3&y=4";211
+212
+ Mock::given(method("GET"))213
+ .and(path("/_matrix/client/v3/notifications"))214
+ .and(query_param("from", cursor))215
+ .and(query_param("limit", "8"))216
+ .and(query_param("only", "highlight"))217
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({218
+ "next_token": next_cursor,219
+ "notifications": [],220
+ })))221
+ .expect(1)222
+ .mount(server.server())223
+ .await;224
+225
+ let mut request = v3::Request::new();226
+ request.from = Some(cursor.to_owned());227
+ request.limit = Some(UInt::new(8).unwrap());228
+ request.only = Some("highlight".to_owned());229
+ let response = client.send(request).await.unwrap();230
+231
+ assert_eq!(response.next_token.as_deref(), Some(next_cursor));232
+}@@ -0,0 +1,882 @@
1
+mod support;2
+3
+use std::{4
+ collections::{BTreeMap, HashSet},5
+ sync::{6
+ Arc, Mutex,7
+ atomic::{AtomicUsize, Ordering},8
+ },9
+ time::Duration,10
+};11
+12
+#[cfg(feature = "sqlite")]13
+use macltok::{EncryptedSessionStore, PasswordLogin, Restore, Session, SessionStore, StoreConfig};14
+use macltok::{Engine, EngineConfig, Event, MentionsPage};15
+use matrix_sdk::{16
+ Client,17
+ ruma::{OwnedRoomId, OwnedUserId, event_id, events::Mentions, room_id, user_id},18
+ test_utils::mocks::{MatrixMockServer, RoomContextResponseTemplate},19
+};20
+use matrix_sdk_test::{ALICE, JoinedRoomBuilder, event_factory::EventFactory};21
+use ruma::push::{EventMatchConditionData, NewConditionalPushRule, PushCondition, Ruleset};22
+use serde_json::{Value, json};23
+#[cfg(feature = "sqlite")]24
+use support::receive_until;25
+use support::shut_down;26
+use wiremock::{27
+ Mock, Request, Respond, ResponseTemplate,28
+ matchers::{method, path, path_regex, query_param},29
+};30
+31
+#[cfg(feature = "serde")]32
+use macltok::{MentionCoverage, MentionsBoundary, MentionsCommand};33
+34
+const SLIDING_SYNC_PATH: &str = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync";35
+36
+struct SlidingFeed {37
+ rooms: Arc<Mutex<BTreeMap<OwnedRoomId, Vec<Value>>>>,38
+ required_state: Arc<Mutex<BTreeMap<OwnedRoomId, Vec<Value>>>>,39
+ requests: Arc<AtomicUsize>,40
+}41
+42
+impl SlidingFeed {43
+ fn new(room_id: OwnedRoomId, events: Vec<Value>) -> Self {44
+ Self {45
+ rooms: Arc::new(Mutex::new(BTreeMap::from([(room_id, events)]))),46
+ required_state: Arc::new(Mutex::new(BTreeMap::new())),47
+ requests: Arc::new(AtomicUsize::new(0)),48
+ }49
+ }50
+51
+ fn replace(&self, room_id: &ruma::RoomId, events: Vec<Value>) {52
+ self.rooms53
+ .lock()54
+ .unwrap()55
+ .insert(room_id.to_owned(), events);56
+ }57
+58
+ fn add_room(&self, room_id: &ruma::RoomId, events: Vec<Value>) {59
+ self.rooms60
+ .lock()61
+ .unwrap()62
+ .insert(room_id.to_owned(), events);63
+ }64
+65
+ fn set_required_state(&self, room_id: &ruma::RoomId, events: Vec<Value>) {66
+ self.required_state67
+ .lock()68
+ .unwrap()69
+ .insert(room_id.to_owned(), events);70
+ }71
+72
+ async fn mount(&self, server: &MatrixMockServer) {73
+ Mock::given(method("POST"))74
+ .and(path(SLIDING_SYNC_PATH))75
+ .respond_with(self.clone())76
+ .mount(server.server())77
+ .await;78
+ }79
+}80
+81
+impl Clone for SlidingFeed {82
+ fn clone(&self) -> Self {83
+ Self {84
+ rooms: self.rooms.clone(),85
+ required_state: self.required_state.clone(),86
+ requests: self.requests.clone(),87
+ }88
+ }89
+}90
+91
+impl Respond for SlidingFeed {92
+ fn respond(&self, _request: &Request) -> ResponseTemplate {93
+ let request = self.requests.fetch_add(1, Ordering::AcqRel);94
+ let required_state = self.required_state.lock().unwrap().clone();95
+ let rooms: serde_json::Map<String, Value> = self96
+ .rooms97
+ .lock()98
+ .unwrap()99
+ .iter()100
+ .map(|(room_id, events)| {101
+ (102
+ room_id.to_string(),103
+ json!({104
+ "initial": request == 0,105
+ "required_state": required_state106
+ .get(room_id)107
+ .cloned()108
+ .unwrap_or_default(),109
+ "timeline": events,110
+ }),111
+ )112
+ })113
+ .collect();114
+ ResponseTemplate::new(200)115
+ .set_body_json(json!({116
+ "txn_id": null,117
+ "pos": format!("mentions-{request}"),118
+ "lists": {},119
+ "rooms": rooms,120
+ "extensions": {},121
+ }))122
+ .set_delay(Duration::from_millis(20))123
+ }124
+}125
+126
+async fn joined_client(server: &MatrixMockServer, room_id: &ruma::RoomId) -> (Client, OwnedUserId) {127
+ let client = server.client_builder().build().await;128
+ let own_user = client.user_id().unwrap().to_owned();129
+ let room_id = room_id.to_owned();130
+ server131
+ .mock_sync()132
+ .ok_and_run(&client, {133
+ let own_user = own_user.clone();134
+ move |builder| {135
+ let factory = EventFactory::new().room(&room_id);136
+ builder.add_global_account_data(137
+ factory.push_rules(Ruleset::server_default(&own_user)),138
+ );139
+ builder.add_joined_room(140
+ JoinedRoomBuilder::new(&room_id).add_state_event(factory.member(&own_user)),141
+ );142
+ }143
+ })144
+ .await;145
+ (client, own_user)146
+}147
+148
+fn message(149
+ event_id: &str,150
+ sender: &str,151
+ timestamp: u64,152
+ body: &str,153
+ mentioned: Option<&str>,154
+) -> Value {155
+ let mut content = json!({ "msgtype": "m.text", "body": body });156
+ if let Some(mentioned) = mentioned {157
+ content["m.mentions"] = json!({ "user_ids": [mentioned] });158
+ }159
+ json!({160
+ "type": "m.room.message",161
+ "event_id": event_id,162
+ "sender": sender,163
+ "origin_server_ts": timestamp,164
+ "content": content,165
+ })166
+}167
+168
+fn room_mention(event_id: &str, sender: &str, timestamp: u64) -> Value {169
+ let mut event = message(event_id, sender, timestamp, "room mention", None);170
+ event["content"]["m.mentions"] = json!({ "room": true });171
+ event172
+}173
+174
+async fn mentions_page(175
+ events: &mut macltok::EventStream,176
+ room_id: &ruma::RoomId,177
+ predicate: impl Fn(&MentionsPage) -> bool,178
+) -> MentionsPage {179
+ mentions_page_with_diagnostics(events, room_id, predicate, None).await180
+}181
+182
+async fn mentions_page_with_diagnostics(183
+ events: &mut macltok::EventStream,184
+ room_id: &ruma::RoomId,185
+ predicate: impl Fn(&MentionsPage) -> bool,186
+ diagnostic_server: Option<&MatrixMockServer>,187
+) -> MentionsPage {188
+ let mut last_page = None;189
+ let result = tokio::time::timeout(Duration::from_secs(5), async {190
+ while let Some(event) = events.recv().await {191
+ match event {192
+ Event::TimelineReset { timeline_id, .. } if timeline_id.room_id() == room_id => {193
+ panic!("the unopened room produced a timeline reset");194
+ }195
+ Event::MentionsChanged(update) => {196
+ let snapshot = update.snapshot;197
+ let matched = predicate(&snapshot);198
+ last_page = Some(snapshot.clone());199
+ if matched {200
+ return snapshot;201
+ }202
+ }203
+ _ => {}204
+ }205
+ }206
+ panic!("event stream closed before the expected mentions page");207
+ })208
+ .await;209
+210
+ match result {211
+ Ok(page) => page,212
+ Err(_) => {213
+ let last_state = last_page.map(|page: MentionsPage| {214
+ (215
+ page.revision,216
+ page.unread_count,217
+ page.entries218
+ .iter()219
+ .map(|entry| {220
+ (221
+ entry.key.room_id.to_string(),222
+ entry.key.event_id.to_string(),223
+ entry.capture.clone(),224
+ entry.revision,225
+ )226
+ })227
+ .collect::<Vec<_>>(),228
+ )229
+ });230
+ let requests = if let Some(server) = diagnostic_server {231
+ server.server().received_requests().await.map(|requests| {232
+ requests233
+ .iter()234
+ .filter(|request| {235
+ let path = request.url.path();236
+ path.contains(room_id.as_str())237
+ && (path.contains("/context/") || path.ends_with("/messages"))238
+ })239
+ .map(|request| format!("{} {}", request.method, request.url))240
+ .collect::<Vec<_>>()241
+ })242
+ } else {243
+ None244
+ };245
+ panic!(246
+ "expected mentions update did not arrive, last state {last_state:?}, requests {requests:?}"247
+ );248
+ }249
+ }250
+}251
+252
+fn entry_ids(page: &MentionsPage) -> HashSet<String> {253
+ page.entries254
+ .iter()255
+ .map(|entry| entry.key.event_id.to_string())256
+ .collect()257
+}258
+259
+async fn wait_for_requests(feed: &SlidingFeed, minimum: usize) {260
+ tokio::time::timeout(Duration::from_secs(5), async {261
+ while feed.requests.load(Ordering::Acquire) < minimum {262
+ tokio::task::yield_now().await;263
+ }264
+ })265
+ .await266
+ .expect("the sliding sync feed did not receive another request");267
+}268
+269
+async fn mount_messages_page(270
+ server: &MatrixMockServer,271
+ room_id: &ruma::RoomId,272
+ direction: &str,273
+ token: &str,274
+ limit: u16,275
+) {276
+ Mock::given(method("GET"))277
+ .and(path_regex(format!(278
+ r"^/_matrix/client/v3/rooms/{room_id}/messages$"279
+ )))280
+ .and(query_param("dir", direction))281
+ .and(query_param("from", token))282
+ .and(move |request: &Request| {283
+ request284
+ .url285
+ .query_pairs()286
+ .find(|(key, _)| key == "limit")287
+ .map_or(limit == 10, |(_, value)| value == limit.to_string())288
+ })289
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({290
+ "start": token,291
+ "end": format!("{token}-next"),292
+ "chunk": [],293
+ "state": [],294
+ })))295
+ .expect(1)296
+ .mount(server.server())297
+ .await;298
+}299
+300
+#[tokio::test]301
+async fn unopened_room_mentions_are_deduplicated_and_read_through_uses_a_revision_boundary() {302
+ let server = MatrixMockServer::new().await;303
+ let room_id = room_id!("!mentions-inbox:localhost");304
+ let (client, own_user) = joined_client(&server, room_id).await;305
+ let sender = user_id!("@alice:localhost");306
+ let initial = vec![307
+ message(308
+ "$mention",309
+ sender.as_str(),310
+ 2_000,311
+ "direct mention",312
+ Some(own_user.as_str()),313
+ ),314
+ message(315
+ "$own-mention",316
+ own_user.as_str(),317
+ 2_100,318
+ "own event",319
+ Some(own_user.as_str()),320
+ ),321
+ message(322
+ "$unrelated",323
+ sender.as_str(),324
+ 2_200,325
+ "ordinary message",326
+ None,327
+ ),328
+ ];329
+ let feed = SlidingFeed::new(room_id.to_owned(), initial.clone());330
+ feed.mount(&server).await;331
+332
+ let (engine, handle, mut events) = Engine::from_client(333
+ client,334
+ EngineConfig::default().with_initial_history_limit(0),335
+ )336
+ .await337
+ .unwrap();338
+ let running = tokio::spawn(engine.run());339
+340
+ let first = mentions_page(&mut events, room_id, |page| !page.entries.is_empty()).await;341
+ assert_eq!(entry_ids(&first), HashSet::from(["$mention".to_owned()]));342
+ assert_eq!(first.unread_count, 1);343
+ let first_boundary = first.boundary.clone();344
+345
+ wait_for_requests(&feed, 3).await;346
+ let replayed = handle.mentions_page(None, 50).await.unwrap();347
+ assert_eq!(entry_ids(&replayed), HashSet::from(["$mention".to_owned()]));348
+349
+ let key = replayed.entries[0].key.clone();350
+ handle351
+ .set_mentions_read(vec![key.clone()], true)352
+ .await353
+ .unwrap();354
+ let read = handle.mentions_page(None, 50).await.unwrap();355
+ assert_eq!(read.unread_count, 0);356
+357
+ feed.replace(358
+ room_id,359
+ [360
+ initial,361
+ vec![message(362
+ "$older-mention",363
+ sender.as_str(),364
+ 1_000,365
+ "older direct mention",366
+ Some(own_user.as_str()),367
+ )],368
+ ]369
+ .concat(),370
+ );371
+ let updated = mentions_page(&mut events, room_id, |page| page.entries.len() == 2).await;372
+ assert_eq!(373
+ entry_ids(&updated),374
+ HashSet::from(["$mention".to_owned(), "$older-mention".to_owned(),])375
+ );376
+377
+ handle378
+ .mark_mentions_read_through(first_boundary)379
+ .await380
+ .unwrap();381
+ let after_boundary = handle.mentions_page(None, 50).await.unwrap();382
+ let read_states: Vec<_> = after_boundary383
+ .entries384
+ .iter()385
+ .map(|entry| (entry.key.event_id.to_string(), entry.read))386
+ .collect();387
+ assert!(read_states.contains(&("$mention".to_owned(), true)));388
+ assert!(read_states.contains(&("$older-mention".to_owned(), false)));389
+ assert_eq!(after_boundary.unread_count, 1);390
+391
+ shut_down(&handle, &mut events, running).await;392
+}393
+394
+#[tokio::test]395
+async fn edits_can_remove_and_restore_a_mention_until_redaction_removes_its_target() {396
+ let server = MatrixMockServer::new().await;397
+ let room_id = room_id!("!mentions-edits:localhost");398
+ let (client, own_user) = joined_client(&server, room_id).await;399
+ let sender = user_id!("@alice:localhost");400
+ let original = message(401
+ "$edited-target",402
+ sender.as_str(),403
+ 1_000,404
+ "original mention",405
+ Some(own_user.as_str()),406
+ );407
+ let removal = json!({408
+ "type": "m.room.message",409
+ "event_id": "$remove-edit",410
+ "sender": sender,411
+ "origin_server_ts": 2_000,412
+ "content": {413
+ "msgtype": "m.text",414
+ "body": "* edited without a mention",415
+ "m.relates_to": { "rel_type": "m.replace", "event_id": "$edited-target" },416
+ "m.new_content": { "msgtype": "m.text", "body": "edited without a mention" },417
+ },418
+ });419
+ let addition = json!({420
+ "type": "m.room.message",421
+ "event_id": "$add-edit",422
+ "sender": sender,423
+ "origin_server_ts": 3_000,424
+ "content": {425
+ "msgtype": "m.text",426
+ "body": "* edited to mention you",427
+ "m.relates_to": { "rel_type": "m.replace", "event_id": "$edited-target" },428
+ "m.new_content": {429
+ "msgtype": "m.text",430
+ "body": "edited to mention you",431
+ "m.mentions": { "user_ids": [own_user.as_str()] },432
+ },433
+ },434
+ });435
+ let redaction = json!({436
+ "type": "m.room.redaction",437
+ "event_id": "$redaction",438
+ "sender": sender,439
+ "origin_server_ts": 4_000,440
+ "redacts": "$edited-target",441
+ "content": {},442
+ });443
+ let feed = SlidingFeed::new(room_id.to_owned(), vec![original.clone()]);444
+ feed.mount(&server).await;445
+446
+ let (engine, handle, mut events) = Engine::from_client(447
+ client,448
+ EngineConfig::default().with_initial_history_limit(0),449
+ )450
+ .await451
+ .unwrap();452
+ let running = tokio::spawn(engine.run());453
+454
+ let discovered = mentions_page(&mut events, room_id, |page| !page.entries.is_empty()).await;455
+ assert_eq!(456
+ entry_ids(&discovered),457
+ HashSet::from(["$edited-target".to_owned()])458
+ );459
+460
+ feed.replace(room_id, vec![original.clone(), removal.clone()]);461
+ let removed = mentions_page(&mut events, room_id, |page| {462
+ page.revision > discovered.revision && page.entries.is_empty()463
+ })464
+ .await;465
+ assert!(removed.entries.is_empty());466
+ assert_eq!(467
+ handle468
+ .mention_context(room_id, ruma::event_id!("$edited-target"))469
+ .await470
+ .unwrap_err()471
+ .kind,472
+ macltshow full diff (262144 bytes truncated)