atagen

macltok

public
Star0
Watch0

Commit cbb715e11963

tarball

atagen <boss@atagen.co>2026-10-25 02:35 UTC

32 files changed+6595-193
mentions: persist an encrypted account inbox

Timeline observers cannot fence Sliding Sync checkpoints or retain encrypted candidates until keys arrive. ResponseCommitHook lets the inbox commit before the SDK advances its position, while coverage records the limits of notification history.
Modified file: Cargo.lock+12 additions−10 deletions
@@ -1839,6 +1839,7 @@ dependencies = [
18391839 "mime",
18401840 "reqwest",
18411841 "ruma",
1842+ "rusqlite",
18421843 "serde",
18431844 "serde_json",
18441845 "sha2",
@@ -1849,6 +1850,7 @@ dependencies = [
18491850 "uuid",
18501851 "wasm-bindgen",
18511852 "wasm-bindgen-futures",
1853+ "wasm-bindgen-test",
18521854 "web-sys",
18531855 "wiremock",
18541856 ]
@@ -2000,7 +2002,7 @@ dependencies = [
20002002 [[package]]
20012003 name = "matrix-sdk"
20022004 version = "0.18.0"
2003-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2005+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
20042006 dependencies = [
20052007 "anymap2",
20062008 "aquamarine",
@@ -2063,7 +2065,7 @@ dependencies = [
20632065 [[package]]
20642066 name = "matrix-sdk-base"
20652067 version = "0.18.0"
2066-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2068+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
20672069 dependencies = [
20682070 "as_variant",
20692071 "assert_matches",
@@ -2093,7 +2095,7 @@ dependencies = [
20932095 [[package]]
20942096 name = "matrix-sdk-common"
20952097 version = "0.18.0"
2096-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2098+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
20972099 dependencies = [
20982100 "eyeball-im",
20992101 "futures-core",
@@ -2116,7 +2118,7 @@ dependencies = [
21162118 [[package]]
21172119 name = "matrix-sdk-crypto"
21182120 version = "0.18.0"
2119-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2121+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
21202122 dependencies = [
21212123 "aes",
21222124 "aquamarine",
@@ -2157,7 +2159,7 @@ dependencies = [
21572159 [[package]]
21582160 name = "matrix-sdk-indexeddb"
21592161 version = "0.18.0"
2160-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2162+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
21612163 dependencies = [
21622164 "async-trait",
21632165 "base64",
@@ -2189,7 +2191,7 @@ dependencies = [
21892191 [[package]]
21902192 name = "matrix-sdk-sqlite"
21912193 version = "0.18.0"
2192-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2194+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
21932195 dependencies = [
21942196 "as_variant",
21952197 "async-trait",
@@ -2216,7 +2218,7 @@ dependencies = [
22162218 [[package]]
22172219 name = "matrix-sdk-store-encryption"
22182220 version = "0.18.0"
2219-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2221+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
22202222 dependencies = [
22212223 "base64",
22222224 "blake3",
@@ -2237,7 +2239,7 @@ dependencies = [
22372239 [[package]]
22382240 name = "matrix-sdk-test"
22392241 version = "0.18.0"
2240-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2242+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
22412243 dependencies = [
22422244 "as_variant",
22432245 "ctor",
@@ -2260,7 +2262,7 @@ dependencies = [
22602262 [[package]]
22612263 name = "matrix-sdk-test-macros"
22622264 version = "0.18.0"
2263-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2265+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
22642266 dependencies = [
22652267 "quote",
22662268 "syn 2.0.119",
@@ -2269,7 +2271,7 @@ dependencies = [
22692271 [[package]]
22702272 name = "matrix-sdk-ui"
22712273 version = "0.18.0"
2272-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2274+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
22732275 dependencies = [
22742276 "as_variant",
22752277 "async-rx",
Modified file: Cargo.toml+16 additions−11 deletions
@@ -8,7 +8,7 @@ description = "A compact application backend on top of the Matrix Rust SDK"
88 default = []
99 mirror = []
1010 serde = ["dep:serde", "imbl/serde"]
11-sqlite = ["matrix-sdk/sqlite", "serde", "dep:url", "dep:matrix-sdk-store-encryption"]
11+sqlite = ["matrix-sdk/sqlite", "serde", "dep:url", "dep:matrix-sdk-store-encryption", "dep:rusqlite"]
1212 bundled-sqlite = ["sqlite", "matrix-sdk/bundled-sqlite"]
1313 indexeddb = [
1414 "matrix-sdk/indexeddb",
@@ -26,11 +26,11 @@ indexeddb = [
2626 futures-util = { version = "0.3.31", default-features = false, features = ["std"] }
2727 http = "1"
2828 imbl = { version = "6.1.0", default-features = false }
29-matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["e2e-encryption"] }
30-matrix-sdk-base = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false }
31-matrix-sdk-crypto = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false }
32-matrix-sdk-store-encryption = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", optional = true }
33-matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false }
29+matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["e2e-encryption"] }
30+matrix-sdk-base = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false }
31+matrix-sdk-crypto = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false }
32+matrix-sdk-store-encryption = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", optional = true }
33+matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false }
3434 ruma = { version = "0.16.0", default-features = false, features = ["unstable-msc2545", "unstable-msc4095"] }
3535 serde = { version = "1.0.219", features = ["derive", "rc"], optional = true }
3636 serde_json = "1.0.143"
@@ -40,6 +40,7 @@ sha2 = "0.10.9"
4040 tokio = { version = "1.47.1", features = ["macros", "sync"] }
4141 tracing = "0.1.41"
4242 url = { version = "2.5.7", optional = true }
43+rusqlite = { version = "0.37", optional = true }
4344
4445 [target.'cfg(target_family = "wasm")'.dependencies]
4546 getrandom = { version = "0.2", features = ["js"] }
@@ -47,8 +48,8 @@ getrandom3 = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
4748 getrandom4 = { package = "getrandom", version = "0.4", features = ["wasm_js"] }
4849 indexed_db_futures = { package = "matrix_indexed_db_futures", version = "0.7.0", optional = true, features = ["typed-arrays"] }
4950 js-sys = { version = "0.3.105", optional = true }
50-matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["js"] }
51-matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["js"] }
51+matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["js"] }
52+matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["js"] }
5253 # the sdk's own wasm reqwest leaves out stream, which the bounded media download needs
5354 reqwest = { version = "0.13.1", default-features = false, features = ["stream"] }
5455 uuid = { version = "1", features = ["js"] }
@@ -62,12 +63,16 @@ web-sys = { version = "0.3.105", optional = true, features = [
6263 "WorkerGlobalScope",
6364 ] }
6465
65-[dev-dependencies]
66-matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767", default-features = false, features = ["testing"] }
67-matrix-sdk-test = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "cc2b13ea185c0f694523a3a48d8aadfe366dd767" }
66+[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
67+matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48", default-features = false, features = ["testing"] }
68+matrix-sdk-test = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "7aace883fbe03d4c3c118b55188fe9519ba8ce48" }
6869 tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread", "signal", "time"] }
6970 wiremock = "0.6.5"
7071
72+[target.'cfg(target_family = "wasm")'.dev-dependencies]
73+tokio = { version = "1.47.1", features = ["macros", "rt"] }
74+wasm-bindgen-test = "0.3"
75+
7176 [[example]]
7277 name = "headless"
7378 required-features = ["sqlite"]
Added file: docs/mentions-discovery.md+341 additions
@@ -0,0 +1,341 @@
1+# Persistent mentions discovery
2+
3+This records the SDK audit and current persistent mentions implementation.
4+Persistent sessions now commit discovery with Sliding Sync responses, retain an
5+encrypted inbox, and expose it through `ClientHandle` and the mirror.
6+Historical coverage remains incomplete because room-list sync and notification
7+history cannot prove that every past mention was observed.
8+
9+The audited dependency is Matrix Rust SDK 0.18.0 at fork revision
10+`cc2b13ea185c0f694523a3a48d8aadfe366dd767`. macltok now pins fork revision
11+`7aace883fbe03d4c3c118b55188fe9519ba8ce48`, which adds the response commit
12+hook described below. SDK paths below are relative to the audited revision.
13+
14+## Findings
15+
16+The SDK can deliver and decrypt events without opening a UI timeline. macltok
17+records observed events and recovers candidates through notification history,
18+but neither source establishes complete historical coverage.
19+
20+The SDK audit found that an awaited event handler can hold back the Sliding
21+Sync position until a macltok transaction commits, while a background
22+broadcast consumer cannot provide that guarantee. Handlers cannot propagate
23+commit errors, and response processing outlives cancellation of the calling
24+sync task. These limits motivated the fallible response-commit hook now
25+implemented in the pinned SDK fork.
26+
27+## Sync coverage
28+
29+macltok constructs `SyncService` in `src/engine/mod.rs` and uses the SDK room
30+list configuration. The list starts with the selective range `0..=19`. After
31+setup it grows by 100 rooms per response, without an SDK-side maximum, until
32+it reaches the server's room count. Recovery and sufficiently long idle
33+periods can return it to the initial selective range before it grows again.
34+The default list timeline limit is one event per room on initial delivery.
35+Subsequent changes for covered rooms continue arriving through sync.
36+
37+Opening a room separately subscribes it with a
38+timeline limit of 20. macltok's `update_room_subscriptions` and
39+`watchers::room_subscriptions` manage those explicit subscriptions for active
40+room actors only. The mentions response hook filters entries to joined rooms.
41+The built-in list still includes invited rooms.
42+
43+Sources are `crates/matrix-sdk-ui/src/room_list_service/mod.rs`,
44+`crates/matrix-sdk-ui/src/room_list_service/state.rs`, and
45+`crates/matrix-sdk/src/sliding_sync/list/request_generator.rs`.
46+
47+The default one-event initial window cannot recover all earlier pings, including
48+messages sent while the client was offline. Raising
49+`SyncServiceBuilder::with_room_list_timeline_limit` increases the initial
50+window but does not establish continuous history. The response hook processes
51+joined-room events and records limited timelines, initial room coverage,
52+checkpoint discontinuities, and recovery gaps.
53+
54+## Observation and durable commit
55+
56+`Client::subscribe_to_all_room_updates` returns a bounded broadcast receiver
57+of `RoomUpdates`. Its `joined` map includes processed timeline events with raw
58+decrypted content when available and optional evaluated push actions. It does
59+not require a UI timeline. Lag is an error that requires recovery, not an
60+invitation to skip to the next update.
61+
62+`Client::observe_events` retains only the latest observation and can miss
63+intermediate events. It is unsuitable for durable discovery.
64+
65+An earlier probe registers `Client::add_event_handler` for
66+`Raw<AnySyncTimelineEvent>` with `Room` context before sync starts. Its awaited
67+handler demonstrates event ordering, but it cannot report commit failures
68+reliably and is not the production ingestion path. Production uses the
69+fallible Sliding Sync response hook described below.
70+
71+For Sliding Sync v5, the response processor awaits event handlers before
72+updating and persisting `pos`. The response hook commits the inbox first, so
73+a crash permits replay without skipping committed candidates. Stable event
74+keys make replay idempotent. The hook acknowledgement does not wait for
75+context capture, a UI subscriber, or the engine event stream to drain.
76+
77+For ordinary event handlers, the SDK logs and discards handler errors, so
78+returning an error after a failed write still permits `pos` to advance. A
79+pending handler alone also does not provide a reliable shutdown path. The SDK
80+runs response handling in a detached task that outlives cancellation of
81+`sync_once` or its caller. Aborting the engine's sync task cannot prevent a
82+suspended handler from later returning and advancing the position. Registering
83+a handler and placing work on a queue without awaiting its commit does not
84+close the crash window.
85+
86+macltok uses shared `pos` through its SDK service and commits each response
87+batch through one serialized mentions actor before the hook acknowledges it.
88+The actor records joined-room events, raw encrypted candidates, limited
89+timeline gaps, and the connection checkpoint in the same inbox revision.
90+Stable room and event keys make replay idempotent. Context capture, mirror
91+publication, and event-stream delivery continue after the commit. The classic
92+sync `next_batch` ordering differs and does not establish this guarantee.
93+Nor does this ordering make arbitrary concurrent sync loops safe.
94+
95+Sources are `crates/matrix-sdk/src/event_handler/mod.rs`,
96+`crates/matrix-sdk/src/event_handler/static_events.rs`,
97+`crates/matrix-sdk/src/sliding_sync/client.rs`,
98+`crates/matrix-sdk/src/sliding_sync/mod.rs`, and
99+`crates/matrix-sdk/src/sliding_sync/cache.rs`.
100+
101+### SDK commit hook implementation
102+
103+The local SDK workspace at `../matrix-rust-sdk-mentions` implements the
104+fallible asynchronous response-commit hook through `ResponseCommitHook` and
105+`ResponseCommitBatch`. `SyncServiceBuilder` forwards it only to its room-list
106+connection, so independent notification lookups are unaffected.
107+
108+The hook receives the processed room events, original room payloads, limited
109+flags, pagination tokens, connection identity, and previous and next position
110+identifiers. It runs before response handlers and observers and before `pos`
111+is updated or saved. SDK room state may already have been saved. A successful
112+acknowledgement follows the macltok inbox transaction. A failed or dropped
113+acknowledgement returns a sync error without advancing the position.
114+
115+The hook runs outside SDK store locks and supports local futures on wasm and
116+sendable futures natively. The default absent hook preserves existing SDK
117+behavior. Detached response processing observes a dropped store
118+acknowledgement after its caller is cancelled. Checkpoint replay works after
119+SDK room state was saved, and stable event keys deduplicate repeated commits.
120+
121+The SDK audit and focused validation covered commit ordering, failed and
122+dropped acknowledgements, caller cancellation, replay after room-state
123+persistence, restoring a previous checkpoint, absent-hook behavior, builder
124+forwarding, and connection isolation. The SDK UI library also received a
125+focused wasm compilation check with its `js` feature. The SDK changes are
126+published to the user's fork under `mentions-commit-hook`, and the manifest
127+and both macltok lockfiles pin revision
128+`7aace883fbe03d4c3c118b55188fe9519ba8ce48`.
129+
130+## Event cache and delayed decryption
131+
132+`EventCache::subscribe` starts global room-update ingestion. The SDK room list
133+service already calls it during construction. Its generic update subscription
134+identifies the affected room, then `Room::event_cache` and
135+`RoomEventCache::find_event` can retrieve an event. This does not require a
136+`matrix-sdk-ui::Timeline` or a retained subscriber per room.
137+
138+`EventCache::subscribe_to_decryption_reports` reports resolved event IDs,
139+lag, and backup availability. `EventCache::request_decryption` accepts room
140+and Megolm session IDs for explicit retries. `Room::decrypt_event` is also
141+public and can decrypt a retained raw event independently of a UI timeline.
142+
143+The late-key probe confirms automatic decryption and retrieval of a structured
144+mention through the global cache without a UI timeline. Cached push actions
145+are optional. Re-evaluate them with `Room::event_push_actions` when needed,
146+and treat missing push context as unresolved rather than a negative result.
147+The room's own membership state is needed to build that push context.
148+
149+The cache is not a durable ingestion journal. Its room broadcasts and generic
150+updates can lag, and cache retention can remove events. SQLite and IndexedDB
151+builders select persistent SDK event-cache stores, while the default
152+store-free client uses memory. Even with persistence, the asynchronous cache
153+consumer is not awaited before Sliding Sync saves its position.
154+
155+The inbox retains unresolved encrypted candidates as encrypted raw events
156+within a configurable bound. It retries due candidates at startup and resets
157+their retry time when decryption reports arrive. Candidate eviction sets the
158+coverage truncation flag. Imported keys and backups cannot reconstruct events
159+already discarded by every cache.
160+
161+Sources are `crates/matrix-sdk/src/event_cache/mod.rs`,
162+`crates/matrix-sdk/src/event_cache/redecryptor.rs`,
163+`crates/matrix-sdk/src/event_cache/caches/room/mod.rs`, and
164+`crates/matrix-sdk/src/room/mod.rs`.
165+
166+## Mention policy
167+
168+Structured `m.mentions.user_ids` and `m.mentions.room` supply semantic
169+candidates. The classifier does not infer pings from rendered bodies or
170+unread counts. A reply relationship alone does not prove that a reply notifies
171+the user, so reply reasons require notification metadata and confirmation that
172+the referenced event belongs to the signed-in user.
173+
174+`Room::event_push_actions` evaluates push rules but does not independently
175+filter ignored senders. The classifier calls `Client::is_user_ignored` and
176+rechecks entries when the ignore list changes.
177+
178+A room mute can suppress evaluated notification actions while structured
179+mention metadata remains present. The classifier retains explicit user
180+mentions in muted rooms and marks notification delivery as suppressed. Reply
181+reasons require both a reply to the user and a notifying highlight. Room-wide
182+reasons require the relevant push conditions and sender power level. Missing
183+authorization context remains unresolved for reevaluation. Keyword-only
184+notifications are outside the inbox. The classifier excludes the signed-in
185+user's own events even when they contain explicit mention metadata.
186+
187+The existing `NotificationClient` combines notification evaluation with
188+ignored-sender filtering. Its public lookup takes known room and event IDs.
189+It is useful for resolving a candidate, but filtered results erase the
190+distinction between mute policy and an absent semantic mention. It is not the
191+sole classifier for this inbox.
192+
193+## Historical candidates and gaps
194+
195+Ruma exposes
196+`api::client::push::get_notifications::v3::Request`, sent with `Client::send`.
197+It accepts opaque `from`, optional `limit`, and optional `only` fields.
198+Responses contain notifications and an optional `next_token`. macltok requests
199+50 notifications per page. Entries include raw events, room IDs, timestamps,
200+actions, and server notification read state, which does not initialize or
201+replace local inbox read state.
202+
203+The service uses this endpoint as a bounded candidate source, validates
204+events after decryption, and applies the live mention policy. Each page's
205+observed candidates and cursor are committed together in the encrypted inbox,
206+including unresolved encrypted candidates. A failed commit leaves the prior
207+cursor available for replay.
208+
209+An exhausted notification list does not mean complete room history. Muted-room
210+pings, encrypted mentions not recognized by the server, expired notifications,
211+and server-specific retention can be absent. Recovery inspects at most 20 pages
212+of 50 notifications per run and resumes from committed cursors later. The
213+older-history cursor is separate from the cursor for scans from the newest
214+notifications. A newest scan records an anchor and runs at startup and every
215+60 seconds. It follows the committed cursor until it reaches that anchor,
216+resuming on later runs when needed to find newly added notifications.
217+
218+SDK cache gaps retain pagination tokens and support room pagination, but
219+running that pagination mutates the live cache. Mention context capture uses
220+independent `/context` and `/messages` requests. Tokens may expire. Context can
221+be retried around a known event while the room remains joined, and known gaps
222+remain visible when continuity cannot be established.
223+
224+Coverage exposes the earliest observed timestamp, history-endpoint support,
225+recovery activity, known room gaps, unresolved candidate count, and retention
226+truncation. The earliest timestamp is a summary, not proof that every room is
227+covered after that time. The inbox keeps its incomplete-history flag even
228+after notification pagination ends.
229+
230+Complete offline mention recovery across arbitrary muted and encrypted rooms
231+is not supplied by this endpoint or by the current sync configuration. It
232+would require a retained, replayable event source or bounded per-room history
233+recovery with proof of continuity. The notification endpoint does not provide
234+that guarantee.
235+
236+## Storage, context, and API
237+
238+`src/mentions/store` implements encrypted SQLite and IndexedDB record stores.
239+Persistent session construction opens the account store, and local logout
240+deletes it alongside the SDK stores. The mentions actor owns the store for
241+inbox commits and discovery checkpoints.
242+
243+The passphrase wraps a random SDK `StoreCipher`. Logical keys are hashed, and
244+authenticated encrypted records include schema, account identity, and logical
245+key. Restore rejects mismatched accounts, unsupported schemas, corrupt records,
246+and inconsistent record counts or byte totals. Atomic transactions compare the
247+previous encrypted header before publishing a new revision, so stale writers
248+cannot overwrite a newer checkpoint.
249+
250+The default budget is 64 MiB of encrypted records, hashed keys, header, and
251+wrapped cipher, excluding database page overhead. The store record cap is
252+35,128. The inbox defaults to 5,000 entries, 25,000 shared context events, and
253+5,000 unresolved candidates within that byte budget. When a size or record
254+limit requires eviction, the store drops bounded data and marks coverage
255+truncated.
256+
257+The storage tests cover atomic restore of records and checkpoints, stale
258+writers, capacity failure, encrypted physical records, wrong credentials or
259+account, corruption, unsupported schemas, and injected SQLite write failure.
260+The inbox tests cover batch failure, notification recovery, live context
261+updates, and capture persistence across session resume.
262+
263+
264+The inbox is serialized into one `inbox` value and encrypted as a single
265+record on every revision. Each update reserializes and re-encrypts the retained
266+inbox, so write cost scales with retained data. Decrypted previews, saved
267+context, raw candidates, pending capture state, media source registrations,
268+read state, coverage, and sync checkpoints stay inside the authenticated
269+encrypted payload. SDK SQLite custom values remain plaintext and are not used
270+for decrypted mentions.
271+
272+Context capture in `src/engine/context.rs` sizes the before and after requests
273+independently and retains the media registrations produced by normalization.
274+Each capture requests up to 10 events on each side. At most two capture jobs
275+run concurrently, with up to 128 pending or failed entries considered for
276+scheduling at a time. Capture uses independent `/context` and `/messages`
277+requests, so it does not paginate the live cache. Media source registrations
278+are encrypted with saved events and registered again when context is read.
279+Media bodies and key material do not enter the mentions public model.
280+
281+One account-scoped mentions actor serializes store commits for entries,
282+unresolved candidates, shared context events, read state, revisions, discovery
283+checkpoints, and retention decisions.
284+
285+Pending capture state is durable with the inbox, so interrupted capture can
286+resume after startup. When a room leaves the joined state, saved context
287+remains readable and its capture state becomes unavailable. Refetch is
288+disabled after departure.
289+
290+Store publication follows commit. `ClientHandle` exposes page, context, retry,
291+read, read-through, and recovery methods. The public `Command`, mention models,
292+mirror state, and updates support serde when the `serde` feature is enabled,
293+so native and browser consumers can transport them. Browser worker integration
294+is a consumer phase after SDK and macltok revision publication. Malice's worker
295+still needs mention variants in its explicit `Request` dispatch and TypeScript
296+methods.
297+
298+Mirror snapshots contain at most 50 entry previews, coverage, unread count,
299+and a committed revision boundary, with no saved context bodies. The watcher
300+coalesces updates and computes upserts and removals relative to the previous
301+bounded snapshot it observed. A missed update therefore leaves the latest
302+snapshot authoritative. A removal can mean an entry fell out of the top 50,
303+and does not imply that it was deleted from the inbox. Pagination uses
304+timestamp and stable event key within a discovery revision boundary.
305+Read-through boundaries exclude mentions first discovered or reactivated after
306+that boundary, including older messages resolved by late decryption.
307+
308+## Executable probes
309+
310+`tests/mentions_discovery.rs` exercises unopened-room update and cache access,
311+push-rule behavior for ignored senders and muted rooms, and typed
312+notification-history pagination.
313+
314+`tests/mentions_decryption.rs` exchanges a real encrypted event and a delayed
315+room key between two mock clients, then retrieves the decrypted mention
316+without opening a UI timeline.
317+
318+`tests/mentions_checkpoint.rs` holds an event handler pending and verifies
319+that the SDK has not persisted its position. It cancels the calling task,
320+then acknowledges the handler and verifies that detached response processing
321+still saves the position. A new Sliding Sync instance restores that position.
322+This is the historical event-handler ordering probe. It does not exercise the
323+production response-commit hook or the durable inbox transaction. It also
324+demonstrates that returning a handler error still advances the position.
325+
326+`tests/mentions_inbox.rs` covers live mention classification, read-through
327+with revision boundaries, independent context limits, and persistence across
328+session resume. `tests/mentions_late_decryption.rs` verifies that a delayed room key
329+updates the production inbox once. `src/mentions/service_tests.rs` covers
330+checkpoint rollback, notification-history recovery, and context updates.
331+`src/mentions/store/tests.rs` exercises encrypted record restore, capacity,
332+stale writers, and corruption handling.
333+
334+These probes establish SDK behavior against mocked responses. They do not
335+establish homeserver notification retention, room-list completeness, or
336+complete history on a deployed server.
337+
338+Focused validation passed 63 native tests and 12 shared inbox and IndexedDB
339+tests in headless Firefox. Compilation checks also pass without storage
340+features and for wasm with IndexedDB and mirror support. The full flake gate
341+was skipped by request.
Modified file: harness/web/Cargo.lock+9 additions−8 deletions
@@ -1733,6 +1733,7 @@ dependencies = [
17331733 "reqwest",
17341734 "ruma",
17351735 "serde",
1736+ "serde_json",
17361737 "sha2",
17371738 "thiserror 2.0.20",
17381739 "tokio",
@@ -1906,7 +1907,7 @@ dependencies = [
19061907 [[package]]
19071908 name = "matrix-sdk"
19081909 version = "0.18.0"
1909-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
1910+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
19101911 dependencies = [
19111912 "anymap2",
19121913 "aquamarine",
@@ -1966,7 +1967,7 @@ dependencies = [
19661967 [[package]]
19671968 name = "matrix-sdk-base"
19681969 version = "0.18.0"
1969-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
1970+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
19701971 dependencies = [
19711972 "as_variant",
19721973 "async-trait",
@@ -1992,7 +1993,7 @@ dependencies = [
19921993 [[package]]
19931994 name = "matrix-sdk-common"
19941995 version = "0.18.0"
1995-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
1996+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
19961997 dependencies = [
19971998 "eyeball-im",
19981999 "futures-core",
@@ -2015,7 +2016,7 @@ dependencies = [
20152016 [[package]]
20162017 name = "matrix-sdk-crypto"
20172018 version = "0.18.0"
2018-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2019+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
20192020 dependencies = [
20202021 "aes",
20212022 "aquamarine",
@@ -2055,7 +2056,7 @@ dependencies = [
20552056 [[package]]
20562057 name = "matrix-sdk-indexeddb"
20572058 version = "0.18.0"
2058-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2059+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
20592060 dependencies = [
20602061 "async-trait",
20612062 "base64",
@@ -2087,7 +2088,7 @@ dependencies = [
20872088 [[package]]
20882089 name = "matrix-sdk-sqlite"
20892090 version = "0.18.0"
2090-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2091+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
20912092 dependencies = [
20922093 "as_variant",
20932094 "async-trait",
@@ -2114,7 +2115,7 @@ dependencies = [
21142115 [[package]]
21152116 name = "matrix-sdk-store-encryption"
21162117 version = "0.18.0"
2117-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2118+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
21182119 dependencies = [
21192120 "base64",
21202121 "blake3",
@@ -2135,7 +2136,7 @@ dependencies = [
21352136 [[package]]
21362137 name = "matrix-sdk-ui"
21372138 version = "0.18.0"
2138-source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=cc2b13ea185c0f694523a3a48d8aadfe366dd767#cc2b13ea185c0f694523a3a48d8aadfe366dd767"
2139+source = "git+ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk?rev=7aace883fbe03d4c3c118b55188fe9519ba8ce48#7aace883fbe03d4c3c118b55188fe9519ba8ce48"
21392140 dependencies = [
21402141 "as_variant",
21412142 "async-rx",
Modified file: src/account.rs+29 additions
@@ -42,6 +42,7 @@ pub struct Session {
4242
4343 /// taken as soon as the client exists, since the sdk can refresh before there's an engine
4444 struct Tracking {
45+ store: Arc<dyn SessionStore>,
4546 session_changes: broadcast::Receiver<SessionChange>,
4647 persistence: Option<Persistence>,
4748 #[cfg(not(target_family = "wasm"))]
@@ -312,6 +313,17 @@ impl StoreConfig {
312313 &self.passphrase
313314 }
314315
316+ #[cfg(feature = "sqlite")]
317+ pub(crate) fn mentions_location(&self) -> PathBuf {
318+ let StoreLocation::Sqlite(path) = &self.location;
319+ path.join("macltok-mentions.sqlite3")
320+ }
321+
322+ #[cfg(feature = "indexeddb")]
323+ pub(crate) fn mentions_location(&self) -> String {
324+ format!("{}::macltok-mentions", self.indexeddb_name())
325+ }
326+
315327 #[cfg(feature = "indexeddb")]
316328 pub(crate) fn indexeddb_name(&self) -> &str {
317329 let StoreLocation::IndexedDb(name) = &self.location;
@@ -834,12 +846,28 @@ impl Session {
834846 self,
835847 config: EngineConfig,
836848 ) -> Result<(Engine, ClientHandle, EventStream), AccountError> {
849+ let account = crate::mentions::store::Account {
850+ homeserver: self.client.homeserver().to_string(),
851+ user_id: self
852+ .client
853+ .user_id()
854+ .ok_or(AccountError::MissingSession)?
855+ .to_owned(),
856+ };
857+ let mentions = crate::mentions::store::MentionsStore::open(
858+ self.tracking.store.store(),
859+ account,
860+ config.max_mentions_bytes,
861+ )
862+ .await
863+ .map_err(|error| account_error("opening the mentions store", error))?;
837864 Engine::with_persistence(
838865 self.client,
839866 config,
840867 self.tracking.session_changes,
841868 self.tracking.persistence,
842869 Some(self.store_lock),
870+ Some(mentions),
843871 )
844872 .await
845873 .map_err(AccountError::from)
@@ -872,6 +900,7 @@ impl Tracking {
872900 #[cfg(not(target_family = "wasm"))]
873901 let inline = save_inline(client, sessions.clone(), results)?;
874902 Ok(Self {
903+ store: sessions.clone(),
875904 session_changes,
876905 persistence: Some(Persistence {
877906 save: Box::new(move |client| {
Modified file: src/engine/context.rs+164 additions−131 deletions
@@ -1,20 +1,23 @@
11 use std::sync::Arc;
22
3-use matrix_sdk::{room::MessagesOptions, Room};
4-use matrix_sdk_ui::timeline::{RoomExt, TimelineEventFocusThreadMode, TimelineFocus};
5-use ruma::{uint, OwnedEventId};
3+use matrix_sdk::{
4+ Room, deserialized_responses::TimelineEvent as SdkTimelineEvent, room::MessagesOptions,
5+};
6+use ruma::{OwnedEventId, OwnedUserId, RoomId, UInt};
67
78 use crate::{
8- normalise::{normalise_timeline, MediaScope},
99 Error, ErrorKind, EventContext, EventContextCommand, EventContextDirection, EventContextPage,
1010 TimelineItem,
11+ engine::MediaService,
12+ normalise::{MediaRegistration, normalise_observed},
1113 };
1214
1315 const MAX_CONTEXT_LIMIT: u16 = 50;
1416
1517 pub(crate) async fn execute(
1618 room: Room,
17- current_user_id: Option<ruma::OwnedUserId>,
19+ current_user_id: Option<OwnedUserId>,
20+ media: Arc<MediaService>,
1821 command: EventContextCommand,
1922 ) -> Result<crate::CommandOutput, Error> {
2023 if let EventContextCommand::Before { token, limit, .. }
@@ -30,6 +33,7 @@ pub(crate) async fn execute(
3033 return execute_page(
3134 room,
3235 current_user_id,
36+ media,
3337 token,
3438 *limit,
3539 matches!(command, EventContextCommand::Before { .. }),
@@ -37,7 +41,7 @@ pub(crate) async fn execute(
3741 .await;
3842 }
3943
40- let (event_id, limit) = match &command {
44+ let (event_id, before_limit, after_limit) = match &command {
4145 EventContextCommand::Fetch {
4246 event_id,
4347 before_limit,
@@ -46,71 +50,139 @@ pub(crate) async fn execute(
4650 } => {
4751 validate_limit(*before_limit)?;
4852 validate_limit(*after_limit)?;
49- (event_id.clone(), (*before_limit).max(*after_limit))
53+ (event_id.clone(), *before_limit, *after_limit)
5054 }
5155 EventContextCommand::Before { .. } | EventContextCommand::After { .. } => unreachable!(),
5256 };
53- validate_limit(limit)?;
54-
55- let raw_context = if matches!(command, EventContextCommand::Fetch { .. }) {
56- Some(
57- room.event_with_context(&event_id, false, uint!(0), None)
58- .await
59- .map_err(|source| {
60- let error = Error::from_sdk("fetching event context failed", source);
61- if error
62- .server
63- .as_ref()
64- .is_some_and(|server| server.status == 404)
65- {
66- Error::new(ErrorKind::EventNotFound, "event was not found")
67- } else {
68- error
69- }
70- })?,
71- )
72- } else {
73- None
74- };
57+ let (context, registrations, _) =
58+ capture(room, current_user_id, event_id, before_limit, after_limit).await?;
59+ media.register(registrations).await;
60+ Ok(crate::CommandOutput::EventContext(context))
61+}
7562
76- let focus = TimelineFocus::Event {
77- target: event_id.clone(),
78- num_context_events: limit,
79- thread_mode: TimelineEventFocusThreadMode::Automatic {
80- hide_threaded_events: true,
81- },
82- };
83- let timeline = room
84- .timeline_builder()
85- .with_focus(focus)
86- .build()
63+pub(crate) async fn capture(
64+ room: Room,
65+ current_user_id: Option<OwnedUserId>,
66+ event_id: OwnedEventId,
67+ before_limit: u16,
68+ after_limit: u16,
69+) -> Result<(EventContext, Vec<MediaRegistration>, Vec<serde_json::Value>), Error> {
70+ validate_limit(before_limit)?;
71+ validate_limit(after_limit)?;
72+ let raw = room
73+ .event_with_context(&event_id, false, UInt::from(0_u16), None)
8774 .await
8875 .map_err(|source| {
89- Error::with_source(ErrorKind::Timeline, "building event context failed", source)
76+ let error = Error::from_sdk("fetching event context failed", source);
77+ if error
78+ .server
79+ .as_ref()
80+ .is_some_and(|server| server.status == 404)
81+ {
82+ Error::new(ErrorKind::EventNotFound, "event was not found")
83+ } else {
84+ error
85+ }
9086 })?;
91-
92- if let EventContextCommand::Fetch { .. } = command {
93- let raw = raw_context.expect("fetch context response");
94- let normalized = normalize_items(timeline.items().await, current_user_id.as_deref());
95- let (before, event, after) = split_items(normalized, &event_id);
96- if event.is_none() && raw.event.is_none() {
97- return Err(Error::new(ErrorKind::EventNotFound, "event was not found"));
98- }
99- return Ok(crate::CommandOutput::EventContext(EventContext {
100- events_before: before,
101- event,
102- events_after: after,
103- start_token: raw.prev_batch_token,
104- end_token: raw.next_batch_token,
105- }));
87+ if raw.event.is_none() {
88+ return Err(Error::new(ErrorKind::EventNotFound, "event was not found"));
10689 }
90+ let before = fetch_side(
91+ &room,
92+ raw.prev_batch_token.as_deref(),
93+ before_limit,
94+ true,
95+ current_user_id.as_deref(),
96+ );
97+ let after = fetch_side(
98+ &room,
99+ raw.next_batch_token.as_deref(),
100+ after_limit,
101+ false,
102+ current_user_id.as_deref(),
103+ );
104+ let (events_before, mut before_media, start_token, mut observed) = before.await?;
105+ let (events_after, after_media, end_token, following) = after.await?;
106+ observed.extend(following);
107+ if let Some(value) = raw
108+ .event
109+ .as_ref()
110+ .and_then(|event| serde_json::from_str(event.raw().json().get()).ok())
111+ {
112+ observed.push(value);
113+ }
114+ let event = raw.event.as_ref().and_then(|event| {
115+ normalise_sdk_event(event, current_user_id.as_deref(), room.room_id()).map(
116+ |(item, media)| {
117+ before_media.extend(media);
118+ item
119+ },
120+ )
121+ });
122+ before_media.extend(after_media);
123+ Ok((
124+ EventContext {
125+ events_before,
126+ event,
127+ events_after,
128+ start_token,
129+ end_token,
130+ },
131+ before_media,
132+ observed,
133+ ))
134+}
107135
108- unreachable!("fetch context returned above")
136+async fn fetch_side(
137+ room: &Room,
138+ token: Option<&str>,
139+ limit: u16,
140+ backwards: bool,
141+ current_user_id: Option<&ruma::UserId>,
142+) -> Result<
143+ (
144+ Vec<TimelineItem>,
145+ Vec<MediaRegistration>,
146+ Option<String>,
147+ Vec<serde_json::Value>,
148+ ),
149+ Error,
150+> {
151+ if limit == 0 {
152+ return Ok((Vec::new(), Vec::new(), token.map(str::to_owned), Vec::new()));
153+ }
154+ let Some(token) = token else {
155+ return Ok((Vec::new(), Vec::new(), None, Vec::new()));
156+ };
157+ let mut options = if backwards {
158+ MessagesOptions::backward()
159+ } else {
160+ MessagesOptions::forward()
161+ };
162+ options = options.from(token);
163+ options.limit = UInt::from(limit);
164+ let messages = room
165+ .messages(options)
166+ .await
167+ .map_err(|source| Error::from_sdk("fetching event context events failed", source))?;
168+ let mut events = messages.chunk;
169+ events.truncate(usize::from(limit));
170+ if backwards {
171+ events.reverse();
172+ }
173+ let next_token = messages.end;
174+ let observed = events
175+ .iter()
176+ .filter_map(|event| serde_json::from_str(event.raw().json().get()).ok())
177+ .collect();
178+ let (items, media) = normalise_events(events, current_user_id, room.room_id());
179+ Ok((items, media, next_token, observed))
109180 }
110181
111182 async fn execute_page(
112183 room: Room,
113- current_user_id: Option<ruma::OwnedUserId>,
184+ current_user_id: Option<OwnedUserId>,
185+ media: Arc<MediaService>,
114186 token: &str,
115187 limit: u16,
116188 backwards: bool,
@@ -121,55 +193,20 @@ async fn execute_page(
121193 MessagesOptions::forward()
122194 };
123195 options = options.from(token);
124- options.limit = ruma::UInt::from(limit);
196+ options.limit = UInt::from(limit);
125197 let messages = room
126198 .messages(options)
127199 .await
128200 .map_err(|source| Error::from_sdk("fetching event context page failed", source))?;
129-
130- let ids: Vec<_> = messages
131- .chunk
132- .iter()
133- .filter_map(|event| event.event_id())
134- .collect();
135- let normalized = if let Some(target) = ids.first() {
136- let focus = TimelineFocus::Event {
137- target: target.clone(),
138- num_context_events: limit,
139- thread_mode: TimelineEventFocusThreadMode::Automatic {
140- hide_threaded_events: false,
141- },
142- };
143- let timeline = room
144- .timeline_builder()
145- .with_focus(focus)
146- .build()
147- .await
148- .map_err(|source| {
149- Error::with_source(
150- ErrorKind::Timeline,
151- "building event context page failed",
152- source,
153- )
154- })?;
155- normalize_items(timeline.items().await, current_user_id.as_deref())
156- } else {
157- Vec::new()
158- };
159-
160- let mut by_id = std::collections::HashMap::new();
161- for item in normalized {
162- if let TimelineItem::Event(event) = &item {
163- if let Some(event_id) = &event.event_id {
164- by_id.insert(event_id.clone(), item);
165- }
166- }
167- }
168- let mut events: Vec<_> = ids.iter().filter_map(|id| by_id.remove(id)).collect();
201+ let next_token = messages.end.clone();
202+ let mut events = messages.chunk;
203+ events.truncate(usize::from(limit));
169204 if backwards {
170205 events.reverse();
171206 }
172- let next_token = messages.end.clone();
207+ let (events, registrations) =
208+ normalise_events(events, current_user_id.as_deref(), room.room_id());
209+ media.register(registrations).await;
173210 let end_reached = next_token.is_none();
174211 let direction = if backwards {
175212 EventContextDirection::Before
@@ -184,41 +221,37 @@ async fn execute_page(
184221 }))
185222 }
186223
187-fn validate_limit(limit: u16) -> Result<(), Error> {
188- if limit > MAX_CONTEXT_LIMIT {
189- Err(Error::new(ErrorKind::Invalid, "context limit exceeds 50"))
190- } else {
191- Ok(())
224+fn normalise_events(
225+ events: Vec<SdkTimelineEvent>,
226+ current_user_id: Option<&ruma::UserId>,
227+ room_id: &RoomId,
228+) -> (Vec<TimelineItem>, Vec<MediaRegistration>) {
229+ let mut items = Vec::with_capacity(events.len());
230+ let mut media = Vec::new();
231+ for event in events {
232+ if let Some((item, registrations)) = normalise_sdk_event(&event, current_user_id, room_id) {
233+ items.push(item);
234+ media.extend(registrations);
235+ }
192236 }
237+ (items, media)
193238 }
194239
195-fn normalize_items(
196- items: imbl::Vector<Arc<matrix_sdk_ui::timeline::TimelineItem>>,
240+fn normalise_sdk_event(
241+ event: &SdkTimelineEvent,
197242 current_user_id: Option<&ruma::UserId>,
198-) -> Vec<TimelineItem> {
199- let scope = MediaScope::new();
200- normalise_timeline(items.iter(), current_user_id, &scope)
201- .items
202- .into_iter()
203- .collect()
243+ room_id: &RoomId,
244+) -> Option<(TimelineItem, Vec<MediaRegistration>)> {
245+ let value: serde_json::Value = serde_json::from_str(event.raw().json().get()).ok()?;
246+ let event_id = value.get("event_id")?.as_str()?;
247+ let identity = format!("{room_id}\n{event_id}");
248+ normalise_observed(&value, current_user_id, &identity)
204249 }
205250
206-fn split_items(
207- items: Vec<TimelineItem>,
208- event_id: &OwnedEventId,
209-) -> (Vec<TimelineItem>, Option<TimelineItem>, Vec<TimelineItem>) {
210- let items: Vec<_> = items
211- .into_iter()
212- .filter(|item| matches!(item, TimelineItem::Event(_)))
213- .collect();
214- let target = items.iter().position(|item| {
215- matches!(item, TimelineItem::Event(event) if event.event_id.as_ref() == Some(event_id))
216- });
217- let Some(index) = target else {
218- return (items, None, Vec::new());
219- };
220- let mut before = items;
221- let after = before.split_off(index + 1);
222- let event = before.pop();
223- (before, event, after)
251+fn validate_limit(limit: u16) -> Result<(), Error> {
252+ if limit > MAX_CONTEXT_LIMIT {
253+ Err(Error::new(ErrorKind::Invalid, "context limit exceeds 50"))
254+ } else {
255+ Ok(())
256+ }
224257 }
Modified file: src/engine/mod.rs+99 additions−16 deletions
@@ -1,5 +1,5 @@
11 mod commands;
2-mod context;
2+pub(crate) mod context;
33 pub(crate) mod executor;
44 mod media;
55 mod mutations;
@@ -33,6 +33,11 @@ pub(crate) use crate::store_lock::StoreLock;
3333 #[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
3434 pub(crate) enum StoreLock {}
3535
36+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
37+pub(crate) use crate::mentions::store::MentionsStore;
38+#[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
39+pub(crate) enum MentionsStore {}
40+
3641 use self::{
3742 commands::Job,
3843 executor::{EngineBoxFuture, TaskSet},
@@ -58,12 +63,40 @@ pub struct EngineConfig {
5863 pub initial_history_limit: u16,
5964 pub max_media_bytes: usize,
6065 pub max_media_sources: usize,
66+ pub max_mentions_bytes: usize,
67+ pub max_mentions_entries: usize,
68+ pub max_mention_context_events: usize,
69+ pub max_mention_candidates: usize,
6170 /// warm a room up for sending when it's opened or someone starts typing in it
6271 pub prepare_to_send: bool,
6372 }
6473
6574 impl EngineConfig {
6675 #[must_use]
76+ pub const fn with_max_mentions_bytes(mut self, bytes: usize) -> Self {
77+ self.max_mentions_bytes = bytes;
78+ self
79+ }
80+
81+ #[must_use]
82+ pub const fn with_max_mentions_entries(mut self, entries: usize) -> Self {
83+ self.max_mentions_entries = entries;
84+ self
85+ }
86+
87+ #[must_use]
88+ pub const fn with_max_mention_context_events(mut self, events: usize) -> Self {
89+ self.max_mention_context_events = events;
90+ self
91+ }
92+
93+ #[must_use]
94+ pub const fn with_max_mention_candidates(mut self, candidates: usize) -> Self {
95+ self.max_mention_candidates = candidates;
96+ self
97+ }
98+
99+ #[must_use]
67100 pub const fn with_command_capacity(mut self, capacity: usize) -> Self {
68101 self.command_capacity = capacity;
69102 self
@@ -108,6 +141,10 @@ impl Default for EngineConfig {
108141 initial_history_limit: 30,
109142 max_media_bytes: 20 * 1024 * 1024,
110143 max_media_sources: 2_048,
144+ max_mentions_bytes: 64 * 1024 * 1024,
145+ max_mentions_entries: 5_000,
146+ max_mention_context_events: 25_000,
147+ max_mention_candidates: 5_000,
111148 prepare_to_send: true,
112149 }
113150 }
@@ -117,10 +154,12 @@ impl Default for EngineConfig {
117154 pub enum EngineError {
118155 #[error("the Matrix client is not authenticated")]
119156 Unauthenticated,
120- #[error("command, event, and media source capacities must be nonzero")]
157+ #[error("command, event, media source, and mentions capacities must be nonzero")]
121158 ZeroCapacity,
122159 #[error("could not build the sync service")]
123160 SyncService(#[source] matrix_sdk_ui::sync_service::Error),
161+ #[error("could not open the mentions service")]
162+ MentionsStore(#[source] crate::Error),
124163 #[error("the event receiver has closed")]
125164 EventReceiverClosed,
126165 #[error("engine task {task} panicked")]
@@ -133,9 +172,10 @@ impl EngineError {
133172 kind: match self {
134173 Self::EventReceiverClosed => EngineErrorKind::EventReceiverClosed,
135174 Self::TaskPanicked { .. } => EngineErrorKind::Watcher,
136- Self::Unauthenticated | Self::ZeroCapacity | Self::SyncService(_) => {
137- EngineErrorKind::Other
138- }
175+ Self::Unauthenticated
176+ | Self::ZeroCapacity
177+ | Self::SyncService(_)
178+ | Self::MentionsStore(_) => EngineErrorKind::Other,
139179 },
140180 message: self.to_string(),
141181 }
@@ -282,6 +322,9 @@ pub struct Engine {
282322 preparations: prepare::Preparations,
283323 /// shared with every `ClientHandle`, so the store stays locked while the client is reachable
284324 _store_lock: Option<Arc<StoreLock>>,
325+ mentions: crate::mentions::service::Handle,
326+ mentions_service: Option<crate::mentions::service::Service>,
327+ mentions_updates: Option<watch::Receiver<crate::MentionsPage>>,
285328 }
286329
287330 impl Engine {
@@ -291,7 +334,7 @@ impl Engine {
291334 config: EngineConfig,
292335 ) -> Result<(Self, ClientHandle, EventStream), EngineError> {
293336 let session_changes = client.subscribe_to_session_changes();
294- Self::with_persistence(client, config, session_changes, None, None).await
337+ Self::with_persistence(client, config, session_changes, None, None, None).await
295338 }
296339
297340 /// `session_changes` is taken when the client is built, since the sdk can refresh
@@ -302,6 +345,7 @@ impl Engine {
302345 session_changes: broadcast::Receiver<SessionChange>,
303346 persistence: Option<Persistence>,
304347 store_lock: Option<StoreLock>,
348+ mentions: Option<MentionsStore>,
305349 ) -> Result<(Self, ClientHandle, EventStream), EngineError> {
306350 if client.user_id().is_none() {
307351 return Err(EngineError::Unauthenticated);
@@ -309,9 +353,27 @@ impl Engine {
309353 if config.command_capacity == 0
310354 || config.event_capacity == 0
311355 || config.max_media_sources == 0
356+ || config.max_mentions_entries == 0
357+ || config.max_mention_context_events == 0
358+ || config.max_mention_candidates == 0
312359 {
313360 return Err(EngineError::ZeroCapacity);
314361 }
362+ let store_lock = store_lock.map(Arc::new);
363+ let media = Arc::new(MediaService::new(
364+ client.clone(),
365+ config.max_media_bytes,
366+ config.max_media_sources,
367+ ));
368+ let (mentions, mentions_service, mentions_updates) =
369+ crate::mentions::service::Service::new(
370+ client.clone(),
371+ media.clone(),
372+ mentions,
373+ config.clone(),
374+ )
375+ .await
376+ .map_err(EngineError::MentionsStore)?;
315377 let sync = Arc::new(
316378 SyncService::builder(client.clone())
317379 .with_offline_mode()
@@ -319,16 +381,11 @@ impl Engine {
319381 (ROOM_PACKS.into(), "*".to_owned()),
320382 (ROOM_PREVIEW_URLS.into(), String::new()),
321383 ])
384+ .with_room_list_response_commit_hook(mentions.hook())
322385 .build()
323386 .await
324387 .map_err(EngineError::SyncService)?,
325388 );
326- let store_lock = store_lock.map(Arc::new);
327- let media = Arc::new(MediaService::new(
328- client.clone(),
329- config.max_media_bytes,
330- config.max_media_sources,
331- ));
332389 let (handle, port, events) = transport_pair(
333390 client.clone(),
334391 sync.clone(),
@@ -374,6 +431,9 @@ impl Engine {
374431 stops,
375432 logouts: Arc::new(watch::Sender::new(Logouts::default())),
376433 _store_lock: store_lock,
434+ mentions,
435+ mentions_service: Some(mentions_service),
436+ mentions_updates: Some(mentions_updates),
377437 };
378438 Ok((engine, handle, events))
379439 }
@@ -459,6 +519,17 @@ impl Engine {
459519
460520 fn start_tasks(&mut self) {
461521 let queues = self.queues.take().expect("engine tasks start once");
522+ if let Some(service) = self.mentions_service.take() {
523+ self.tasks
524+ .spawn(TaskContext::Watcher("mentions service"), service.run());
525+ }
526+ if let Some(updates) = self.mentions_updates.take() {
527+ let events = self.port.event_sender();
528+ self.tasks.spawn(
529+ TaskContext::Watcher("mentions updates"),
530+ crate::mentions::service::watch_updates(updates, events),
531+ );
532+ }
462533 let (incoming, incoming_verifications) = mpsc::unbounded_channel();
463534 self.verification_handler = Some(self.verification_handler(incoming));
464535 let client = &self.client;
@@ -754,9 +825,18 @@ impl Engine {
754825 return None;
755826 };
756827 let current_user_id = client.user_id().map(ToOwned::to_owned);
757- self.tasks.spawn(TaskContext::Command("event context"), async move {
758- reply.send(context::execute(room, current_user_id, command).await);
759- });
828+ let media = self.media.clone();
829+ self.tasks
830+ .spawn(TaskContext::Command("event context"), async move {
831+ reply.send(context::execute(room, current_user_id, media, command).await);
832+ });
833+ }
834+ Command::Mentions(command) => {
835+ let mentions = self.mentions.clone();
836+ self.tasks
837+ .spawn(TaskContext::Command("mentions command"), async move {
838+ reply.send(mentions.execute(command).await);
839+ });
760840 }
761841 Command::Media(MediaCommand::Upload(upload)) => self.upload(upload, reply),
762842 Command::Encryption(command) => enqueue(&self.encryption, Job { command, reply }),
@@ -1040,10 +1120,13 @@ impl Engine {
10401120 self.client.remove_event_handler(handler);
10411121 }
10421122 self.port.close();
1123+ self.sync.stop().await;
1124+ if self.mentions_service.is_none() {
1125+ self.mentions.stop().await;
1126+ }
10431127 self.tasks.shutdown().await;
10441128 self.rooms.clear();
10451129 self.media.clear().await;
1046- self.sync.stop().await;
10471130 self.port
10481131 .connection_sender()
10491132 .send_replace(ConnectionState::Stopped);
Modified file: src/lib.rs+9 additions−3 deletions
@@ -3,6 +3,8 @@
33 mod encryption;
44 mod engine;
55 mod error;
6+mod mentions;
7+mod mentions_model;
68 #[cfg(feature = "mirror")]
79 mod mirror;
810 mod model;
@@ -41,6 +43,10 @@ pub use session_store::{EncryptedSessionStore, SessionStore};
4143 pub use engine::{Engine, EngineConfig, EngineError};
4244 pub use error::{Error, ErrorKind, ServerError};
4345 pub use imbl;
46+pub use mentions_model::{
47+ MentionCaptureState, MentionCoverage, MentionEntry, MentionGap, MentionKey, MentionReason,
48+ MentionsBoundary, MentionsCommand, MentionsPage, MentionsUpdate, SavedMentionContext,
49+};
4450 #[cfg(feature = "mirror")]
4551 pub use mirror::{
4652 Mirror, MirrorHandle, SessionStatus, State, StateChange, Subscription, TimelineChange,
@@ -49,7 +55,8 @@ pub use mirror::{
4955 pub use model::{
5056 AccountCommand, AttachmentEncryption, AttachmentUpload, Change, Command, CommandOutput,
5157 ConnectionState, CreateRoomOptions, EncryptionCommand, EncryptionState, EngineErrorKind,
52- EngineFailure, Event, EventRef, HistoryState, ImagePack, ImagePackSource, ImagePacks,
58+ EngineFailure, Event, EventContext, EventContextCommand, EventContextDirection,
59+ EventContextPage, EventRef, HistoryState, ImagePack, ImagePackSource, ImagePacks,
5360 KeyBackupState, LatestEvent, LifecycleCommand, LinkPreview, LinkPreviewImage, Media,
5461 MediaCommand, MediaDescriptor, MediaFile, MediaId, MediaKind, MediaLocation, MediaSize,
5562 MemberFilter, MemberMembership, MemberSummary, MembershipChange, Message, MessageContent,
@@ -62,8 +69,7 @@ pub use model::{
6269 Sender, SpaceChild, Termination, ThreadSummary, TimelineCommand, TimelineContent, TimelineDiff,
6370 TimelineDiffError, TimelineEvent, TimelineId, TimelineItem, TimelineItemId, TypingState,
6471 UploadDetails, UploadOutcome, UploadThumbnail, UtdCause, VerificationAction,
65- VerificationFailure, VerificationState, VerificationTarget, EventContext, EventContextCommand,
66- EventContextDirection, EventContextPage,
72+ VerificationFailure, VerificationState, VerificationTarget,
6773 };
6874 pub use ruma;
6975 pub use transport::{ClientHandle, EventStream, Pending, SpaceHierarchy};
Added file: src/mentions/mod.rs+6 additions
@@ -0,0 +1,6 @@
1+pub(crate) mod service;
2+mod state;
3+#[cfg(test)]
4+mod state_tests;
5+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
6+pub(crate) mod store;
Added file: src/mentions/service.rs+1314 additions
@@ -0,0 +1,1314 @@
1+use std::{
2+ collections::{BTreeMap, BTreeSet},
3+ sync::Arc,
4+ time::Duration,
5+};
6+
7+use futures_util::{StreamExt, stream::FuturesUnordered};
8+use matrix_sdk::{
9+ Client, RoomState,
10+ sliding_sync::{ResponseCommitBatch, ResponseCommitHook},
11+};
12+use ruma::{
13+ OwnedRoomId, OwnedUserId,
14+ events::{AnySyncTimelineEvent, room::encrypted::OriginalSyncRoomEncryptedEvent},
15+ serde::Raw,
16+};
17+use serde_json::Value;
18+use tokio::sync::{mpsc, oneshot, watch};
19+
20+use super::state::{Candidate, Data, SavedEvent, Window, id, invalid, now};
21+use crate::{
22+ CommandOutput, EngineConfig, Error, ErrorKind, Event, MentionCaptureState, MentionEntry,
23+ MentionGap, MentionKey, MentionReason, MentionsCommand, MentionsPage, MentionsUpdate,
24+ TimelineContent, TimelineItem,
25+ engine::{MediaService, MentionsStore, context, executor::EngineBoxFuture},
26+ normalise::{MediaRegistration, normalise_observed},
27+ transport::EventSender,
28+};
29+
30+#[cfg(all(test, feature = "sqlite"))]
31+#[path = "service_tests.rs"]
32+mod tests;
33+
34+#[derive(Clone)]
35+pub(crate) struct Handle(mpsc::Sender<Request>);
36+
37+enum Request {
38+ Batch(ResponseCommitBatch, oneshot::Sender<Result<(), Error>>),
39+ Command(
40+ MentionsCommand,
41+ oneshot::Sender<Result<CommandOutput, Error>>,
42+ ),
43+ Stop(oneshot::Sender<()>),
44+}
45+
46+pub(crate) struct Service {
47+ client: Client,
48+ account: OwnedUserId,
49+ media: Arc<MediaService>,
50+ store: Option<MentionsStore>,
51+ data: Data,
52+ config: EngineConfig,
53+ requests: mpsc::Receiver<Request>,
54+ updates: watch::Sender<MentionsPage>,
55+ recovery_budget: usize,
56+ latest_scan: bool,
57+ recovery_retry: u64,
58+}
59+
60+enum Work {
61+ Capture(
62+ String,
63+ u64,
64+ Result<(crate::EventContext, Vec<MediaRegistration>, Vec<Value>), Error>,
65+ ),
66+ Candidate(String, Result<Value, Error>),
67+ History(
68+ bool,
69+ Option<String>,
70+ Result<ruma::api::client::push::get_notifications::v3::Response, Error>,
71+ ),
72+}
73+
74+impl Handle {
75+ pub(crate) fn hook(&self) -> ResponseCommitHook {
76+ let handle = self.clone();
77+ ResponseCommitHook::new(move |batch| {
78+ let handle = handle.clone();
79+ async move {
80+ let (reply, response) = oneshot::channel();
81+ handle
82+ .0
83+ .send(Request::Batch(batch, reply))
84+ .await
85+ .map_err(|_| "mentions service stopped".to_owned())?;
86+ response
87+ .await
88+ .map_err(|_| "mentions commit acknowledgement dropped".to_owned())?
89+ .map_err(|error| error.to_string())
90+ }
91+ })
92+ }
93+
94+ pub(crate) async fn execute(&self, command: MentionsCommand) -> Result<CommandOutput, Error> {
95+ let (reply, response) = oneshot::channel();
96+ self.0
97+ .send(Request::Command(command, reply))
98+ .await
99+ .map_err(|_| Error::interrupted())?;
100+ response.await.map_err(|_| Error::interrupted())?
101+ }
102+
103+ pub(crate) async fn stop(&self) {
104+ let (reply, response) = oneshot::channel();
105+ if self.0.send(Request::Stop(reply)).await.is_ok() {
106+ let _ = response.await;
107+ }
108+ }
109+}
110+
111+impl Service {
112+ pub(crate) async fn new(
113+ client: Client,
114+ media: Arc<MediaService>,
115+ store: Option<MentionsStore>,
116+ config: EngineConfig,
117+ ) -> Result<(Handle, Self, watch::Receiver<MentionsPage>), Error> {
118+ let account = client
119+ .user_id()
120+ .ok_or_else(|| {
121+ Error::new(
122+ ErrorKind::Unauthenticated,
123+ "mentions require an authenticated account",
124+ )
125+ })?
126+ .to_owned();
127+ #[cfg(any(feature = "sqlite", feature = "indexeddb"))]
128+ let data: Data = store
129+ .as_ref()
130+ .map(|store| store.read("inbox"))
131+ .transpose()?
132+ .flatten()
133+ .unwrap_or_default();
134+ #[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
135+ let data = Data::default();
136+ #[cfg(any(feature = "sqlite", feature = "indexeddb"))]
137+ if store
138+ .as_ref()
139+ .is_some_and(|store| store.revision() != data.revision)
140+ {
141+ return Err(Error::new(
142+ ErrorKind::Store,
143+ "mentions revision does not match its checkpoint",
144+ ));
145+ }
146+ let page = data.page(&account, None, 50)?;
147+ media
148+ .register(
149+ page.entries
150+ .iter()
151+ .filter_map(|entry| data.events.get(&id(&entry.key)))
152+ .flat_map(|event| event.media.clone())
153+ .collect(),
154+ )
155+ .await;
156+ let (updates, receiver) = watch::channel(page);
157+ let (sender, requests) = mpsc::channel(config.command_capacity.max(1));
158+ Ok((
159+ Handle(sender),
160+ Self {
161+ client,
162+ account,
163+ media,
164+ store,
165+ data,
166+ config,
167+ requests,
168+ updates,
169+ recovery_budget: 20,
170+ latest_scan: true,
171+ recovery_retry: 0,
172+ },
173+ receiver,
174+ ))
175+ }
176+
177+ async fn save(&mut self, mut next: Data) -> Result<(), Error> {
178+ next.revision = self
179+ .data
180+ .revision
181+ .checked_add(1)
182+ .ok_or_else(|| invalid("mentions revision exhausted"))?;
183+ next.bound(
184+ self.config.max_mentions_entries,
185+ self.config.max_mention_context_events,
186+ self.config.max_mention_candidates,
187+ );
188+ #[cfg(any(feature = "sqlite", feature = "indexeddb"))]
189+ if let Some(store) = &mut self.store {
190+ loop {
191+ let value = serde_json::to_value(&next).map_err(|error| {
192+ Error::with_source(
193+ ErrorKind::Store,
194+ "encoding the mentions inbox failed",
195+ error,
196+ )
197+ })?;
198+ match store
199+ .commit(
200+ self.data.revision,
201+ vec![super::store::Write::Set("inbox".to_owned(), value)],
202+ )
203+ .await
204+ {
205+ Ok(_) => break,
206+ Err(error)
207+ if error.message == "mentions storage limit exceeded"
208+ && next.evict_one() => {}
209+ Err(error) => return Err(error),
210+ }
211+ }
212+ }
213+ self.data = next;
214+ let page = self.data.page(&self.account, None, 50)?;
215+ self.media
216+ .register(
217+ page.entries
218+ .iter()
219+ .filter_map(|entry| self.data.events.get(&id(&entry.key)))
220+ .flat_map(|event| event.media.clone())
221+ .collect(),
222+ )
223+ .await;
224+ self.updates.send_replace(page);
225+ Ok(())
226+ }
227+
228+ async fn command(&mut self, command: MentionsCommand) -> Result<CommandOutput, Error> {
229+ match command {
230+ MentionsCommand::Page { cursor, limit } => {
231+ let page = self.data.page(&self.account, cursor.as_deref(), limit)?;
232+ self.media
233+ .register(
234+ page.entries
235+ .iter()
236+ .filter_map(|entry| self.data.events.get(&id(&entry.key)))
237+ .flat_map(|event| event.media.clone())
238+ .collect(),
239+ )
240+ .await;
241+ Ok(CommandOutput::MentionsPage(page))
242+ }
243+ MentionsCommand::Context(key) => {
244+ let (context, media) = self.data.context(&key)?;
245+ self.media.register(media).await;
246+ Ok(CommandOutput::MentionContext(context))
247+ }
248+ MentionsCommand::RetryContext(key) => {
249+ let room = self
250+ .client
251+ .get_room(&key.room_id)
252+ .filter(|room| room.state() == RoomState::Joined)
253+ .ok_or_else(|| {
254+ Error::new(
255+ ErrorKind::Forbidden,
256+ "saved context cannot be refetched after leaving the room",
257+ )
258+ })?;
259+ drop(room);
260+ let mut next = self.data.clone();
261+ let entry = next
262+ .entries
263+ .get_mut(&id(&key))
264+ .ok_or_else(|| Error::new(ErrorKind::EventNotFound, "mention was not found"))?;
265+ entry.capture = MentionCaptureState::Pending;
266+ entry.revision = self.data.revision + 1;
267+ next.windows.entry(id(&key)).or_insert(Window {
268+ before: Vec::new(),
269+ target: Some(id(&key)),
270+ after: Vec::new(),
271+ start: None,
272+ end: None,
273+ });
274+ self.save(next).await?;
275+ let (context, media) = self.data.context(&key)?;
276+ self.media.register(media).await;
277+ Ok(CommandOutput::MentionContext(context))
278+ }
279+ MentionsCommand::SetRead { keys, read } => {
280+ let mut next = self.data.clone();
281+ for key in keys {
282+ if let Some(entry) = next.entries.get_mut(&id(&key)) {
283+ entry.read = read;
284+ entry.revision = self.data.revision + 1;
285+ }
286+ }
287+ self.save(next).await?;
288+ Ok(CommandOutput::Done)
289+ }
290+ MentionsCommand::ReadThrough(boundary) => {
291+ if boundary.account != self.account || boundary.revision > self.data.revision {
292+ return Err(invalid("invalid mentions read boundary"));
293+ }
294+ let mut next = self.data.clone();
295+ for entry in next
296+ .entries
297+ .values_mut()
298+ .filter(|entry| entry.discovery_revision <= boundary.revision)
299+ {
300+ entry.read = true;
301+ entry.revision = self.data.revision + 1;
302+ }
303+ self.save(next).await?;
304+ Ok(CommandOutput::Done)
305+ }
306+ MentionsCommand::Recover => {
307+ self.recovery_budget = 20;
308+ self.latest_scan = true;
309+ self.recovery_retry = 0;
310+ let mut next = self.data.clone();
311+ next.coverage.recovering = true;
312+ next.coverage.historical_supported = None;
313+ self.save(next).await?;
314+ Ok(CommandOutput::Done)
315+ }
316+ }
317+ }
318+
319+ fn gap(data: &mut Data, room_id: Option<OwnedRoomId>, reason: &str, token: Option<String>) {
320+ if !data
321+ .coverage
322+ .gaps
323+ .iter()
324+ .any(|gap| gap.room_id == room_id && gap.reason == reason && gap.token == token)
325+ {
326+ data.coverage.gaps.push(MentionGap {
327+ room_id,
328+ reason: reason.to_owned(),
329+ token,
330+ });
331+ if data.coverage.gaps.len() > 128 {
332+ data.coverage.gaps.remove(0);
333+ data.coverage.truncated = true;
334+ }
335+ }
336+ }
337+
338+ async fn batch(&mut self, batch: ResponseCommitBatch) -> Result<(), Error> {
339+ let mut next = self.data.clone();
340+ if next.checkpoints.get(&batch.connection_id) == Some(&batch.next_pos) {
341+ return Ok(());
342+ }
343+ if next.checkpoints.get(&batch.connection_id) != batch.previous_pos.as_ref() {
344+ Self::gap(
345+ &mut next,
346+ None,
347+ "sync checkpoint continuity is unknown",
348+ None,
349+ );
350+ }
351+ for (room_id, raw) in &batch.raw_rooms {
352+ if raw.limited || !next.observed_rooms.contains(room_id) {
353+ Self::gap(
354+ &mut next,
355+ Some(room_id.clone()),
356+ if raw.limited {
357+ "limited sync timeline"
358+ } else {
359+ "initial room history is incomplete"
360+ },
361+ raw.prev_batch.clone(),
362+ );
363+ }
364+ }
365+ let room_ids: BTreeSet<_> = batch
366+ .raw_rooms
367+ .keys()
368+ .chain(batch.response.rooms.joined.keys())
369+ .cloned()
370+ .collect();
371+ for room_id in &room_ids {
372+ if !self
373+ .client
374+ .get_room(room_id)
375+ .is_some_and(|room| room.state() == RoomState::Joined)
376+ {
377+ continue;
378+ }
379+ next.observed_rooms.insert(room_id.clone());
380+ let mut observed = BTreeMap::new();
381+ if let Some(raw) = batch.raw_rooms.get(room_id) {
382+ for event in &raw.timeline {
383+ let value: Value =
384+ serde_json::from_str(event.json().get()).map_err(|error| {
385+ Error::with_source(ErrorKind::Invalid, "invalid observed event", error)
386+ })?;
387+ if let Some(event_id) = value["event_id"].as_str() {
388+ observed.insert(event_id.to_owned(), value);
389+ }
390+ }
391+ }
392+ if let Some(update) = batch.response.rooms.joined.get(room_id) {
393+ for event in &update.timeline.events {
394+ let value: Value =
395+ serde_json::from_str(event.raw().json().get()).map_err(|error| {
396+ Error::with_source(ErrorKind::Invalid, "invalid observed event", error)
397+ })?;
398+ if let Some(event_id) = value["event_id"].as_str() {
399+ observed.insert(event_id.to_owned(), value);
400+ }
401+ }
402+ }
403+ let mut observed: Vec<_> = observed.into_values().collect();
404+ observed.sort_by_key(|value| value["origin_server_ts"].as_u64().unwrap_or(0));
405+ for value in observed {
406+ self.observe(&mut next, room_id, value).await?;
407+ }
408+ }
409+ for room_id in batch.response.rooms.left.keys() {
410+ Self::gap(
411+ &mut next,
412+ Some(room_id.clone()),
413+ "room left, retained context is read only",
414+ None,
415+ );
416+ next.candidates
417+ .retain(|_, candidate| &candidate.room_id != room_id);
418+ for entry in next
419+ .entries
420+ .values_mut()
421+ .filter(|entry| &entry.key.room_id == room_id)
422+ {
423+ entry.capture = MentionCaptureState::Unavailable;
424+ entry.revision = next.revision + 1;
425+ }
426+ }
427+ let reevaluate: Vec<_> = next
428+ .entries
429+ .values()
430+ .filter(|entry| room_ids.contains(&entry.key.room_id))
431+ .filter_map(|entry| {
432+ next.events
433+ .get(&id(&entry.key))
434+ .and_then(|event| event.raw.clone())
435+ .map(|raw| (entry.key.room_id.clone(), raw))
436+ })
437+ .collect();
438+ for (room_id, raw) in reevaluate {
439+ self.observe(&mut next, &room_id, raw).await?;
440+ }
441+ self.filter_ignored(&mut next).await;
442+ next.checkpoints.insert(batch.connection_id, batch.next_pos);
443+ self.save(next).await
444+ }
445+
446+ async fn filter_ignored(&self, next: &mut Data) {
447+ let mut removed = Vec::new();
448+ for (key, entry) in &next.entries {
449+ if let TimelineItem::Event(event) = &entry.event {
450+ if self.client.is_user_ignored(&event.sender.user_id).await {
451+ removed.push(key.clone());
452+ }
453+ }
454+ }
455+ for key in removed {
456+ next.entries.remove(&key);
457+ next.windows.remove(&key);
458+ }
459+ next.collect_events();
460+ }
461+
462+ async fn observe(
463+ &self,
464+ data: &mut Data,
465+ room_id: &OwnedRoomId,
466+ mut value: Value,
467+ ) -> Result<(), Error> {
468+ let Some(event_id) = value["event_id"]
469+ .as_str()
470+ .and_then(|value| value.parse().ok())
471+ else {
472+ return Ok(());
473+ };
474+ let key = MentionKey {
475+ room_id: room_id.clone(),
476+ event_id,
477+ };
478+ let identity = id(&key);
479+ let Some(timestamp) = value["origin_server_ts"].as_u64() else {
480+ return Ok(());
481+ };
482+ data.coverage.earliest_timestamp_ms = Some(
483+ data.coverage
484+ .earliest_timestamp_ms
485+ .map_or(timestamp, |old| old.min(timestamp)),
486+ );
487+ if value["type"] == "m.room.redaction" {
488+ if let Some(target) = value["redacts"]
489+ .as_str()
490+ .or_else(|| value["content"]["redacts"].as_str())
491+ .and_then(|value| value.parse().ok())
492+ {
493+ let target = id(&MentionKey {
494+ room_id: room_id.clone(),
495+ event_id: target,
496+ });
497+ data.redacted.insert(target.clone(), timestamp);
498+ data.entries.remove(&target);
499+ data.windows.remove(&target);
500+ data.candidates.remove(&target);
501+ data.edits.remove(&target);
502+ data.candidates.retain(|_, candidate| {
503+ candidate.room_id != *room_id
504+ || candidate.raw["content"]["m.relates_to"]["event_id"].as_str()
505+ != target.split_once('\n').map(|(_, event)| event)
506+ });
507+ if let Some(saved) = data.events.get_mut(&target) {
508+ redact(saved);
509+ }
510+ data.touch_contexts(&target);
511+ }
512+ return Ok(());
513+ }
514+ if data.redacted.contains_key(&identity)
515+ || value["unsigned"].get("redacted_because").is_some()
516+ {
517+ data.redacted.insert(identity.clone(), timestamp);
518+ data.entries.remove(&identity);
519+ data.windows.remove(&identity);
520+ data.candidates.remove(&identity);
521+ if let Some(saved) = data.events.get_mut(&identity) {
522+ redact(saved);
523+ }
524+ data.touch_contexts(&identity);
525+ return Ok(());
526+ }
527+ if value["content"]["m.relates_to"]["rel_type"] == "m.replace" {
528+ if let Some(target) = value["content"]["m.relates_to"]["event_id"]
529+ .as_str()
530+ .and_then(|value| value.parse().ok())
531+ {
532+ let target = id(&MentionKey {
533+ room_id: room_id.clone(),
534+ event_id: target,
535+ });
536+ if data.redacted.contains_key(&target) {
537+ return Ok(());
538+ }
539+ let newer = data.edits.get(&target).is_none_or(|old| {
540+ (old["origin_server_ts"].as_u64(), old["event_id"].as_str())
541+ < (Some(timestamp), value["event_id"].as_str())
542+ });
543+ if newer {
544+ if let Some(original) =
545+ data.events.get(&target).and_then(|saved| saved.raw.clone())
546+ {
547+ if original["sender"] != value["sender"] {
548+ return Ok(());
549+ }
550+ data.edits.insert(target.clone(), value.clone());
551+ value = edited(original, &value);
552+ return Box::pin(self.observe(data, room_id, value)).await;
553+ }
554+ data.candidates.insert(
555+ identity,
556+ Candidate {
557+ room_id: room_id.clone(),
558+ raw: value,
559+ attempts: 0,
560+ retry_at_ms: 0,
561+ },
562+ );
563+ }
564+ }
565+ return Ok(());
566+ }
567+ if let Some(edit) = value["unsigned"]["m.relations"].get("m.replace").cloned() {
568+ if edit["sender"] == value["sender"]
569+ && edit["content"]["m.new_content"].is_object()
570+ && edit["content"]["m.relates_to"]["event_id"]
571+ .as_str()
572+ .is_none_or(|target| target == key.event_id.as_str())
573+ && edit["origin_server_ts"].as_u64().is_some()
574+ && data.edits.get(&identity).is_none_or(|old| {
575+ (old["origin_server_ts"].as_u64(), old["event_id"].as_str())
576+ < (edit["origin_server_ts"].as_u64(), edit["event_id"].as_str())
577+ })
578+ {
579+ data.edits.insert(identity.clone(), edit);
580+ }
581+ }
582+ if let Some(edit) = data.edits.get(&identity) {
583+ if value["sender"] == edit["sender"] {
584+ value = edited(value, edit);
585+ }
586+ }
587+ if let Some(edit) = value["unsigned"].get("macltok_edit").cloned() {
588+ data.edits.insert(identity.clone(), edit);
589+ }
590+ if value["type"] == "m.room.encrypted" {
591+ data.candidates.entry(identity).or_insert(Candidate {
592+ room_id: room_id.clone(),
593+ raw: value,
594+ attempts: 0,
595+ retry_at_ms: 0,
596+ });
597+ return Ok(());
598+ }
599+ let Some((item, media)) = normalise_observed(&value, Some(&self.account), &identity) else {
600+ return Ok(());
601+ };
602+ self.supplement(data, &key, &item, &media, &value);
603+ let (reasons, suppressed, unresolved) = self.classify(data, room_id, &value).await;
604+ if unresolved {
605+ data.candidates
606+ .entry(identity.clone())
607+ .or_insert(Candidate {
608+ room_id: room_id.clone(),
609+ raw: value.clone(),
610+ attempts: 0,
611+ retry_at_ms: now() + 30_000,
612+ });
613+ } else {
614+ data.candidates.remove(&identity);
615+ }
616+ if reasons.is_empty() {
617+ data.entries.remove(&identity);
618+ data.windows.remove(&identity);
619+ if data.events.contains_key(&identity) || data.edits.contains_key(&identity) {
620+ data.events.insert(
621+ identity,
622+ SavedEvent {
623+ item,
624+ media,
625+ raw: Some(value),
626+ },
627+ );
628+ }
629+ return Ok(());
630+ }
631+ let previous = data.entries.get(&identity);
632+ let changed = previous.is_none_or(|entry| {
633+ entry.event != item
634+ || entry.reasons != reasons
635+ || entry.notification_suppressed != suppressed
636+ });
637+ let entry = MentionEntry {
638+ key,
639+ timestamp_ms: timestamp,
640+ discovered_at_ms: previous.map_or_else(now, |entry| entry.discovered_at_ms),
641+ discovery_revision: previous
642+ .map_or(data.revision + 1, |entry| entry.discovery_revision),
643+ revision: if changed {
644+ data.revision + 1
645+ } else {
646+ previous.unwrap().revision
647+ },
648+ event: item.clone(),
649+ reasons,
650+ read: previous.is_some_and(|entry| entry.read),
651+ capture: previous.map_or(MentionCaptureState::Pending, |entry| {
652+ if changed {
653+ MentionCaptureState::Pending
654+ } else {
655+ entry.capture.clone()
656+ }
657+ }),
658+ room_name: self.client.get_room(room_id).and_then(|room| room.name()),
659+ notification_suppressed: suppressed,
660+ };
661+ data.entries.insert(identity.clone(), entry);
662+ data.windows.entry(identity.clone()).or_insert(Window {
663+ before: Vec::new(),
664+ target: Some(identity.clone()),
665+ after: Vec::new(),
666+ start: None,
667+ end: None,
668+ });
669+ data.events.insert(
670+ identity,
671+ SavedEvent {
672+ item,
673+ media,
674+ raw: Some(value),
675+ },
676+ );
677+ Ok(())
678+ }
679+
680+ fn supplement(
681+ &self,
682+ data: &mut Data,
683+ key: &MentionKey,
684+ item: &TimelineItem,
685+ media: &[MediaRegistration],
686+ raw: &Value,
687+ ) {
688+ let TimelineItem::Event(event) = item else {
689+ return;
690+ };
691+ let identity = id(key);
692+ if data
693+ .events
694+ .get(&identity)
695+ .is_some_and(|saved| saved.item != *item)
696+ {
697+ data.touch_contexts(&identity);
698+ }
699+ let mut retained = data.events.contains_key(&identity);
700+ let saved_events = &data.events;
701+ for (target, window) in &mut data.windows {
702+ let Some(entry) = data.entries.get_mut(target) else {
703+ continue;
704+ };
705+ if entry.key.room_id != key.room_id
706+ || (entry.timestamp_ms, target.as_str()) >= (event.timestamp_ms, identity.as_str())
707+ || window.after.contains(&identity)
708+ {
709+ continue;
710+ }
711+ let previous = window.after.clone();
712+ window.after.push(identity.clone());
713+ window.after.sort_by_key(|key| {
714+ if key == &identity {
715+ (event.timestamp_ms, key.clone())
716+ } else {
717+ (
718+ saved_events
719+ .get(key)
720+ .and_then(|saved| match &saved.item {
721+ TimelineItem::Event(event) => Some(event.timestamp_ms),
722+ _ => None,
723+ })
724+ .unwrap_or(0),
725+ key.clone(),
726+ )
727+ }
728+ });
729+ window.after.truncate(10);
730+ if window.after == previous {
731+ continue;
732+ }
733+ window.end = None;
734+ retained |= window.after.contains(&identity);
735+ if entry.capture == MentionCaptureState::Partial
736+ && window.after.len() == 10
737+ && (window.before.len() == 10 || window.start.is_none())
738+ {
739+ entry.capture = MentionCaptureState::Ready;
740+ }
741+ entry.revision = data.revision + 1;
742+ }
743+ if retained {
744+ data.events.insert(
745+ identity,
746+ SavedEvent {
747+ item: item.clone(),
748+ media: media.to_vec(),
749+ raw: Some(raw.clone()),
750+ },
751+ );
752+ }
753+ }
754+
755+ async fn classify(
756+ &self,
757+ data: &Data,
758+ room_id: &OwnedRoomId,
759+ value: &Value,
760+ ) -> (Vec<MentionReason>, bool, bool) {
761+ let Some(sender) = value["sender"]
762+ .as_str()
763+ .and_then(|sender| sender.parse::<OwnedUserId>().ok())
764+ else {
765+ return (vec![], false, false);
766+ };
767+ if sender == self.account
768+ || self.client.is_user_ignored(&sender).await
769+ || value["type"] != "m.room.message"
770+ {
771+ return (vec![], false, false);
772+ }
773+ let Some(room) = self
774+ .client
775+ .get_room(room_id)
776+ .filter(|room| room.state() == RoomState::Joined)
777+ else {
778+ return (vec![], false, false);
779+ };
780+ let raw: Raw<AnySyncTimelineEvent> =
781+ Raw::from_json_string(value.to_string()).expect("serialized event");
782+ let actions = room.event_push_actions(&raw).await.ok().flatten();
783+ let notify = actions.as_ref().is_some_and(|actions| {
784+ actions
785+ .iter()
786+ .any(|action| matches!(action, ruma::push::Action::Notify))
787+ });
788+ let highlight = actions.as_ref().is_some_and(|actions| actions.iter().any(|action| matches!(action, ruma::push::Action::SetTweak(ruma::push::Tweak::Highlight(value)) if value != &ruma::push::HighlightTweakValue::No)));
789+ let direct = value["content"]["m.mentions"]["user_ids"]
790+ .as_array()
791+ .is_some_and(|users| {
792+ users
793+ .iter()
794+ .any(|user| user.as_str() == Some(self.account.as_str()))
795+ });
796+ let mut reasons = Vec::new();
797+ if direct {
798+ reasons.push(MentionReason::Direct);
799+ }
800+ let mut unresolved = false;
801+ if let Some(reply) = value["content"]["m.relates_to"]["m.in_reply_to"]["event_id"]
802+ .as_str()
803+ .and_then(|value| value.parse().ok())
804+ {
805+ let target = data.events.get(&id(&MentionKey {
806+ room_id: room_id.clone(),
807+ event_id: reply,
808+ }));
809+ let reply_to_self = value["unsigned"]["macltok_reply_sender"].as_str() == Some(self.account.as_str()) || target.is_some_and(|target| matches!(&target.item, TimelineItem::Event(event) if event.sender.user_id == self.account));
810+ if reply_to_self && (direct || (notify && highlight)) {
811+ reasons.push(MentionReason::Reply);
812+ } else if target.is_none()
813+ && value["unsigned"]["macltok_reply_sender"].as_str().is_none()
814+ && (direct || (notify && highlight))
815+ {
816+ unresolved = true;
817+ }
818+ }
819+ if value["content"]["m.mentions"]["room"] == true {
820+ match room.power_levels().await {
821+ Ok(levels) if levels.user_can_trigger_room_notification(&sender) => {
822+ if notify && highlight {
823+ reasons.push(MentionReason::Room);
824+ } else if actions.is_none() {
825+ unresolved = true;
826+ }
827+ }
828+ Err(_) => unresolved = true,
829+ _ => {}
830+ }
831+ }
832+ (reasons, !notify, unresolved)
833+ }
834+
835+ pub(crate) async fn run(mut self) {
836+ let reports_client = self.client.clone();
837+ let reports = reports_client
838+ .event_cache()
839+ .subscribe_to_decryption_reports();
840+ futures_util::pin_mut!(reports);
841+ let mut ignores = self.client.subscribe_to_ignore_user_list_changes();
842+ let mut work: FuturesUnordered<EngineBoxFuture<'static, Work>> = FuturesUnordered::new();
843+ let mut captures = BTreeSet::new();
844+ let mut retries = BTreeSet::new();
845+ let mut history = false;
846+ let mut next_scan = now() + 60_000;
847+ let mut tick = Box::pin(matrix_sdk::sleep::sleep(Duration::from_secs(1)));
848+ loop {
849+ self.schedule(&mut work, &mut captures, &mut retries, &mut history);
850+ tokio::select! {
851+ biased;
852+ request = self.requests.recv() => match request {
853+ Some(Request::Stop(reply)) => { let _ = reply.send(()); return; }
854+ Some(Request::Batch(batch, reply)) => { let result = self.batch(batch).await; let _ = reply.send(result); }
855+ Some(Request::Command(command, reply)) => { let result = self.command(command).await; let _ = reply.send(result); }
856+ None => return,
857+ },
858+ Some(result) = work.next(), if !work.is_empty() => {
859+ let mut next = self.data.clone();
860+ let recovery = (self.latest_scan, self.recovery_budget);
861+ match result {
862+ Work::Capture(key, revision, result) => {
863+ captures.remove(&key);
864+ let observed = result.as_ref().ok().map(|(_, _, observed)| observed.clone()).unwrap_or_default();
865+ let room_id = next.entries.get(&key).filter(|entry| entry.revision == revision)
866+ .map(|entry| entry.key.room_id.clone())
867+ .filter(|room_id| self.client.get_room(room_id).is_some_and(|room| room.state() == RoomState::Joined));
868+ self.captured(&mut next, key, revision, result);
869+ if let Some(room_id) = room_id {
870+ for value in observed {
871+ let identity = value["event_id"].as_str().map(|event_id| format!("{room_id}\n{event_id}"));
872+ if identity.as_ref().is_some_and(|key| next.events.get(key).is_some_and(|event| event.raw.is_some())) { continue; }
873+ self.observe(&mut next, &room_id, value).await.ok();
874+ }
875+ }
876+ }
877+ Work::Candidate(key, result) => {
878+ retries.remove(&key);
879+ if let Some(mut candidate) = next.candidates.remove(&key) {
880+ match result {
881+ Ok(value) => {
882+ if let Err(error) = self.observe(&mut next, &candidate.room_id, value).await { tracing::warn!(%error, "mention retry failed"); }
883+ if let Some(unresolved) = next.candidates.get_mut(&key) { unresolved.attempts = candidate.attempts.saturating_add(1); unresolved.retry_at_ms = now() + (1000u64 << unresolved.attempts.min(8)); }
884+ }
885+ Err(error) => {
886+ if candidate.raw["type"] != "m.room.encrypted" && (matches!(error.kind, ErrorKind::EventNotFound | ErrorKind::Forbidden) || error.server.as_ref().is_some_and(|server| matches!(server.status, 403 | 404))) {
887+ Self::gap(&mut next, Some(candidate.room_id), "mention notification context is unavailable", None);
888+ } else { candidate.attempts = candidate.attempts.saturating_add(1); candidate.retry_at_ms = retry_at(&error, 1000u64 << candidate.attempts.min(8)); next.candidates.insert(key, candidate); }
889+ }
890+ }
891+ }
892+ }
893+ Work::History(latest, from, result) => {
894+ history = false; self.recovery_budget = self.recovery_budget.saturating_sub(1);
895+ match result {
896+ Ok(page) => {
897+ next.coverage.historical_supported = Some(true);
898+ let mut reached_anchor = false;
899+ for notification in page.notifications {
900+ if let Some(event_id) = notification.event.get_field::<String>("event_id").ok().flatten() {
901+ let identity = format!("{}\n{}", notification.room_id, event_id);
902+ if latest && from.is_none() && next.latest_head.is_none() { next.latest_head = Some(identity.clone()); }
903+ if latest && next.latest_anchor.as_ref() == Some(&identity) { reached_anchor = true; break; }
904+ }
905+ if self.client.get_room(&notification.room_id).is_some_and(|room| room.state() == RoomState::Joined) {
906+ if let Ok(value) = serde_json::from_str(notification.event.json().get()) { let _ = self.observe(&mut next, &notification.room_id, value).await; }
907+ }}
908+ let token = page.next_token.filter(|token| Some(token) != from.as_ref() && !reached_anchor);
909+ if latest { next.latest_cursor = token.clone(); if token.is_none() { self.latest_scan = false; next.latest_anchor = next.latest_head.take().or(next.latest_anchor); } }
910+ else { next.history_cursor = token.clone(); next.history_exhausted = token.is_none(); }
911+ }
912+ Err(error) => {
913+ if error.server.as_ref().is_some_and(|server| matches!(server.status, 404 | 405 | 501)) { next.coverage.historical_supported = Some(false); self.recovery_budget = 0; }
914+ if error.server.as_ref().is_some_and(|server| server.status == 400) {
915+ if latest { next.latest_cursor = None; next.latest_head = None; } else { next.history_cursor = None; }
916+ }
917+ Self::gap(&mut next, None, "notification history could not be recovered", from);
918+ self.recovery_retry = retry_at(&error, 60_000);
919+ }
920+ }
921+ next.coverage.recovering = self.recovery_budget > 0 && (self.latest_scan || !next.history_exhausted) && next.coverage.historical_supported != Some(false);
922+ }
923+ }
924+ if let Err(error) = self.save(next).await { tracing::error!(%error, "saving mention background work failed"); self.latest_scan = recovery.0; self.recovery_budget = recovery.1; self.recovery_retry = now() + 60_000; }
925+ }
926+ Some(_) = reports.next() => { for candidate in self.data.candidates.values_mut() { candidate.retry_at_ms = 0; } }
927+ Some(_) = ignores.next() => {
928+ let mut next = self.data.clone(); self.filter_ignored(&mut next).await;
929+ if let Err(error) = self.save(next).await { tracing::error!(%error, "saving ignored mention removals failed"); }
930+ }
931+ () = &mut tick => {
932+ tick = Box::pin(matrix_sdk::sleep::sleep(Duration::from_secs(1)));
933+ if now() >= next_scan && !history { self.recovery_budget = 20; self.latest_scan = true; next_scan = now() + 60_000; }
934+ }
935+ }
936+ }
937+ }
938+
939+ fn schedule(
940+ &mut self,
941+ work: &mut FuturesUnordered<EngineBoxFuture<'static, Work>>,
942+ captures: &mut BTreeSet<String>,
943+ retries: &mut BTreeSet<String>,
944+ history: &mut bool,
945+ ) {
946+ for (key, entry) in self.data.entries.iter().filter(|(_, entry)| matches!(entry.capture, MentionCaptureState::Pending) || matches!(entry.capture, MentionCaptureState::Failed {retry_at_ms, ..} if retry_at_ms <= now())).take(128) {
947+ if captures.len() >= 2 { break; }
948+ if captures.contains(key) { continue; }
949+ let Some(room) = self.client.get_room(&entry.key.room_id).filter(|room| room.state() == RoomState::Joined) else { continue; };
950+ captures.insert(key.clone());
951+ let key = key.clone(); let event_id = entry.key.event_id.clone(); let account = self.account.clone(); let revision = entry.revision;
952+ work.push(Box::pin(async move { Work::Capture(key, revision, context::capture(room, Some(account), event_id, 10, 10).await) }));
953+ }
954+ for (key, candidate) in &self.data.candidates {
955+ if retries.len() >= 2 {
956+ break;
957+ }
958+ if candidate.retry_at_ms > now() || retries.contains(key) {
959+ continue;
960+ }
961+ let Some(room) = self
962+ .client
963+ .get_room(&candidate.room_id)
964+ .filter(|room| room.state() == RoomState::Joined)
965+ else {
966+ continue;
967+ };
968+ let key = key.clone();
969+ let candidate = candidate.clone();
970+ let key_for_work = key.clone();
971+ retries.insert(key);
972+ work.push(Box::pin(async move {
973+ let result = async {
974+ if candidate.raw["type"] == "m.room.encrypted" {
975+ let raw: Raw<OriginalSyncRoomEncryptedEvent> =
976+ Raw::from_json_string(candidate.raw.to_string()).map_err(|error| {
977+ Error::with_source(
978+ ErrorKind::Invalid,
979+ "invalid encrypted mention candidate",
980+ error,
981+ )
982+ })?;
983+ let event = room.decrypt_event(&raw, None).await.map_err(|error| {
984+ Error::from_sdk("decrypting a mention candidate failed", error)
985+ })?;
986+ let value: Value =
987+ serde_json::from_str(event.raw().json().get()).map_err(|error| {
988+ Error::with_source(
989+ ErrorKind::Invalid,
990+ "invalid decrypted mention",
991+ error,
992+ )
993+ })?;
994+ if value["type"] == "m.room.encrypted" {
995+ return Err(Error::new(
996+ ErrorKind::Encryption,
997+ "mention candidate still awaits keys",
998+ ));
999+ }
1000+ return Ok(value);
1001+ }
1002+ let target = candidate.raw["content"]["m.relates_to"]["event_id"]
1003+ .as_str()
1004+ .filter(|_| {
1005+ candidate.raw["content"]["m.relates_to"]["rel_type"] == "m.replace"
1006+ });
1007+ if let Some(target) =
1008+ target.and_then(|target| target.parse::<ruma::OwnedEventId>().ok())
1009+ {
1010+ let event = room.event(&target, None).await.map_err(|error| {
1011+ Error::from_sdk("fetching an edited mention failed", error)
1012+ })?;
1013+ let original: Value = serde_json::from_str(event.raw().json().get())
1014+ .map_err(|error| {
1015+ Error::with_source(
1016+ ErrorKind::Invalid,
1017+ "invalid edited mention",
1018+ error,
1019+ )
1020+ })?;
1021+ if original["sender"] == candidate.raw["sender"] {
1022+ let mut original = edited(original, &candidate.raw);
1023+ original["unsigned"]["macltok_edit"] = candidate.raw;
1024+ return Ok(original);
1025+ }
1026+ return Err(Error::new(
1027+ ErrorKind::Forbidden,
1028+ "an edit belongs to a different sender",
1029+ ));
1030+ }
1031+ let mut value = candidate.raw;
1032+ if let Some(target) =
1033+ value["content"]["m.relates_to"]["m.in_reply_to"]["event_id"]
1034+ .as_str()
1035+ .and_then(|target| target.parse::<ruma::OwnedEventId>().ok())
1036+ {
1037+ let event = room.event(&target, None).await.map_err(|error| {
1038+ Error::from_sdk("fetching a mention reply target failed", error)
1039+ })?;
1040+ let target: Value = serde_json::from_str(event.raw().json().get())
1041+ .map_err(|error| {
1042+ Error::with_source(
1043+ ErrorKind::Invalid,
1044+ "invalid mention reply target",
1045+ error,
1046+ )
1047+ })?;
1048+ value["unsigned"]["macltok_reply_sender"] = target["sender"].clone();
1049+ }
1050+ Ok(value)
1051+ }
1052+ .await;
1053+ Work::Candidate(key_for_work, result)
1054+ }));
1055+ }
1056+ if !*history
1057+ && self.recovery_budget > 0
1058+ && now() >= self.recovery_retry
1059+ && (self.latest_scan || !self.data.history_exhausted)
1060+ && self.data.coverage.historical_supported != Some(false)
1061+ {
1062+ *history = true;
1063+ let latest = self.latest_scan;
1064+ let from = if latest {
1065+ self.data.latest_cursor.clone()
1066+ } else {
1067+ self.data.history_cursor.clone()
1068+ };
1069+ let client = self.client.clone();
1070+ work.push(Box::pin(async move {
1071+ let mut request = ruma::api::client::push::get_notifications::v3::Request::new();
1072+ request.from = from.clone();
1073+ request.limit = Some(ruma::uint!(50));
1074+ let result = client.send(request).await.map_err(|error| {
1075+ Error::from_sdk("fetching mention history failed", error.into())
1076+ });
1077+ Work::History(latest, from, result)
1078+ }));
1079+ }
1080+ }
1081+
1082+ fn captured(
1083+ &self,
1084+ data: &mut Data,
1085+ key: String,
1086+ revision: u64,
1087+ result: Result<(crate::EventContext, Vec<MediaRegistration>, Vec<Value>), Error>,
1088+ ) {
1089+ let Some(entry) = data.entries.get(&key) else {
1090+ return;
1091+ };
1092+ if entry.revision != revision {
1093+ return;
1094+ }
1095+ let room_id = entry.key.room_id.clone();
1096+ if !self
1097+ .client
1098+ .get_room(&room_id)
1099+ .is_some_and(|room| room.state() == RoomState::Joined)
1100+ {
1101+ return;
1102+ }
1103+ match result {
1104+ Ok((context, media, _)) => {
1105+ data.capture_attempts.remove(&key);
1106+ let live_after = data
1107+ .windows
1108+ .get(&key)
1109+ .map(|window| window.after.clone())
1110+ .unwrap_or_default();
1111+ let mut insert = |item: TimelineItem| -> Option<String> {
1112+ let TimelineItem::Event(event) = &item else {
1113+ return None;
1114+ };
1115+ let event_id = event.event_id.clone()?;
1116+ let event_key = id(&MentionKey {
1117+ room_id: room_id.clone(),
1118+ event_id,
1119+ });
1120+ let registrations = media_for_item(&item, &media);
1121+ let mut saved = SavedEvent {
1122+ item,
1123+ media: registrations,
1124+ raw: None,
1125+ };
1126+ if let Some(existing) = data.events.get(&event_key) {
1127+ saved = existing.clone();
1128+ }
1129+ if data.redacted.contains_key(&event_key) {
1130+ redact(&mut saved);
1131+ }
1132+ data.events.insert(event_key.clone(), saved);
1133+ Some(event_key)
1134+ };
1135+ let before: Vec<_> = context
1136+ .events_before
1137+ .into_iter()
1138+ .filter_map(&mut insert)
1139+ .collect();
1140+ let target = context.event.and_then(&mut insert);
1141+ let mut after: Vec<_> = context
1142+ .events_after
1143+ .into_iter()
1144+ .filter_map(&mut insert)
1145+ .collect();
1146+ let extended = live_after.iter().any(|key| !after.contains(key));
1147+ after.extend(live_after);
1148+ after.sort_by_key(|key| {
1149+ (
1150+ data.events
1151+ .get(key)
1152+ .and_then(|saved| match &saved.item {
1153+ TimelineItem::Event(event) => Some(event.timestamp_ms),
1154+ _ => None,
1155+ })
1156+ .unwrap_or(0),
1157+ key.clone(),
1158+ )
1159+ });
1160+ after.dedup();
1161+ after.truncate(10);
1162+ let entry = data.entries.get_mut(&key).unwrap();
1163+ entry.capture = if target.is_none() {
1164+ MentionCaptureState::Unavailable
1165+ } else if after.len() < 10 || (before.len() < 10 && context.start_token.is_some()) {
1166+ MentionCaptureState::Partial
1167+ } else {
1168+ MentionCaptureState::Ready
1169+ };
1170+ entry.revision = data.revision + 1;
1171+ data.windows.insert(
1172+ key,
1173+ Window {
1174+ before,
1175+ target,
1176+ after,
1177+ start: context.start_token,
1178+ end: if extended { None } else { context.end_token },
1179+ },
1180+ );
1181+ }
1182+ Err(error) => {
1183+ let attempts = data.capture_attempts.entry(key.clone()).or_default();
1184+ *attempts = attempts.saturating_add(1);
1185+ let delay = (1000u64 << (*attempts).min(11)).min(3_600_000);
1186+ let entry = data.entries.get_mut(&key).unwrap();
1187+ entry.capture = if matches!(
1188+ error.kind,
1189+ ErrorKind::EventNotFound | ErrorKind::Forbidden | ErrorKind::RoomUnknown
1190+ ) || error
1191+ .server
1192+ .as_ref()
1193+ .is_some_and(|server| matches!(server.status, 403 | 404))
1194+ {
1195+ MentionCaptureState::Unavailable
1196+ } else {
1197+ MentionCaptureState::Failed {
1198+ message: error.to_string(),
1199+ retry_at_ms: retry_at(&error, delay),
1200+ }
1201+ };
1202+ entry.revision = data.revision + 1;
1203+ }
1204+ }
1205+ }
1206+}
1207+
1208+fn edited(mut original: Value, edit: &Value) -> Value {
1209+ original["content"] = edit["content"]["m.new_content"].clone();
1210+ original["unsigned"]["macltok_edited"] = Value::Bool(true);
1211+ original
1212+}
1213+
1214+fn retry_at(error: &Error, fallback_ms: u64) -> u64 {
1215+ let mut source = std::error::Error::source(error);
1216+ while let Some(current) = source {
1217+ if let Some(sdk) = current.downcast_ref::<matrix_sdk::Error>() {
1218+ if let Some(ruma::api::error::ErrorKind::LimitExceeded(data)) =
1219+ sdk.client_api_error_kind()
1220+ {
1221+ if let Some(delay) = data.retry_after {
1222+ let delay = match delay {
1223+ ruma::api::error::RetryAfter::Delay(delay) => delay,
1224+ ruma::api::error::RetryAfter::DateTime(time) => time
1225+ .duration_since(ruma::time::SystemTime::now())
1226+ .unwrap_or_default(),
1227+ };
1228+ return now().saturating_add(
1229+ u64::try_from(delay.as_millis())
1230+ .unwrap_or(u64::MAX)
1231+ .max(fallback_ms),
1232+ );
1233+ }
1234+ }
1235+ }
1236+ source = current.source();
1237+ }
1238+ now().saturating_add(fallback_ms)
1239+}
1240+
1241+fn redact(event: &mut SavedEvent) {
1242+ if let TimelineItem::Event(item) = &mut event.item {
1243+ item.content = TimelineContent::Redacted;
1244+ item.reply_to = None;
1245+ item.reactions.clear();
1246+ }
1247+ event.media.clear();
1248+ event.raw = None;
1249+}
1250+
1251+fn media_for_item(
1252+ item: &TimelineItem,
1253+ registrations: &[MediaRegistration],
1254+) -> Vec<MediaRegistration> {
1255+ let TimelineItem::Event(event) = item else {
1256+ return Vec::new();
1257+ };
1258+ let mut ids = Vec::new();
1259+ match &event.content {
1260+ TimelineContent::Message(message) => {
1261+ ids.extend(message.media.iter().map(|media| &media.id));
1262+ ids.extend(
1263+ message
1264+ .link_previews
1265+ .iter()
1266+ .filter_map(|preview| preview.image.as_ref().map(|image| &image.media_id)),
1267+ );
1268+ }
1269+ TimelineContent::Sticker { media, .. } => ids.push(&media.id),
1270+ _ => {}
1271+ }
1272+ registrations
1273+ .iter()
1274+ .filter(|registration| ids.contains(&&registration.id))
1275+ .cloned()
1276+ .collect()
1277+}
1278+
1279+pub(crate) async fn watch_updates(mut updates: watch::Receiver<MentionsPage>, events: EventSender) {
1280+ let mut previous: BTreeMap<String, MentionEntry> = BTreeMap::new();
1281+ loop {
1282+ let snapshot = updates.borrow_and_update().clone();
1283+ let current: BTreeMap<_, _> = snapshot
1284+ .entries
1285+ .iter()
1286+ .map(|entry| (id(&entry.key), entry.clone()))
1287+ .collect();
1288+ let upserts = current
1289+ .iter()
1290+ .filter(|(key, entry)| previous.get(*key) != Some(entry))
1291+ .map(|(_, entry)| entry.clone())
1292+ .collect();
1293+ let removed = previous
1294+ .iter()
1295+ .filter(|(key, _)| !current.contains_key(*key))
1296+ .map(|(_, entry)| entry.key.clone())
1297+ .collect();
1298+ previous = current;
1299+ if events
1300+ .send(Event::MentionsChanged(MentionsUpdate {
1301+ snapshot,
1302+ upserts,
1303+ removed,
1304+ }))
1305+ .await
1306+ .is_err()
1307+ {
1308+ return;
1309+ }
1310+ if updates.changed().await.is_err() {
1311+ return;
1312+ }
1313+ }
1314+}
Added file: src/mentions/service_tests.rs+454 additions
@@ -0,0 +1,454 @@
1+use std::{
2+ sync::{
3+ Arc,
4+ atomic::{AtomicUsize, Ordering},
5+ },
6+ time::Duration,
7+};
8+
9+use matrix_sdk::ruma::events::room::MediaSource;
10+use matrix_sdk::{
11+ ruma::{event_id, events::Mentions, push::Ruleset, room_id, user_id},
12+ test_utils::mocks::MatrixMockServer,
13+};
14+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};
15+use rusqlite::Connection;
16+use serde_json::json;
17+use wiremock::{
18+ Mock, ResponseTemplate,
19+ matchers::{method, path, query_param, query_param_is_missing},
20+};
21+
22+use super::super::store::{Account, MentionsStore};
23+use super::*;
24+use crate::{
25+ MediaId, MediaKind, StoreConfig, TimelineContent, TimelineItem,
26+ normalise::{MediaRegistration, MediaSourceInfo},
27+};
28+
29+static STORES: AtomicUsize = AtomicUsize::new(0);
30+const MAX_BYTES: usize = 64 * 1024 * 1024;
31+
32+fn config() -> StoreConfig {
33+ let number = STORES.fetch_add(1, Ordering::Relaxed);
34+ StoreConfig::sqlite(
35+ std::env::temp_dir().join(format!(
36+ "macltok-mentions-service-{}-{number}",
37+ std::process::id()
38+ )),
39+ "service commit test passphrase",
40+ )
41+}
42+
43+fn account(client: &Client) -> Account {
44+ Account {
45+ homeserver: client.homeserver().to_string(),
46+ user_id: client.user_id().unwrap().to_owned(),
47+ }
48+}
49+
50+async fn joined_client(
51+ server: &MatrixMockServer,
52+ room_id: &ruma::RoomId,
53+) -> (Client, ruma::OwnedUserId) {
54+ let client = server.client_builder().build().await;
55+ let own_user = client.user_id().unwrap().to_owned();
56+ let room_id = room_id.to_owned();
57+ server
58+ .mock_sync()
59+ .ok_and_run(&client, {
60+ let own_user = own_user.clone();
61+ move |builder| {
62+ let factory = EventFactory::new().room(&room_id);
63+ builder.add_global_account_data(
64+ factory.push_rules(Ruleset::server_default(&own_user)),
65+ );
66+ builder.add_joined_room(
67+ JoinedRoomBuilder::new(&room_id).add_state_event(factory.member(&own_user)),
68+ );
69+ }
70+ })
71+ .await;
72+ (client, own_user)
73+}
74+
75+#[tokio::test]
76+async fn failed_batch_keeps_inbox_and_checkpoint_at_the_same_commit_boundary() {
77+ let server = MatrixMockServer::new().await;
78+ let room_id = room_id!("!mentions:localhost");
79+ let (client, own_user) = joined_client(&server, room_id).await;
80+ let event = EventFactory::new()
81+ .room(room_id)
82+ .text_msg("commit boundary")
83+ .mentions(Mentions::with_user_ids([own_user.clone()]))
84+ .event_id(event_id!("$commit-boundary:localhost"))
85+ .sender(user_id!("@bob:localhost"))
86+ .into_raw_sync();
87+ let raw_room = serde_json::from_value(json!({
88+ "initial": true,
89+ "timeline": [serde_json::to_value(event).unwrap()],
90+ }))
91+ .unwrap();
92+ let batch = ResponseCommitBatch {
93+ connection_id: "mentions-connection".to_owned(),
94+ previous_pos: None,
95+ next_pos: "mentions-position-1".to_owned(),
96+ response: Default::default(),
97+ raw_rooms: [(room_id.to_owned(), raw_room)].into(),
98+ };
99+ let config = config();
100+ let store = MentionsStore::open(&config, account(&client), MAX_BYTES)
101+ .await
102+ .unwrap();
103+ let media = Arc::new(MediaService::new(
104+ client.clone(),
105+ EngineConfig::default().max_media_bytes,
106+ EngineConfig::default().max_media_sources,
107+ ));
108+ let (_handle, mut service, updates) =
109+ Service::new(client, media, Some(store), EngineConfig::default())
110+ .await
111+ .unwrap();
112+
113+ let connection = Connection::open(config.mentions_location()).unwrap();
114+ connection
115+ .execute_batch(
116+ "CREATE TRIGGER fail_mentions_inbox_insert BEFORE INSERT ON records WHEN NEW.key NOT IN ('!cipher', '!head') BEGIN SELECT RAISE(ABORT, 'injected inbox write failure'); END;
117+ CREATE TRIGGER fail_mentions_inbox_update BEFORE UPDATE ON records WHEN OLD.key NOT IN ('!cipher', '!head') BEGIN SELECT RAISE(ABORT, 'injected inbox write failure'); END;",
118+ )
119+ .unwrap();
120+
121+ assert!(service.batch(batch.clone()).await.is_err());
122+ assert_eq!(service.data.revision, 0);
123+ assert!(service.data.entries.is_empty());
124+ assert!(service.data.checkpoints.is_empty());
125+ assert_eq!(updates.borrow().revision, 0);
126+ assert!(updates.borrow().entries.is_empty());
127+
128+ let reopened = MentionsStore::open(&config, account(&service.client), MAX_BYTES)
129+ .await
130+ .unwrap();
131+ assert_eq!(reopened.revision(), 0);
132+ assert!(reopened.read::<Data>("inbox").unwrap().is_none());
133+ drop(reopened);
134+
135+ connection
136+ .execute_batch(
137+ "DROP TRIGGER fail_mentions_inbox_insert;
138+ DROP TRIGGER fail_mentions_inbox_update;",
139+ )
140+ .unwrap();
141+ drop(connection);
142+
143+ service.batch(batch.clone()).await.unwrap();
144+ assert_eq!(service.data.revision, 1);
145+ assert_eq!(service.data.entries.len(), 1);
146+ assert_eq!(
147+ service.data.checkpoints.get("mentions-connection"),
148+ Some(&batch.next_pos)
149+ );
150+ assert_eq!(updates.borrow().revision, 1);
151+ assert_eq!(updates.borrow().entries.len(), 1);
152+
153+ let reopened = MentionsStore::open(&config, account(&service.client), MAX_BYTES)
154+ .await
155+ .unwrap();
156+ let persisted: Data = reopened.read("inbox").unwrap().unwrap();
157+ assert_eq!(reopened.revision(), 1);
158+ assert_eq!(persisted.revision, 1);
159+ assert_eq!(persisted.entries.len(), 1);
160+ assert_eq!(
161+ persisted.checkpoints.get("mentions-connection"),
162+ Some(&batch.next_pos)
163+ );
164+ drop(reopened);
165+
166+ service.batch(batch).await.unwrap();
167+ assert_eq!(service.data.revision, 1);
168+ assert_eq!(service.store.as_ref().unwrap().revision(), 1);
169+
170+ drop(service);
171+ config.delete().await.unwrap();
172+}
173+
174+#[tokio::test]
175+async fn service_recovers_notification_history_into_durable_unread_inbox() {
176+ let server = MatrixMockServer::new().await;
177+ let room_id = room_id!("!mentions:localhost");
178+ let (client, own_user) = joined_client(&server, room_id).await;
179+ let first = EventFactory::new()
180+ .room(room_id)
181+ .text_msg("history mention one")
182+ .mentions(Mentions::with_user_ids([own_user.clone()]))
183+ .event_id(event_id!("$history-one:localhost"))
184+ .sender(user_id!("@bob:localhost"))
185+ .into_raw_sync();
186+ let second = EventFactory::new()
187+ .room(room_id)
188+ .text_msg("history mention two")
189+ .mentions(Mentions::with_user_ids([own_user]))
190+ .event_id(event_id!("$history-two:localhost"))
191+ .sender(user_id!("@bob:localhost"))
192+ .into_raw_sync();
193+ let first = serde_json::from_str::<serde_json::Value>(first.json().get()).unwrap();
194+ let second = serde_json::from_str::<serde_json::Value>(second.json().get()).unwrap();
195+ Mock::given(method("GET"))
196+ .and(path("/_matrix/client/v3/notifications"))
197+ .and(query_param_is_missing("from"))
198+ .and(query_param("limit", "50"))
199+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
200+ "notifications": [{
201+ "actions": ["notify"],
202+ "event": first,
203+ "read": true,
204+ "room_id": room_id.as_str(),
205+ "ts": 1,
206+ }],
207+ "next_token": "history-page-two",
208+ })))
209+ .mount(server.server())
210+ .await;
211+ Mock::given(method("GET"))
212+ .and(path("/_matrix/client/v3/notifications"))
213+ .and(query_param("from", "history-page-two"))
214+ .and(query_param("limit", "50"))
215+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
216+ "notifications": [{
217+ "actions": ["notify"],
218+ "event": second,
219+ "read": true,
220+ "room_id": room_id.as_str(),
221+ "ts": 2,
222+ }],
223+ })))
224+ .mount(server.server())
225+ .await;
226+
227+ let config = config();
228+ let store = MentionsStore::open(&config, account(&client), MAX_BYTES)
229+ .await
230+ .unwrap();
231+ let media = Arc::new(MediaService::new(
232+ client.clone(),
233+ EngineConfig::default().max_media_bytes,
234+ EngineConfig::default().max_media_sources,
235+ ));
236+ let (handle, service, mut updates) =
237+ Service::new(client.clone(), media, Some(store), EngineConfig::default())
238+ .await
239+ .unwrap();
240+ let service_task = tokio::spawn(service.run());
241+
242+ let discovered = tokio::time::timeout(Duration::from_secs(10), async {
243+ loop {
244+ let page = updates.borrow_and_update().clone();
245+ if page.entries.len() == 2 && !page.coverage.recovering {
246+ break page;
247+ }
248+ updates.changed().await.unwrap();
249+ }
250+ })
251+ .await;
252+ handle.stop().await;
253+ service_task.await.unwrap();
254+ let discovered = discovered.expect("notification history should be recovered");
255+ assert_eq!(discovered.entries.len(), 2);
256+ assert!(discovered.entries.iter().all(|entry| !entry.read));
257+
258+ let reopened = MentionsStore::open(&config, account(&client), MAX_BYTES)
259+ .await
260+ .unwrap();
261+ let persisted: Data = reopened.read("inbox").unwrap().unwrap();
262+ assert_eq!(reopened.revision(), persisted.revision);
263+ assert_eq!(persisted.entries.len(), 2);
264+ assert!(persisted.entries.values().all(|entry| !entry.read));
265+ assert!(persisted.latest_cursor.is_none());
266+ assert!(persisted.history_cursor.is_none());
267+ drop(reopened);
268+
269+ config.delete().await.unwrap();
270+}
271+
272+fn message_body(item: &TimelineItem) -> Option<&str> {
273+ match item {
274+ TimelineItem::Event(event) => match &event.content {
275+ TimelineContent::Message(message) => Some(&message.body),
276+ _ => None,
277+ },
278+ _ => None,
279+ }
280+}
281+
282+fn media_registration() -> MediaRegistration {
283+ MediaRegistration {
284+ id: MediaId::new("test-scope#image".to_owned()),
285+ kind: MediaKind::Image,
286+ full: MediaSourceInfo {
287+ source: MediaSource::Plain(ruma::mxc_uri!("mxc://mentions.example/image").to_owned()),
288+ mimetype: Some("image/png".to_owned()),
289+ declared_size: Some(1234),
290+ },
291+ thumbnail: None,
292+ }
293+}
294+
295+#[tokio::test]
296+async fn live_context_survives_capture_then_edit_and_redaction() {
297+ let server = MatrixMockServer::new().await;
298+ let room_id = room_id!("!mentions:localhost");
299+ let (client, own_user) = joined_client(&server, room_id).await;
300+ let mention = EventFactory::new()
301+ .room(room_id)
302+ .text_msg("direct mention")
303+ .mentions(Mentions::with_user_ids([own_user]))
304+ .event_id(event_id!("$live-mention:localhost"))
305+ .sender(user_id!("@bob:localhost"))
306+ .into_raw_sync();
307+ let mut mention: serde_json::Value = serde_json::from_str(mention.json().get()).unwrap();
308+ mention["origin_server_ts"] = json!(10);
309+ let following = EventFactory::new()
310+ .room(room_id)
311+ .text_msg("following")
312+ .event_id(event_id!("$live-following:localhost"))
313+ .sender(user_id!("@bob:localhost"))
314+ .into_raw_sync();
315+ let mut following: serde_json::Value = serde_json::from_str(following.json().get()).unwrap();
316+ following["origin_server_ts"] = json!(11);
317+
318+ let media = Arc::new(MediaService::new(
319+ client.clone(),
320+ EngineConfig::default().max_media_bytes,
321+ EngineConfig::default().max_media_sources,
322+ ));
323+ let (_handle, mut service, _updates) =
324+ Service::new(client, media, None, EngineConfig::default())
325+ .await
326+ .unwrap();
327+ let room_id = room_id.to_owned();
328+ let mention_key = MentionKey {
329+ room_id: room_id.clone(),
330+ event_id: event_id!("$live-mention:localhost").to_owned(),
331+ };
332+ let following_id = format!("{room_id}\n$live-following:localhost");
333+ let mut data = Data::default();
334+
335+ service.observe(&mut data, &room_id, mention).await.unwrap();
336+ service
337+ .observe(&mut data, &room_id, following)
338+ .await
339+ .unwrap();
340+
341+ let mention_identity = id(&mention_key);
342+ let entry = data.entries.get(&mention_identity).unwrap();
343+ let revision = entry.revision;
344+ let target = entry.event.clone();
345+ service.captured(
346+ &mut data,
347+ mention_identity.clone(),
348+ revision,
349+ Ok((
350+ crate::EventContext {
351+ events_before: Vec::new(),
352+ event: Some(target),
353+ events_after: Vec::new(),
354+ start_token: None,
355+ end_token: None,
356+ },
357+ Vec::new(),
358+ Vec::new(),
359+ )),
360+ );
361+
362+ assert_eq!(
363+ data.entries[&mention_identity].capture,
364+ MentionCaptureState::Partial
365+ );
366+ assert!(data.events.contains_key(&following_id));
367+ let (saved, _) = data.context(&mention_key).unwrap();
368+ let context = saved.context.unwrap();
369+ assert_eq!(context.events_after.len(), 1);
370+ assert_eq!(message_body(&context.events_after[0]), Some("following"));
371+
372+ let edit = json!({
373+ "content": {
374+ "body": "* edited following",
375+ "m.new_content": {"body": "edited following", "msgtype": "m.text"},
376+ "m.relates_to": {"event_id": "$live-following:localhost", "rel_type": "m.replace"},
377+ "msgtype": "m.text"
378+ },
379+ "event_id": "$live-following-edit:localhost",
380+ "origin_server_ts": 12,
381+ "room_id": room_id.as_str(),
382+ "sender": "@bob:localhost",
383+ "type": "m.room.message"
384+ });
385+ service.observe(&mut data, &room_id, edit).await.unwrap();
386+ let (saved, _) = data.context(&mention_key).unwrap();
387+ let context = saved.context.unwrap();
388+ assert_eq!(
389+ message_body(&context.events_after[0]),
390+ Some("edited following")
391+ );
392+
393+ let saved_following = data.events.get_mut(&following_id).unwrap();
394+ assert!(saved_following.raw.is_some());
395+ saved_following.media.push(media_registration());
396+ assert_eq!(saved_following.media.len(), 1);
397+ let redaction = json!({
398+ "content": {},
399+ "event_id": "$live-following-redaction:localhost",
400+ "origin_server_ts": 13,
401+ "redacts": "$live-following:localhost",
402+ "room_id": room_id.as_str(),
403+ "sender": "@bob:localhost",
404+ "type": "m.room.redaction"
405+ });
406+ service
407+ .observe(&mut data, &room_id, redaction)
408+ .await
409+ .unwrap();
410+
411+ let saved_following = data.events.get(&following_id).unwrap();
412+ assert!(matches!(
413+ &saved_following.item,
414+ TimelineItem::Event(event) if event.content == TimelineContent::Redacted
415+ ));
416+ assert!(saved_following.raw.is_none());
417+ assert!(saved_following.media.is_empty());
418+ let (saved, _) = data.context(&mention_key).unwrap();
419+ let context = saved.context.unwrap();
420+ assert!(matches!(
421+ &context.events_after[0],
422+ TimelineItem::Event(event) if event.content == TimelineContent::Redacted
423+ ));
424+ data.windows.remove(&mention_identity);
425+ service.data = data;
426+ let CommandOutput::MentionContext(context) = service
427+ .command(MentionsCommand::RetryContext(mention_key))
428+ .await
429+ .unwrap()
430+ else {
431+ panic!("retry should return saved context");
432+ };
433+ assert_eq!(context.state, MentionCaptureState::Pending);
434+ assert!(context.context.unwrap().event.is_some());
435+}
436+
437+#[tokio::test]
438+async fn explicit_recovery_rechecks_server_support() {
439+ let server = MatrixMockServer::new().await;
440+ let (client, _) = joined_client(&server, room_id!("!mentions:localhost")).await;
441+ let config = EngineConfig::default();
442+ let media = Arc::new(MediaService::new(
443+ client.clone(),
444+ config.max_media_bytes,
445+ config.max_media_sources,
446+ ));
447+ let (_, mut service, _) = Service::new(client, media, None, config).await.unwrap();
448+ service.data.coverage.historical_supported = Some(false);
449+ service.recovery_budget = 0;
450+ service.command(MentionsCommand::Recover).await.unwrap();
451+ assert_eq!(service.data.coverage.historical_supported, None);
452+ assert!(service.data.coverage.recovering);
453+ assert_eq!(service.recovery_budget, 20);
454+}
Added file: src/mentions/state.rs+283 additions
@@ -0,0 +1,283 @@
1+use std::collections::{BTreeMap, BTreeSet};
2+
3+use ruma::OwnedUserId;
4+use serde_json::Value;
5+
6+use crate::{
7+ Error, ErrorKind, EventContext, MentionCaptureState, MentionCoverage, MentionEntry, MentionKey,
8+ MentionsBoundary, MentionsPage, SavedMentionContext, TimelineItem,
9+ normalise::MediaRegistration,
10+};
11+
12+#[derive(Clone)]
13+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
14+pub(super) struct SavedEvent {
15+ pub item: TimelineItem,
16+ pub media: Vec<MediaRegistration>,
17+ pub raw: Option<Value>,
18+}
19+
20+#[derive(Clone)]
21+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
22+pub(super) struct Candidate {
23+ pub room_id: ruma::OwnedRoomId,
24+ pub raw: Value,
25+ pub attempts: u32,
26+ pub retry_at_ms: u64,
27+}
28+
29+#[derive(Clone)]
30+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
31+pub(super) struct Window {
32+ pub before: Vec<String>,
33+ pub target: Option<String>,
34+ pub after: Vec<String>,
35+ pub start: Option<String>,
36+ pub end: Option<String>,
37+}
38+
39+#[derive(Clone, Default)]
40+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
41+pub(super) struct Data {
42+ pub revision: u64,
43+ pub entries: BTreeMap<String, MentionEntry>,
44+ pub events: BTreeMap<String, SavedEvent>,
45+ pub windows: BTreeMap<String, Window>,
46+ pub capture_attempts: BTreeMap<String, u32>,
47+ pub candidates: BTreeMap<String, Candidate>,
48+ pub redacted: BTreeMap<String, u64>,
49+ pub edits: BTreeMap<String, Value>,
50+ pub coverage: MentionCoverage,
51+ pub checkpoints: BTreeMap<String, String>,
52+ pub observed_rooms: BTreeSet<ruma::OwnedRoomId>,
53+ pub history_cursor: Option<String>,
54+ pub history_exhausted: bool,
55+ pub latest_cursor: Option<String>,
56+ #[cfg_attr(feature = "serde", serde(default))]
57+ pub latest_anchor: Option<String>,
58+ #[cfg_attr(feature = "serde", serde(default))]
59+ pub latest_head: Option<String>,
60+}
61+
62+pub(super) fn id(key: &MentionKey) -> String {
63+ format!("{}\n{}", key.room_id, key.event_id)
64+}
65+
66+pub(super) fn now() -> u64 {
67+ matrix_sdk::ruma::MilliSecondsSinceUnixEpoch::now()
68+ .get()
69+ .into()
70+}
71+
72+pub(super) fn invalid(message: &'static str) -> Error {
73+ Error::new(ErrorKind::Invalid, message)
74+}
75+
76+impl Data {
77+ pub fn touch_contexts(&mut self, identity: &str) {
78+ for (key, window) in &self.windows {
79+ if window.target.as_deref() == Some(identity)
80+ || window
81+ .before
82+ .iter()
83+ .chain(&window.after)
84+ .any(|key| key == identity)
85+ {
86+ if let Some(entry) = self.entries.get_mut(key) {
87+ entry.revision = self.revision + 1;
88+ }
89+ }
90+ }
91+ }
92+
93+ pub fn page(
94+ &self,
95+ account: &OwnedUserId,
96+ cursor: Option<&str>,
97+ limit: u16,
98+ ) -> Result<MentionsPage, Error> {
99+ let (boundary, last) = if let Some(cursor) = cursor {
100+ let value: Value =
101+ serde_json::from_str(cursor).map_err(|_| invalid("invalid mentions cursor"))?;
102+ if value["account"].as_str() != Some(account.as_str()) {
103+ return Err(invalid("mentions cursor belongs to a different account"));
104+ }
105+ let boundary = value["revision"]
106+ .as_u64()
107+ .ok_or_else(|| invalid("invalid mentions cursor"))?;
108+ let timestamp = value["timestamp"]
109+ .as_u64()
110+ .ok_or_else(|| invalid("invalid mentions cursor"))?;
111+ let key = value["key"]
112+ .as_str()
113+ .ok_or_else(|| invalid("invalid mentions cursor"))?
114+ .to_owned();
115+ if boundary > self.revision {
116+ return Err(invalid("mentions cursor is newer than this store"));
117+ }
118+ (boundary, Some((timestamp, key)))
119+ } else {
120+ (self.revision, None)
121+ };
122+ let mut entries: Vec<_> = self
123+ .entries
124+ .iter()
125+ .filter(|(key, entry)| {
126+ entry.discovery_revision <= boundary
127+ && last.as_ref().is_none_or(|last| {
128+ (entry.timestamp_ms, key.as_str()) < (last.0, last.1.as_str())
129+ })
130+ })
131+ .map(|(_, entry)| entry.clone())
132+ .collect();
133+ entries.sort_by(|a, b| (b.timestamp_ms, id(&b.key)).cmp(&(a.timestamp_ms, id(&a.key))));
134+ let limit = usize::from(limit.clamp(1, 50));
135+ let end_reached = entries.len() <= limit;
136+ entries.truncate(limit);
137+ let next_cursor = if end_reached {
138+ None
139+ } else {
140+ entries.last().map(|entry| serde_json::json!({
141+ "account": account.as_str(), "revision": boundary, "timestamp": entry.timestamp_ms, "key": id(&entry.key)
142+ }).to_string())
143+ };
144+ let mut coverage = self.coverage.clone();
145+ coverage.unresolved_candidates = self.candidates.len();
146+ Ok(MentionsPage {
147+ entries,
148+ next_cursor,
149+ end_reached,
150+ coverage,
151+ unread_count: self.entries.values().filter(|entry| !entry.read).count(),
152+ revision: self.revision,
153+ boundary: MentionsBoundary {
154+ account: account.clone(),
155+ revision: boundary,
156+ },
157+ })
158+ }
159+
160+ pub fn context(
161+ &self,
162+ key: &MentionKey,
163+ ) -> Result<(SavedMentionContext, Vec<MediaRegistration>), Error> {
164+ let entry = self
165+ .entries
166+ .get(&id(key))
167+ .ok_or_else(|| Error::new(ErrorKind::EventNotFound, "mention was not found"))?;
168+ let mut media = Vec::new();
169+ let mut item = |key: &String| {
170+ self.events.get(key).map(|event| {
171+ media.extend(event.media.clone());
172+ event.item.clone()
173+ })
174+ };
175+ let context = self.windows.get(&id(key)).map(|window| EventContext {
176+ events_before: window.before.iter().filter_map(&mut item).collect(),
177+ event: window.target.as_ref().and_then(&mut item),
178+ events_after: window.after.iter().filter_map(&mut item).collect(),
179+ start_token: window.start.clone(),
180+ end_token: window.end.clone(),
181+ });
182+ Ok((
183+ SavedMentionContext {
184+ key: key.clone(),
185+ state: entry.capture.clone(),
186+ context,
187+ revision: self.revision,
188+ },
189+ media,
190+ ))
191+ }
192+
193+ pub fn collect_events(&mut self) {
194+ let mut used: BTreeSet<String> = self.entries.keys().cloned().collect();
195+ for window in self.windows.values() {
196+ used.extend(window.before.iter().chain(&window.after).cloned());
197+ used.extend(window.target.iter().cloned());
198+ }
199+ self.events.retain(|key, _| used.contains(key));
200+ self.capture_attempts
201+ .retain(|key, _| self.entries.contains_key(key));
202+ }
203+
204+ pub fn evict_one(&mut self) -> bool {
205+ let oldest = |keys: Vec<String>, entries: &BTreeMap<String, MentionEntry>| {
206+ keys.into_iter()
207+ .min_by_key(|key| entries.get(key).map_or(0, |entry| entry.timestamp_ms))
208+ };
209+ if let Some(key) = oldest(
210+ self.windows
211+ .keys()
212+ .filter(|key| self.entries.get(*key).is_some_and(|entry| entry.read))
213+ .cloned()
214+ .collect(),
215+ &self.entries,
216+ )
217+ .or_else(|| oldest(self.windows.keys().cloned().collect(), &self.entries))
218+ {
219+ self.windows.remove(&key);
220+ if let Some(entry) = self.entries.get_mut(&key) {
221+ entry.capture = MentionCaptureState::Unavailable;
222+ entry.revision = self.revision;
223+ }
224+ } else if let Some(key) = self
225+ .entries
226+ .iter()
227+ .min_by_key(|(_, entry)| (!entry.read, entry.timestamp_ms))
228+ .map(|(key, _)| key.clone())
229+ {
230+ self.entries.remove(&key);
231+ } else if let Some(key) = self.candidates.keys().next().cloned() {
232+ self.candidates.remove(&key);
233+ } else if let Some(key) = self.edits.keys().next().cloned() {
234+ self.edits.remove(&key);
235+ } else if let Some(key) = self.redacted.keys().next().cloned() {
236+ self.redacted.remove(&key);
237+ } else {
238+ return false;
239+ }
240+ self.coverage.truncated = true;
241+ self.collect_events();
242+ true
243+ }
244+
245+ pub fn bound(&mut self, entries: usize, events: usize, candidates: usize) {
246+ while self.entries.len() > entries {
247+ if let Some(key) = self
248+ .entries
249+ .iter()
250+ .min_by_key(|(_, entry)| (!entry.read, entry.timestamp_ms))
251+ .map(|(key, _)| key.clone())
252+ {
253+ self.entries.remove(&key);
254+ self.windows.remove(&key);
255+ self.coverage.truncated = true;
256+ }
257+ }
258+ self.collect_events();
259+ while self.events.len() > events && self.evict_one() {}
260+ while self.candidates.len() > candidates {
261+ let key = self.candidates.keys().next().cloned().unwrap();
262+ self.candidates.remove(&key);
263+ self.coverage.truncated = true;
264+ }
265+ for map in [&mut self.redacted] {
266+ while map.len() > entries + candidates {
267+ let key = map
268+ .iter()
269+ .min_by_key(|(_, timestamp)| *timestamp)
270+ .map(|(key, _)| key.clone())
271+ .unwrap();
272+ map.remove(&key);
273+ self.coverage.truncated = true;
274+ }
275+ }
276+ while self.edits.len() > candidates {
277+ let key = self.edits.keys().next().cloned().unwrap();
278+ self.edits.remove(&key);
279+ self.coverage.truncated = true;
280+ }
281+ self.coverage.unresolved_candidates = self.candidates.len();
282+ }
283+}
Added file: src/mentions/state_tests.rs+458 additions
@@ -0,0 +1,458 @@
1+use ruma::{event_id, user_id};
2+
3+use super::state::{Data, SavedEvent, Window, id};
4+use crate::{
5+ MentionCaptureState, MentionEntry, MentionKey, MentionReason, Message, MessageKind, SendState,
6+ Sender, TimelineContent, TimelineEvent, TimelineItem, TimelineItemId,
7+};
8+
9+fn key(room: &str, event: &str) -> MentionKey {
10+ MentionKey {
11+ room_id: format!("!{room}:mentions.example").parse().unwrap(),
12+ event_id: format!("${event}:mentions.example").parse().unwrap(),
13+ }
14+}
15+
16+fn entry(key: MentionKey, timestamp_ms: u64, discovery_revision: u64, read: bool) -> MentionEntry {
17+ MentionEntry {
18+ key,
19+ timestamp_ms,
20+ discovered_at_ms: timestamp_ms,
21+ discovery_revision,
22+ revision: discovery_revision,
23+ event: TimelineItem::DayDivider { timestamp_ms },
24+ reasons: vec![MentionReason::Direct],
25+ read,
26+ capture: MentionCaptureState::Ready,
27+ room_name: None,
28+ notification_suppressed: false,
29+ }
30+}
31+
32+fn message_event(event_id: &str, timestamp_ms: u64, body: &str) -> TimelineItem {
33+ let event_id: ruma::OwnedEventId = event_id.parse().unwrap();
34+ TimelineItem::Event(Box::new(TimelineEvent {
35+ item_id: TimelineItemId::new(format!("item-{event_id}")),
36+ event_id: Some(event_id),
37+ transaction_id: None,
38+ sender: Sender {
39+ user_id: user_id!("@bob:mentions.example").to_owned(),
40+ display_name: Some("Bob".to_owned()),
41+ avatar: None,
42+ name_ambiguous: false,
43+ },
44+ timestamp_ms,
45+ own: false,
46+ send_state: SendState::Sent,
47+ content: TimelineContent::Message(Message {
48+ kind: MessageKind::Text,
49+ body: body.to_owned(),
50+ geo_uri: None,
51+ formatted_body: None,
52+ mentions: None,
53+ edited: false,
54+ media: None,
55+ link_previews: Vec::new(),
56+ }),
57+ reply_to: None,
58+ thread_root: None,
59+ thread_summary: None,
60+ reactions: Vec::new(),
61+ read_by: Vec::new(),
62+ editable: false,
63+ }))
64+}
65+
66+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
67+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
68+async fn pagination_keeps_its_discovery_boundary_and_stable_ties() {
69+ let account = user_id!("@alice:mentions.example").to_owned();
70+ let ordered = vec![
71+ key("room", "latest"),
72+ key("room", "tie-z"),
73+ key("room", "tie-b"),
74+ key("room", "tie-a"),
75+ key("room", "oldest"),
76+ ];
77+ let mut data = Data {
78+ revision: 1,
79+ ..Data::default()
80+ };
81+ for (mention, timestamp) in [
82+ (ordered[0].clone(), 300),
83+ (ordered[1].clone(), 200),
84+ (ordered[2].clone(), 200),
85+ (ordered[3].clone(), 200),
86+ (ordered[4].clone(), 100),
87+ ] {
88+ data.entries
89+ .insert(id(&mention), entry(mention, timestamp, 1, false));
90+ }
91+
92+ let first = data.page(&account, None, 2).unwrap();
93+ assert_eq!(
94+ first
95+ .entries
96+ .iter()
97+ .map(|entry| entry.key.clone())
98+ .collect::<Vec<_>>(),
99+ ordered[..2].to_vec()
100+ );
101+ let mut cursor = first.next_cursor.unwrap();
102+
103+ data.revision = 2;
104+ let later = key("room", "discovered-later");
105+ data.entries.insert(id(&later), entry(later, 50, 2, false));
106+
107+ let mut paged = first
108+ .entries
109+ .into_iter()
110+ .map(|entry| entry.key)
111+ .collect::<Vec<_>>();
112+ loop {
113+ let page = data.page(&account, Some(&cursor), 2).unwrap();
114+ paged.extend(page.entries.into_iter().map(|entry| entry.key));
115+ match page.next_cursor {
116+ Some(next) => cursor = next,
117+ None => break,
118+ }
119+ }
120+ assert_eq!(paged, ordered);
121+}
122+
123+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
124+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
125+async fn pagination_rejects_a_forged_account_and_a_future_revision() {
126+ let account = user_id!("@alice:mentions.example").to_owned();
127+ let mut data = Data {
128+ revision: 1,
129+ ..Data::default()
130+ };
131+ for event in ["one", "two"] {
132+ let mention = key("room", event);
133+ data.entries
134+ .insert(id(&mention), entry(mention, 10, 1, false));
135+ }
136+ let cursor = data.page(&account, None, 1).unwrap().next_cursor.unwrap();
137+
138+ let mut forged_account: serde_json::Value = serde_json::from_str(&cursor).unwrap();
139+ forged_account["account"] = serde_json::Value::String("@mallory:mentions.example".to_owned());
140+ assert!(
141+ data.page(&account, Some(&forged_account.to_string()), 1)
142+ .is_err()
143+ );
144+
145+ let mut future_revision: serde_json::Value = serde_json::from_str(&cursor).unwrap();
146+ future_revision["revision"] = serde_json::json!(data.revision + 1);
147+ assert!(
148+ data.page(&account, Some(&future_revision.to_string()), 1)
149+ .is_err()
150+ );
151+}
152+
153+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
154+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
155+async fn retention_discards_read_context_first_and_keeps_shared_events_until_unused() {
156+ let read_key = key("room", "read-mention");
157+ let unread_key = key("room", "unread-mention");
158+ let shared_key = key("room", "shared-context");
159+ let read_id = id(&read_key);
160+ let unread_id = id(&unread_key);
161+ let shared_id = id(&shared_key);
162+ let mut data = Data {
163+ revision: 3,
164+ ..Data::default()
165+ };
166+ data.entries
167+ .insert(read_id.clone(), entry(read_key, 10, 1, true));
168+ data.entries
169+ .insert(unread_id.clone(), entry(unread_key, 20, 2, false));
170+ data.events.insert(
171+ shared_id.clone(),
172+ SavedEvent {
173+ item: message_event("$shared-context:mentions.example", 15, "shared"),
174+ media: Vec::new(),
175+ raw: None,
176+ },
177+ );
178+ data.windows.insert(
179+ read_id.clone(),
180+ Window {
181+ before: vec![shared_id.clone()],
182+ target: None,
183+ after: Vec::new(),
184+ start: None,
185+ end: None,
186+ },
187+ );
188+ data.windows.insert(
189+ unread_id.clone(),
190+ Window {
191+ before: vec![shared_id.clone()],
192+ target: None,
193+ after: Vec::new(),
194+ start: None,
195+ end: None,
196+ },
197+ );
198+
199+ assert!(data.evict_one());
200+ assert!(!data.windows.contains_key(&read_id));
201+ assert!(data.windows.contains_key(&unread_id));
202+ assert_eq!(
203+ data.entries[&read_id].capture,
204+ MentionCaptureState::Unavailable
205+ );
206+ assert!(!data.entries[&unread_id].read);
207+ assert!(data.events.contains_key(&shared_id));
208+
209+ assert!(data.evict_one());
210+ assert!(data.windows.is_empty());
211+ assert!(data.entries.contains_key(&read_id));
212+ assert!(data.entries.contains_key(&unread_id));
213+ assert!(!data.events.contains_key(&shared_id));
214+}
215+
216+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
217+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
218+async fn context_uses_the_redacted_saved_event_instead_of_the_entry_preview() {
219+ let mention = key("room", "redacted-target");
220+ let mention_id = id(&mention);
221+ let original = message_event(
222+ "$redacted-target:mentions.example",
223+ 42,
224+ "private original body",
225+ );
226+ let mut redacted = original.clone();
227+ if let TimelineItem::Event(event) = &mut redacted {
228+ event.content = TimelineContent::Redacted;
229+ event.reply_to = None;
230+ event.reactions.clear();
231+ }
232+ let mut data = Data::default();
233+ data.entries.insert(
234+ mention_id.clone(),
235+ MentionEntry {
236+ event: original,
237+ ..entry(mention.clone(), 42, 1, false)
238+ },
239+ );
240+ data.events.insert(
241+ mention_id.clone(),
242+ SavedEvent {
243+ item: redacted,
244+ media: Vec::new(),
245+ raw: None,
246+ },
247+ );
248+ data.windows.insert(
249+ mention_id,
250+ Window {
251+ before: Vec::new(),
252+ target: Some(id(&mention)),
253+ after: Vec::new(),
254+ start: Some("start".to_owned()),
255+ end: Some("end".to_owned()),
256+ },
257+ );
258+
259+ let (context, media) = data.context(&mention).unwrap();
260+ let saved = context.context.unwrap().event.unwrap();
261+ let TimelineItem::Event(saved) = saved else {
262+ panic!("the saved target should remain a timeline event");
263+ };
264+ assert_eq!(saved.content, TimelineContent::Redacted);
265+ assert_eq!(
266+ saved.event_id.as_deref(),
267+ Some(event_id!("$redacted-target:mentions.example"))
268+ );
269+ assert!(media.is_empty());
270+}
271+
272+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
273+mod persistence {
274+ use std::sync::atomic::{AtomicUsize, Ordering};
275+
276+ use ruma::{events::room::MediaSource, user_id};
277+ use serde_json::json;
278+
279+ use super::*;
280+ use crate::{
281+ MediaId, MediaKind, StoreConfig,
282+ mentions::store::{Account, MentionsStore, Write},
283+ normalise::{MediaRegistration, MediaSourceInfo},
284+ };
285+
286+ static STORES: AtomicUsize = AtomicUsize::new(0);
287+ const MAX_BYTES: usize = 64 * 1024 * 1024;
288+
289+ fn account() -> Account {
290+ Account {
291+ homeserver: "https://mentions.example/".to_owned(),
292+ user_id: user_id!("@alice:mentions.example").to_owned(),
293+ }
294+ }
295+
296+ fn config() -> StoreConfig {
297+ let number = STORES.fetch_add(1, Ordering::Relaxed);
298+ #[cfg(feature = "sqlite")]
299+ {
300+ StoreConfig::sqlite(
301+ std::env::temp_dir().join(format!(
302+ "macltok-mentions-state-{}-{number}",
303+ std::process::id()
304+ )),
305+ "state test passphrase",
306+ )
307+ }
308+ #[cfg(feature = "indexeddb")]
309+ {
310+ StoreConfig::indexeddb(
311+ format!("macltok-mentions-state-{}-{number}", js_sys::Date::now()),
312+ "state test passphrase",
313+ )
314+ }
315+ }
316+
317+ fn media_registration() -> MediaRegistration {
318+ MediaRegistration {
319+ id: MediaId::new("test-scope#image".to_owned()),
320+ kind: MediaKind::Image,
321+ full: MediaSourceInfo {
322+ source: MediaSource::Plain(
323+ ruma::mxc_uri!("mxc://mentions.example/image").to_owned(),
324+ ),
325+ mimetype: Some("image/png".to_owned()),
326+ declared_size: Some(1234),
327+ },
328+ thumbnail: None,
329+ }
330+ }
331+
332+ fn saved_data() -> Data {
333+ let mention = key("persisted-room", "persisted-mention");
334+ let mention_id = id(&mention);
335+ let context_item =
336+ message_event("$persisted-mention:mentions.example", 42, "saved message");
337+ let mut data = Data {
338+ revision: 1,
339+ ..Data::default()
340+ };
341+ data.entries.insert(
342+ mention_id.clone(),
343+ MentionEntry {
344+ event: context_item.clone(),
345+ ..entry(mention.clone(), 42, 1, true)
346+ },
347+ );
348+ data.events.insert(
349+ mention_id.clone(),
350+ SavedEvent {
351+ item: context_item,
352+ media: vec![media_registration()],
353+ raw: Some(
354+ json!({"event_id": "$persisted-mention:mentions.example", "body": "saved raw"}),
355+ ),
356+ },
357+ );
358+ data.windows.insert(
359+ mention_id.clone(),
360+ Window {
361+ before: Vec::new(),
362+ target: Some(mention_id),
363+ after: Vec::new(),
364+ start: Some("start-token".to_owned()),
365+ end: Some("end-token".to_owned()),
366+ },
367+ );
368+ data.checkpoints
369+ .insert("sync-connection".to_owned(), "position-1".to_owned());
370+ data.history_cursor = Some("history-position".to_owned());
371+ data
372+ }
373+
374+ async fn open(config: &StoreConfig) -> MentionsStore {
375+ MentionsStore::open(config, account(), MAX_BYTES)
376+ .await
377+ .unwrap()
378+ }
379+
380+ #[cfg_attr(not(target_family = "wasm"), tokio::test)]
381+ #[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
382+ async fn encrypted_store_roundtrips_inbox_context_read_state_and_checkpoints() {
383+ let config = config();
384+ let data = saved_data();
385+ let mut store = open(&config).await;
386+ assert_eq!(
387+ store
388+ .commit(
389+ 0,
390+ vec![Write::Set(
391+ "inbox".to_owned(),
392+ serde_json::to_value(&data).unwrap(),
393+ )],
394+ )
395+ .await
396+ .unwrap(),
397+ 1
398+ );
399+ drop(store);
400+
401+ let mut restored = open(&config).await;
402+ let mut restored_data: Data = restored.read("inbox").unwrap().unwrap();
403+ assert_eq!(restored.revision(), restored_data.revision);
404+ assert!(restored_data.entries.values().next().unwrap().read);
405+ assert_eq!(
406+ restored_data
407+ .checkpoints
408+ .get("sync-connection")
409+ .map(String::as_str),
410+ Some("position-1")
411+ );
412+ let mention = restored_data.entries.values().next().unwrap().key.clone();
413+ let (context, media) = restored_data.context(&mention).unwrap();
414+ assert_eq!(
415+ context.context.unwrap().start_token.as_deref(),
416+ Some("start-token")
417+ );
418+ assert_eq!(media.len(), 1);
419+ assert_eq!(media[0].id.as_str(), "test-scope#image");
420+ assert_eq!(media[0].full.mimetype.as_deref(), Some("image/png"));
421+ let event = restored_data.events.values().next().unwrap();
422+ assert_eq!(event.raw.as_ref().unwrap()["body"], "saved raw");
423+
424+ restored_data.revision = 2;
425+ let updated_entry = restored_data.entries.values_mut().next().unwrap();
426+ updated_entry.read = false;
427+ updated_entry.revision = 2;
428+ restored_data
429+ .checkpoints
430+ .insert("sync-connection".to_owned(), "position-2".to_owned());
431+ restored
432+ .commit(
433+ 1,
434+ vec![Write::Set(
435+ "inbox".to_owned(),
436+ serde_json::to_value(&restored_data).unwrap(),
437+ )],
438+ )
439+ .await
440+ .unwrap();
441+ drop(restored);
442+
443+ let restored = open(&config).await;
444+ let restored_data: Data = restored.read("inbox").unwrap().unwrap();
445+ assert_eq!(restored.revision(), 2);
446+ assert_eq!(restored_data.revision, 2);
447+ assert!(!restored_data.entries.values().next().unwrap().read);
448+ assert_eq!(
449+ restored_data
450+ .checkpoints
451+ .get("sync-connection")
452+ .map(String::as_str),
453+ Some("position-2")
454+ );
455+ drop(restored);
456+ config.delete().await.unwrap();
457+ }
458+}
Added file: src/mentions/store/indexeddb.rs+217 additions
@@ -0,0 +1,217 @@
1+use std::{collections::BTreeMap, ops::Deref};
2+
3+use indexed_db_futures::{
4+ Build,
5+ database::Database,
6+ error::{Error as IndexedDbError, OpenDbError},
7+ prelude::*,
8+ transaction::{Transaction, TransactionDurability, TransactionMode, TransactionOptions},
9+ typed_array::{Uint8Array, Uint8ArraySlice},
10+};
11+
12+use super::Snapshot;
13+
14+const RECORDS: &str = "records";
15+const CIPHER: &str = "!cipher";
16+const HEAD: &str = "!head";
17+const INCOMPLETE_METADATA: &str = "mentions database metadata is incomplete";
18+const INVALID_RECORDS: &str = "mentions database records are inconsistent";
19+const RESERVED_ROW_KEY: &str = "mentions row key uses a reserved metadata key";
20+
21+struct DatabaseHandle(Database);
22+
23+impl Deref for DatabaseHandle {
24+ type Target = Database;
25+
26+ fn deref(&self) -> &Self::Target {
27+ &self.0
28+ }
29+}
30+
31+impl Drop for DatabaseHandle {
32+ fn drop(&mut self) {
33+ self.0.clone().close();
34+ }
35+}
36+
37+pub(super) struct Backend {
38+ name: String,
39+}
40+
41+impl Backend {
42+ pub(super) async fn open(name: String) -> Result<Self, crate::Error> {
43+ let _database = open_database(&name)
44+ .await
45+ .map_err(|error| super::storage_error("opening the mentions database", error))?;
46+ Ok(Self { name })
47+ }
48+
49+ pub(super) async fn load(&self) -> Result<Option<Snapshot>, crate::Error> {
50+ let database = open_database(&self.name)
51+ .await
52+ .map_err(|error| super::storage_error("opening the mentions database", error))?;
53+ let snapshot = read_snapshot(&database).await;
54+ match snapshot {
55+ Ok(snapshot) => Ok(snapshot),
56+ Err(ReadError::Invalid(message)) => {
57+ Err(crate::Error::new(crate::ErrorKind::Store, message))
58+ }
59+ Err(ReadError::Storage(error)) => {
60+ Err(super::storage_error("reading the mentions database", error))
61+ }
62+ }
63+ }
64+
65+ pub(super) async fn commit(
66+ &self,
67+ expected: Option<Vec<u8>>,
68+ exported: Vec<u8>,
69+ head: Vec<u8>,
70+ writes: BTreeMap<String, Option<Vec<u8>>>,
71+ ) -> Result<(), crate::Error> {
72+ let database = open_database(&self.name)
73+ .await
74+ .map_err(|error| super::storage_error("opening the mentions database", error))?;
75+ let committed = write_snapshot(&database, expected, exported, head, writes).await;
76+ match committed {
77+ Ok(()) => Ok(()),
78+ Err(CommitError::Conflict) => Err(super::conflict()),
79+ Err(CommitError::Invalid(message)) => {
80+ Err(crate::Error::new(crate::ErrorKind::Store, message))
81+ }
82+ Err(CommitError::Storage(error)) => {
83+ Err(super::storage_error("writing the mentions database", error))
84+ }
85+ }
86+ }
87+}
88+
89+async fn open_database(name: &str) -> Result<DatabaseHandle, OpenDbError> {
90+ Database::open(name)
91+ .with_version(1u32)
92+ .with_on_upgrade_needed(
93+ |_, transaction: &Transaction<'_>| -> Result<(), IndexedDbError> {
94+ transaction.db().create_object_store(RECORDS).build()?;
95+ Ok(())
96+ },
97+ )
98+ .build()?
99+ .await
100+ .map(DatabaseHandle)
101+}
102+
103+enum ReadError {
104+ Invalid(&'static str),
105+ Storage(IndexedDbError),
106+}
107+
108+impl From<IndexedDbError> for ReadError {
109+ fn from(error: IndexedDbError) -> Self {
110+ Self::Storage(error)
111+ }
112+}
113+
114+async fn read_snapshot(database: &Database) -> Result<Option<Snapshot>, ReadError> {
115+ let transaction = database
116+ .transaction(RECORDS)
117+ .with_mode(TransactionMode::Readonly)
118+ .build()?;
119+ let store = transaction.object_store(RECORDS)?;
120+ let keys = store.get_all_keys::<String>().build()?;
121+ let values = store.get_all::<Uint8Array>().build()?;
122+ let keys = keys.await?.collect::<Result<Vec<_>, _>>()?;
123+ let values = values.await?.collect::<Result<Vec<_>, _>>()?;
124+ if keys.len() != values.len() {
125+ return Err(ReadError::Invalid(INVALID_RECORDS));
126+ }
127+ let mut records = keys
128+ .into_iter()
129+ .zip(values)
130+ .map(|(key, value)| (key, Vec::<u8>::from(value)))
131+ .collect::<BTreeMap<_, _>>();
132+ if records.is_empty() {
133+ return Ok(None);
134+ }
135+ let exported = records.remove(CIPHER);
136+ let head = records.remove(HEAD);
137+ match (exported, head) {
138+ (Some(exported), Some(head)) => Ok(Some(Snapshot {
139+ exported,
140+ head,
141+ rows: records,
142+ })),
143+ _ => Err(ReadError::Invalid(INCOMPLETE_METADATA)),
144+ }
145+}
146+
147+enum CommitError {
148+ Conflict,
149+ Invalid(&'static str),
150+ Storage(IndexedDbError),
151+}
152+
153+impl From<IndexedDbError> for CommitError {
154+ fn from(error: IndexedDbError) -> Self {
155+ Self::Storage(error)
156+ }
157+}
158+
159+async fn write_snapshot(
160+ database: &Database,
161+ expected: Option<Vec<u8>>,
162+ exported: Vec<u8>,
163+ head: Vec<u8>,
164+ writes: BTreeMap<String, Option<Vec<u8>>>,
165+) -> Result<(), CommitError> {
166+ let mut options = TransactionOptions::new();
167+ options.set_durability(TransactionDurability::Strict);
168+ let transaction = database
169+ .transaction(RECORDS)
170+ .with_mode(TransactionMode::Readwrite)
171+ .with_options(options)
172+ .build()?;
173+ let current = transaction
174+ .object_store(RECORDS)?
175+ .get::<Uint8Array, _, _>(HEAD)
176+ .await?
177+ .map(Vec::<u8>::from);
178+ if current != expected {
179+ let _ = transaction.abort().await;
180+ return Err(CommitError::Conflict);
181+ }
182+ if writes.keys().any(|key| key == CIPHER || key == HEAD) {
183+ let _ = transaction.abort().await;
184+ return Err(CommitError::Invalid(RESERVED_ROW_KEY));
185+ }
186+ let requests = (|| -> Result<(), IndexedDbError> {
187+ let store = transaction.object_store(RECORDS)?;
188+ store
189+ .put(Uint8ArraySlice::new(&exported))
190+ .with_key(CIPHER)
191+ .build()?;
192+ store
193+ .put(Uint8ArraySlice::new(&head))
194+ .with_key(HEAD)
195+ .build()?;
196+ for (key, value) in writes {
197+ match value {
198+ Some(value) => {
199+ store
200+ .put(Uint8ArraySlice::new(&value))
201+ .with_key(key)
202+ .build()?;
203+ }
204+ None => {
205+ store.delete(key).build()?;
206+ }
207+ }
208+ }
209+ Ok(())
210+ })();
211+ if let Err(error) = requests {
212+ let _ = transaction.abort().await;
213+ return Err(CommitError::Storage(error));
214+ }
215+ transaction.commit().await?;
216+ Ok(())
217+}
Added file: src/mentions/store/mod.rs+336 additions
@@ -0,0 +1,336 @@
1+use std::{collections::BTreeMap, fmt};
2+
3+use matrix_sdk::{SendOutsideWasm, SyncOutsideWasm};
4+use matrix_sdk_store_encryption::StoreCipher;
5+use ruma::OwnedUserId;
6+use serde::{Deserialize, Serialize, de::DeserializeOwned};
7+use serde_json::Value;
8+
9+use crate::{Error, ErrorKind, StoreConfig};
10+
11+#[cfg(feature = "sqlite")]
12+mod sqlite;
13+#[cfg(feature = "sqlite")]
14+use sqlite::Backend;
15+#[cfg(feature = "indexeddb")]
16+mod indexeddb;
17+#[cfg(feature = "indexeddb")]
18+use indexeddb::Backend;
19+
20+const SCHEMA: u32 = 1;
21+const MAX_RECORDS: usize = 35_128;
22+
23+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
24+pub(crate) struct Account {
25+ pub(crate) homeserver: String,
26+ pub(crate) user_id: OwnedUserId,
27+}
28+
29+#[derive(Serialize, Deserialize)]
30+struct Head {
31+ schema: u32,
32+ account: Account,
33+ revision: u64,
34+ records: usize,
35+ bytes: usize,
36+}
37+
38+#[derive(Serialize, Deserialize)]
39+struct Record {
40+ schema: u32,
41+ account: Account,
42+ key: String,
43+ value: Value,
44+}
45+
46+pub(super) struct Snapshot {
47+ pub(super) exported: Vec<u8>,
48+ pub(super) head: Vec<u8>,
49+ pub(super) rows: BTreeMap<String, Vec<u8>>,
50+}
51+
52+pub(crate) enum Write {
53+ Set(String, Value),
54+ Remove(String),
55+}
56+
57+pub(crate) struct MentionsStore {
58+ backend: Backend,
59+ cipher: StoreCipher,
60+ account: Account,
61+ snapshot: Snapshot,
62+ revision: u64,
63+ max_bytes: usize,
64+}
65+
66+impl fmt::Debug for MentionsStore {
67+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
68+ formatter
69+ .debug_struct("MentionsStore")
70+ .field("revision", &self.revision)
71+ .field("records", &self.snapshot.rows.len())
72+ .finish_non_exhaustive()
73+ }
74+}
75+
76+impl MentionsStore {
77+ pub(crate) async fn open(
78+ config: &StoreConfig,
79+ account: Account,
80+ max_bytes: usize,
81+ ) -> Result<Self, Error> {
82+ let backend = Backend::open(config.mentions_location()).await?;
83+ let (cipher, snapshot, revision) = match backend.load().await? {
84+ Some(snapshot) => {
85+ validate_size(&snapshot, max_bytes)?;
86+ let cipher = StoreCipher::import(config.passphrase(), &snapshot.exported)
87+ .map_err(|error| storage_error("opening the mentions cipher", error))?;
88+ let head: Head = cipher
89+ .decrypt_value(&snapshot.head)
90+ .map_err(|error| storage_error("opening the mentions header", error))?;
91+ validate_account(head.schema, &head.account, &account)?;
92+ if head.records != snapshot.rows.len() || head.bytes != row_bytes(&snapshot.rows)? {
93+ return Err(invalid(
94+ "mentions records do not match their committed header",
95+ ));
96+ }
97+ for (key, encrypted) in &snapshot.rows {
98+ let record: Record = cipher
99+ .decrypt_value(encrypted)
100+ .map_err(|error| storage_error("opening a mentions record", error))?;
101+ validate_account(record.schema, &record.account, &account)?;
102+ if storage_key(&cipher, &record.key) != *key {
103+ return Err(invalid("a mentions record belongs to a different key"));
104+ }
105+ }
106+ let revision = head.revision;
107+ (cipher, snapshot, revision)
108+ }
109+ None => {
110+ let cipher = StoreCipher::new()
111+ .map_err(|error| storage_error("creating the mentions cipher", error))?;
112+ let exported = cipher
113+ .export(config.passphrase())
114+ .map_err(|error| storage_error("sealing the mentions cipher", error))?;
115+ let head = cipher
116+ .encrypt_value(&Head {
117+ schema: SCHEMA,
118+ account: account.clone(),
119+ revision: 0,
120+ records: 0,
121+ bytes: 0,
122+ })
123+ .map_err(|error| storage_error("sealing the mentions header", error))?;
124+ let snapshot = Snapshot {
125+ exported,
126+ head,
127+ rows: BTreeMap::new(),
128+ };
129+ validate_size(&snapshot, max_bytes)?;
130+ backend
131+ .commit(
132+ None,
133+ snapshot.exported.clone(),
134+ snapshot.head.clone(),
135+ BTreeMap::new(),
136+ )
137+ .await?;
138+ (cipher, snapshot, 0)
139+ }
140+ };
141+ validate_size(&snapshot, max_bytes)?;
142+ Ok(Self {
143+ backend,
144+ cipher,
145+ account,
146+ snapshot,
147+ revision,
148+ max_bytes,
149+ })
150+ }
151+
152+ pub(crate) fn revision(&self) -> u64 {
153+ self.revision
154+ }
155+
156+ pub(crate) fn read<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>, Error> {
157+ let Some(encrypted) = self.snapshot.rows.get(&storage_key(&self.cipher, key)) else {
158+ return Ok(None);
159+ };
160+ let record: Record = self
161+ .cipher
162+ .decrypt_value(encrypted)
163+ .map_err(|error| storage_error("opening a mentions record", error))?;
164+ validate_account(record.schema, &record.account, &self.account)?;
165+ if record.key != key {
166+ return Err(invalid("a mentions record belongs to a different key"));
167+ }
168+ serde_json::from_value(record.value)
169+ .map(Some)
170+ .map_err(|error| storage_error("decoding a mentions record", error))
171+ }
172+
173+ pub(crate) async fn commit(
174+ &mut self,
175+ expected_revision: u64,
176+ writes: Vec<Write>,
177+ ) -> Result<u64, Error> {
178+ if self.revision != expected_revision {
179+ return Err(conflict());
180+ }
181+ if writes.is_empty() {
182+ return Ok(self.revision);
183+ }
184+ let revision = self
185+ .revision
186+ .checked_add(1)
187+ .ok_or_else(|| invalid("mentions revision exhausted"))?;
188+ let mut changes = BTreeMap::new();
189+ for write in writes {
190+ match write {
191+ Write::Set(key, value) => {
192+ let hashed = storage_key(&self.cipher, &key);
193+ let record = Record {
194+ schema: SCHEMA,
195+ account: self.account.clone(),
196+ key,
197+ value,
198+ };
199+ let encrypted = self
200+ .cipher
201+ .encrypt_value(&record)
202+ .map_err(|error| storage_error("sealing a mentions record", error))?;
203+ changes.insert(hashed, Some(encrypted));
204+ }
205+ Write::Remove(key) => {
206+ changes.insert(storage_key(&self.cipher, &key), None);
207+ }
208+ }
209+ }
210+ let mut records = self.snapshot.rows.len();
211+ let mut bytes = row_bytes(&self.snapshot.rows)?;
212+ for (key, value) in &changes {
213+ if let Some(previous) = self.snapshot.rows.get(key) {
214+ bytes -= key.len() + previous.len();
215+ records -= 1;
216+ }
217+ if let Some(value) = value {
218+ bytes = bytes
219+ .checked_add(key.len())
220+ .and_then(|bytes| bytes.checked_add(value.len()))
221+ .ok_or_else(|| invalid("mentions byte count overflow"))?;
222+ records += 1;
223+ }
224+ }
225+ let head = self
226+ .cipher
227+ .encrypt_value(&Head {
228+ schema: SCHEMA,
229+ account: self.account.clone(),
230+ revision,
231+ records,
232+ bytes,
233+ })
234+ .map_err(|error| storage_error("sealing the mentions header", error))?;
235+ validate_budget(
236+ bytes,
237+ records,
238+ self.snapshot.exported.len(),
239+ head.len(),
240+ self.max_bytes,
241+ )?;
242+ self.backend
243+ .commit(
244+ Some(self.snapshot.head.clone()),
245+ self.snapshot.exported.clone(),
246+ head.clone(),
247+ changes.clone(),
248+ )
249+ .await?;
250+ for (key, value) in changes {
251+ match value {
252+ Some(value) => {
253+ self.snapshot.rows.insert(key, value);
254+ }
255+ None => {
256+ self.snapshot.rows.remove(&key);
257+ }
258+ }
259+ }
260+ self.snapshot.head = head;
261+ self.revision = revision;
262+ Ok(revision)
263+ }
264+}
265+
266+fn validate_account(schema: u32, actual: &Account, expected: &Account) -> Result<(), Error> {
267+ if schema != SCHEMA {
268+ return Err(invalid("unsupported mentions schema"));
269+ }
270+ if actual != expected {
271+ return Err(invalid("mentions store belongs to a different account"));
272+ }
273+ Ok(())
274+}
275+
276+fn validate_size(snapshot: &Snapshot, max_bytes: usize) -> Result<(), Error> {
277+ validate_budget(
278+ row_bytes(&snapshot.rows)?,
279+ snapshot.rows.len(),
280+ snapshot.exported.len(),
281+ snapshot.head.len(),
282+ max_bytes,
283+ )
284+}
285+
286+fn validate_budget(
287+ row_bytes: usize,
288+ records: usize,
289+ exported_len: usize,
290+ head_len: usize,
291+ max_bytes: usize,
292+) -> Result<(), Error> {
293+ let bytes = row_bytes
294+ .checked_add(exported_len)
295+ .and_then(|bytes| bytes.checked_add(head_len))
296+ .ok_or_else(|| invalid("mentions byte count overflow"))?;
297+ if bytes > max_bytes || records > MAX_RECORDS {
298+ return Err(invalid("mentions storage limit exceeded"));
299+ }
300+ Ok(())
301+}
302+
303+fn row_bytes(rows: &BTreeMap<String, Vec<u8>>) -> Result<usize, Error> {
304+ rows.iter().try_fold(0usize, |bytes, (key, value)| {
305+ bytes
306+ .checked_add(key.len())
307+ .and_then(|bytes| bytes.checked_add(value.len()))
308+ .ok_or_else(|| invalid("mentions byte count overflow"))
309+ })
310+}
311+
312+fn storage_key(cipher: &StoreCipher, key: &str) -> String {
313+ cipher
314+ .hash_key("macltok-mentions", key.as_bytes())
315+ .iter()
316+ .map(|byte| format!("{byte:02x}"))
317+ .collect()
318+}
319+
320+fn invalid(message: &'static str) -> Error {
321+ Error::new(ErrorKind::Store, message)
322+}
323+
324+pub(super) fn conflict() -> Error {
325+ invalid("mentions store changed, reopen before retrying")
326+}
327+
328+pub(super) fn storage_error(
329+ context: &'static str,
330+ source: impl std::error::Error + SendOutsideWasm + SyncOutsideWasm + 'static,
331+) -> Error {
332+ Error::with_source(ErrorKind::Store, context, source)
333+}
334+
335+#[cfg(test)]
336+mod tests;
Added file: src/mentions/store/sqlite.rs+214 additions
@@ -0,0 +1,214 @@
1+use std::{
2+ collections::BTreeMap,
3+ fs::{self, OpenOptions},
4+ io,
5+ path::{Path, PathBuf},
6+ time::Duration,
7+};
8+
9+use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
10+
11+use super::{Snapshot, conflict, storage_error};
12+use crate::{Error, ErrorKind};
13+
14+const CIPHER_KEY: &str = "!cipher";
15+const HEAD_KEY: &str = "!head";
16+
17+pub(super) struct Backend {
18+ path: PathBuf,
19+}
20+
21+impl Backend {
22+ pub(super) async fn open(path: PathBuf) -> Result<Self, Error> {
23+ let setup_path = path.clone();
24+ tokio::task::spawn_blocking(move || initialize(&setup_path))
25+ .await
26+ .map_err(|error| storage_error("opening the mentions store", error))??;
27+ Ok(Self { path })
28+ }
29+
30+ pub(super) async fn load(&self) -> Result<Option<Snapshot>, Error> {
31+ let path = self.path.clone();
32+ tokio::task::spawn_blocking(move || load_snapshot(&path))
33+ .await
34+ .map_err(|error| storage_error("loading the mentions store", error))?
35+ }
36+
37+ pub(super) async fn commit(
38+ &self,
39+ expected: Option<Vec<u8>>,
40+ exported: Vec<u8>,
41+ head: Vec<u8>,
42+ writes: BTreeMap<String, Option<Vec<u8>>>,
43+ ) -> Result<(), Error> {
44+ let path = self.path.clone();
45+ tokio::task::spawn_blocking(move || {
46+ commit_snapshot(&path, expected, exported, head, writes)
47+ })
48+ .await
49+ .map_err(|error| storage_error("committing the mentions store", error))?
50+ }
51+}
52+
53+fn initialize(path: &Path) -> Result<(), Error> {
54+ let parent = path
55+ .parent()
56+ .filter(|parent| !parent.as_os_str().is_empty())
57+ .unwrap_or_else(|| Path::new("."));
58+ fs::create_dir_all(parent)
59+ .map_err(|error| storage_error("creating the mentions store directory", error))?;
60+ create_private_file(path)
61+ .map_err(|error| storage_error("creating the mentions database file", error))?;
62+ let connection = connect(path)?;
63+ connection
64+ .execute(
65+ "CREATE TABLE IF NOT EXISTS records (key TEXT PRIMARY KEY, value BLOB NOT NULL)",
66+ [],
67+ )
68+ .map_err(|error| storage_error("initializing the mentions database", error))?;
69+ Ok(())
70+}
71+
72+fn connect(path: &Path) -> Result<Connection, Error> {
73+ let connection = Connection::open(path)
74+ .map_err(|error| storage_error("opening the mentions database", error))?;
75+ connection
76+ .busy_timeout(Duration::from_secs(5))
77+ .map_err(|error| storage_error("configuring the mentions database", error))?;
78+ connection
79+ .execute_batch("PRAGMA synchronous = FULL")
80+ .map_err(|error| storage_error("configuring the mentions database", error))?;
81+ Ok(connection)
82+}
83+
84+fn load_snapshot(path: &Path) -> Result<Option<Snapshot>, Error> {
85+ let mut connection = connect(path)?;
86+ let transaction = connection
87+ .transaction_with_behavior(TransactionBehavior::Deferred)
88+ .map_err(|error| storage_error("starting a mentions snapshot", error))?;
89+ let mut rows = BTreeMap::new();
90+ {
91+ let mut statement = transaction
92+ .prepare("SELECT key, value FROM records")
93+ .map_err(|error| storage_error("reading the mentions snapshot", error))?;
94+ let result = statement
95+ .query_map([], |row| {
96+ Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))
97+ })
98+ .map_err(|error| storage_error("reading the mentions snapshot", error))?;
99+ for row in result {
100+ let (key, value) =
101+ row.map_err(|error| storage_error("reading the mentions snapshot", error))?;
102+ rows.insert(key, value);
103+ }
104+ }
105+ transaction
106+ .commit()
107+ .map_err(|error| storage_error("finishing the mentions snapshot", error))?;
108+ if rows.is_empty() {
109+ return Ok(None);
110+ }
111+ let exported = rows.remove(CIPHER_KEY).ok_or_else(|| {
112+ Error::new(
113+ ErrorKind::Store,
114+ "mentions store is missing cipher metadata",
115+ )
116+ })?;
117+ let head = rows
118+ .remove(HEAD_KEY)
119+ .ok_or_else(|| Error::new(ErrorKind::Store, "mentions store is missing head metadata"))?;
120+ Ok(Some(Snapshot {
121+ exported,
122+ head,
123+ rows,
124+ }))
125+}
126+
127+fn commit_snapshot(
128+ path: &Path,
129+ expected: Option<Vec<u8>>,
130+ exported: Vec<u8>,
131+ head: Vec<u8>,
132+ writes: BTreeMap<String, Option<Vec<u8>>>,
133+) -> Result<(), Error> {
134+ let mut connection = connect(path)?;
135+ let transaction = connection
136+ .transaction_with_behavior(TransactionBehavior::Immediate)
137+ .map_err(|error| storage_error("starting a mentions commit", error))?;
138+ let current: Option<Vec<u8>> = transaction
139+ .query_row(
140+ "SELECT value FROM records WHERE key = ?1",
141+ [HEAD_KEY],
142+ |row| row.get(0),
143+ )
144+ .optional()
145+ .map_err(|error| storage_error("checking the mentions store head", error))?;
146+ if current != expected {
147+ return Err(conflict());
148+ }
149+ if writes.contains_key(CIPHER_KEY) || writes.contains_key(HEAD_KEY) {
150+ return Err(Error::new(
151+ ErrorKind::Store,
152+ "mentions record uses a reserved key",
153+ ));
154+ }
155+ transaction
156+ .execute(
157+ "INSERT INTO records (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
158+ params![CIPHER_KEY, exported],
159+ )
160+ .map_err(|error| storage_error("writing mentions cipher metadata", error))?;
161+ transaction
162+ .execute(
163+ "INSERT INTO records (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
164+ params![HEAD_KEY, head],
165+ )
166+ .map_err(|error| storage_error("writing the mentions store head", error))?;
167+ for (key, value) in writes {
168+ match value {
169+ Some(value) => {
170+ transaction
171+ .execute(
172+ "INSERT INTO records (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
173+ params![key, value],
174+ )
175+ .map_err(|error| storage_error("writing a mentions record", error))?;
176+ }
177+ None => {
178+ transaction
179+ .execute("DELETE FROM records WHERE key = ?1", [key])
180+ .map_err(|error| storage_error("deleting a mentions record", error))?;
181+ }
182+ }
183+ }
184+ transaction
185+ .commit()
186+ .map_err(|error| storage_error("finishing a mentions commit", error))
187+}
188+
189+#[cfg(unix)]
190+fn create_private_file(path: &Path) -> io::Result<()> {
191+ use std::os::unix::fs::OpenOptionsExt;
192+
193+ match OpenOptions::new()
194+ .write(true)
195+ .create_new(true)
196+ .mode(0o600)
197+ .open(path)
198+ {
199+ Ok(file) => drop(file),
200+ Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
201+ Err(error) => return Err(error),
202+ }
203+ Ok(())
204+}
205+
206+#[cfg(not(unix))]
207+fn create_private_file(path: &Path) -> io::Result<()> {
208+ match OpenOptions::new().write(true).create_new(true).open(path) {
209+ Ok(file) => drop(file),
210+ Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
211+ Err(error) => return Err(error),
212+ }
213+ Ok(())
214+}
Added file: src/mentions/store/tests.rs+418 additions
@@ -0,0 +1,418 @@
1+use std::sync::atomic::{AtomicUsize, Ordering};
2+
3+use ruma::user_id;
4+use serde_json::{Value, json};
5+
6+use super::{Account, Head, MentionsStore, Write, row_bytes, storage_key};
7+use crate::StoreConfig;
8+
9+#[cfg(target_family = "wasm")]
10+wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
11+
12+static STORES: AtomicUsize = AtomicUsize::new(0);
13+const MAX_BYTES: usize = 64 * 1024 * 1024;
14+
15+fn account() -> Account {
16+ Account {
17+ homeserver: "https://mentions.example/".to_owned(),
18+ user_id: user_id!("@alice:mentions.example").to_owned(),
19+ }
20+}
21+
22+fn config() -> StoreConfig {
23+ let number = STORES.fetch_add(1, Ordering::Relaxed);
24+ #[cfg(feature = "sqlite")]
25+ return StoreConfig::sqlite(
26+ std::env::temp_dir().join(format!("macltok-mentions-{}-{number}", std::process::id())),
27+ "test store passphrase",
28+ );
29+ #[cfg(feature = "indexeddb")]
30+ return StoreConfig::indexeddb(
31+ format!("macltok-mentions-{}-{number}", js_sys::Date::now()),
32+ "test store passphrase",
33+ );
34+}
35+
36+fn set(key: &str, value: Value) -> Write {
37+ Write::Set(key.to_owned(), value)
38+}
39+
40+async fn open(config: &StoreConfig) -> MentionsStore {
41+ MentionsStore::open(config, account(), MAX_BYTES)
42+ .await
43+ .unwrap()
44+}
45+
46+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
47+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
48+async fn entries_context_read_state_and_checkpoint_restore_together() {
49+ let config = config();
50+ let mut store = open(&config).await;
51+ assert_eq!(store.revision(), 0);
52+ assert_eq!(
53+ store
54+ .commit(
55+ 0,
56+ vec![
57+ set(
58+ "entry/room/event",
59+ json!({"preview": "secret mention preview"})
60+ ),
61+ set("context/room/event", json!({"body": "secret context body"})),
62+ set("read/room/event", json!(false)),
63+ set("checkpoint/sync", json!("position-1")),
64+ ]
65+ )
66+ .await
67+ .unwrap(),
68+ 1
69+ );
70+ drop(store);
71+ let mut restored = open(&config).await;
72+ assert_eq!(restored.revision(), 1);
73+ assert_eq!(
74+ restored.read::<Value>("entry/room/event").unwrap(),
75+ Some(json!({"preview": "secret mention preview"}))
76+ );
77+ assert_eq!(
78+ restored.read::<Value>("context/room/event").unwrap(),
79+ Some(json!({"body": "secret context body"}))
80+ );
81+ assert_eq!(
82+ restored.read::<bool>("read/room/event").unwrap(),
83+ Some(false)
84+ );
85+ assert_eq!(
86+ restored
87+ .read::<String>("checkpoint/sync")
88+ .unwrap()
89+ .as_deref(),
90+ Some("position-1")
91+ );
92+ restored
93+ .commit(
94+ 1,
95+ vec![
96+ Write::Remove("entry/room/event".to_owned()),
97+ set("read/room/event", json!(true)),
98+ set("checkpoint/sync", json!("position-2")),
99+ ],
100+ )
101+ .await
102+ .unwrap();
103+ drop(restored);
104+ let restored = open(&config).await;
105+ assert_eq!(restored.revision(), 2);
106+ assert!(
107+ restored
108+ .read::<Value>("entry/room/event")
109+ .unwrap()
110+ .is_none()
111+ );
112+ assert_eq!(
113+ restored.read::<bool>("read/room/event").unwrap(),
114+ Some(true)
115+ );
116+ assert_eq!(
117+ restored
118+ .read::<String>("checkpoint/sync")
119+ .unwrap()
120+ .as_deref(),
121+ Some("position-2")
122+ );
123+ drop(restored);
124+ config.delete().await.unwrap();
125+}
126+
127+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
128+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
129+async fn stale_writers_cannot_overwrite_a_committed_checkpoint() {
130+ let config = config();
131+ let mut first = open(&config).await;
132+ let mut stale = open(&config).await;
133+ first
134+ .commit(0, vec![set("checkpoint", json!("committed"))])
135+ .await
136+ .unwrap();
137+ assert!(
138+ stale
139+ .commit(
140+ 0,
141+ vec![
142+ set("checkpoint", json!("stale")),
143+ set("entry", json!("uncommitted")),
144+ ]
145+ )
146+ .await
147+ .is_err()
148+ );
149+ assert_eq!(stale.revision(), 0);
150+ assert!(stale.read::<Value>("entry").unwrap().is_none());
151+ assert!(
152+ first
153+ .commit(0, vec![set("checkpoint", json!("wrong revision"))])
154+ .await
155+ .is_err()
156+ );
157+ let restored = open(&config).await;
158+ assert_eq!(restored.revision(), 1);
159+ assert_eq!(
160+ restored.read::<String>("checkpoint").unwrap().as_deref(),
161+ Some("committed")
162+ );
163+ assert!(restored.read::<Value>("entry").unwrap().is_none());
164+ drop((first, stale, restored));
165+ config.delete().await.unwrap();
166+}
167+
168+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
169+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
170+async fn capacity_failure_keeps_the_old_revision_and_checkpoint() {
171+ let config = config();
172+ let mut store = MentionsStore::open(&config, account(), 4096).await.unwrap();
173+ store
174+ .commit(0, vec![set("checkpoint", json!("old"))])
175+ .await
176+ .unwrap();
177+ assert!(
178+ store
179+ .commit(
180+ 1,
181+ vec![
182+ set("checkpoint", json!("new")),
183+ set("entry", json!("x".repeat(16384))),
184+ ]
185+ )
186+ .await
187+ .is_err()
188+ );
189+ assert_eq!(store.revision(), 1);
190+ assert_eq!(
191+ store.read::<String>("checkpoint").unwrap().as_deref(),
192+ Some("old")
193+ );
194+ assert!(store.read::<Value>("entry").unwrap().is_none());
195+ drop(store);
196+ let restored = open(&config).await;
197+ assert_eq!(restored.revision(), 1);
198+ assert_eq!(
199+ restored.read::<String>("checkpoint").unwrap().as_deref(),
200+ Some("old")
201+ );
202+ drop(restored);
203+ config.delete().await.unwrap();
204+}
205+
206+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
207+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
208+async fn records_hide_content_keys_and_account_metadata() {
209+ let config = config();
210+ let mut store = open(&config).await;
211+ store
212+ .commit(
213+ 0,
214+ vec![set(
215+ "private/room/event",
216+ json!({"body": "a secret saved message"}),
217+ )],
218+ )
219+ .await
220+ .unwrap();
221+ let snapshot = store.backend.load().await.unwrap().unwrap();
222+ let mut bytes = snapshot.exported;
223+ bytes.extend(snapshot.head);
224+ for (key, value) in snapshot.rows {
225+ bytes.extend(key.as_bytes());
226+ bytes.extend(value);
227+ }
228+ #[cfg(feature = "sqlite")]
229+ bytes.extend(std::fs::read(config.mentions_location()).unwrap());
230+ for secret in [
231+ "private/room/event",
232+ "a secret saved message",
233+ "mentions.example",
234+ ] {
235+ assert!(
236+ !bytes
237+ .windows(secret.len())
238+ .any(|window| window == secret.as_bytes())
239+ );
240+ }
241+ drop(store);
242+ config.delete().await.unwrap();
243+}
244+
245+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
246+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
247+async fn wrong_account_server_and_passphrase_cannot_open_the_store() {
248+ let config = config();
249+ let store = open(&config).await;
250+ drop(store);
251+ let other = Account {
252+ user_id: user_id!("@bob:mentions.example").to_owned(),
253+ ..account()
254+ };
255+ assert!(
256+ MentionsStore::open(&config, other, MAX_BYTES)
257+ .await
258+ .is_err()
259+ );
260+ let other_server = Account {
261+ homeserver: "https://different.example/".to_owned(),
262+ ..account()
263+ };
264+ assert!(
265+ MentionsStore::open(&config, other_server, MAX_BYTES)
266+ .await
267+ .is_err()
268+ );
269+ #[cfg(feature = "sqlite")]
270+ let wrong_passphrase =
271+ StoreConfig::sqlite(config.mentions_location().parent().unwrap(), "incorrect");
272+ #[cfg(feature = "indexeddb")]
273+ let wrong_passphrase = StoreConfig::indexeddb(config.indexeddb_name(), "incorrect");
274+ assert!(
275+ MentionsStore::open(&wrong_passphrase, account(), MAX_BYTES)
276+ .await
277+ .is_err()
278+ );
279+ drop(open(&config).await);
280+ config.delete().await.unwrap();
281+}
282+
283+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
284+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
285+async fn ciphertext_corruption_and_record_swaps_are_rejected() {
286+ let config = config();
287+ let mut store = open(&config).await;
288+ store
289+ .commit(
290+ 0,
291+ vec![set("one", json!("first")), set("two", json!("other"))],
292+ )
293+ .await
294+ .unwrap();
295+ let one = storage_key(&store.cipher, "one");
296+ let two = storage_key(&store.cipher, "two");
297+ let mut swapped = store.snapshot.rows.clone();
298+ let first = swapped[&one].clone();
299+ swapped.insert(one.clone(), swapped[&two].clone());
300+ swapped.insert(two, first);
301+ let swapped = swapped
302+ .into_iter()
303+ .map(|(key, value)| (key, Some(value)))
304+ .collect();
305+ store
306+ .backend
307+ .commit(
308+ Some(store.snapshot.head.clone()),
309+ store.snapshot.exported.clone(),
310+ store.snapshot.head.clone(),
311+ swapped,
312+ )
313+ .await
314+ .unwrap();
315+ assert!(
316+ MentionsStore::open(&config, account(), MAX_BYTES)
317+ .await
318+ .is_err()
319+ );
320+ let mut corrupt = store.snapshot.rows[&one].clone();
321+ let middle = corrupt.len() / 2;
322+ corrupt[middle] ^= 1;
323+ let mut restored = store.snapshot.rows.clone();
324+ restored.insert(one, corrupt);
325+ let restored = restored
326+ .into_iter()
327+ .map(|(key, value)| (key, Some(value)))
328+ .collect();
329+ store
330+ .backend
331+ .commit(
332+ Some(store.snapshot.head.clone()),
333+ store.snapshot.exported.clone(),
334+ store.snapshot.head.clone(),
335+ restored,
336+ )
337+ .await
338+ .unwrap();
339+ assert!(
340+ MentionsStore::open(&config, account(), MAX_BYTES)
341+ .await
342+ .is_err()
343+ );
344+ drop(store);
345+ config.delete().await.unwrap();
346+}
347+
348+#[cfg_attr(not(target_family = "wasm"), tokio::test)]
349+#[cfg_attr(target_family = "wasm", wasm_bindgen_test::wasm_bindgen_test)]
350+async fn unsupported_schema_fails_without_resetting_data() {
351+ let config = config();
352+ let mut store = open(&config).await;
353+ store
354+ .commit(0, vec![set("checkpoint", json!("retained"))])
355+ .await
356+ .unwrap();
357+ let header = store
358+ .cipher
359+ .encrypt_value(&Head {
360+ schema: 2,
361+ account: account(),
362+ revision: 1,
363+ records: store.snapshot.rows.len(),
364+ bytes: row_bytes(&store.snapshot.rows).unwrap(),
365+ })
366+ .unwrap();
367+ store
368+ .backend
369+ .commit(
370+ Some(store.snapshot.head.clone()),
371+ store.snapshot.exported.clone(),
372+ header.clone(),
373+ Default::default(),
374+ )
375+ .await
376+ .unwrap();
377+ assert!(
378+ MentionsStore::open(&config, account(), MAX_BYTES)
379+ .await
380+ .is_err()
381+ );
382+ assert_eq!(store.backend.load().await.unwrap().unwrap().head, header);
383+ drop(store);
384+ config.delete().await.unwrap();
385+}
386+
387+#[cfg(feature = "sqlite")]
388+#[tokio::test]
389+async fn sql_failure_rolls_back_records_and_header_together() {
390+ let config = config();
391+ let mut store = open(&config).await;
392+ let path = config.mentions_location();
393+ tokio::task::spawn_blocking(move || {
394+ rusqlite::Connection::open(path).unwrap().execute_batch(
395+ "CREATE TRIGGER reject_record BEFORE INSERT ON records WHEN NEW.key NOT LIKE '!%' BEGIN SELECT RAISE(ABORT, 'injected write failure'); END;"
396+ ).unwrap();
397+ }).await.unwrap();
398+ assert!(
399+ store
400+ .commit(
401+ 0,
402+ vec![
403+ set("entry", json!("pending")),
404+ set("checkpoint", json!("new"))
405+ ]
406+ )
407+ .await
408+ .is_err()
409+ );
410+ assert_eq!(store.revision(), 0);
411+ drop(store);
412+ let restored = open(&config).await;
413+ assert_eq!(restored.revision(), 0);
414+ assert!(restored.read::<Value>("entry").unwrap().is_none());
415+ assert!(restored.read::<Value>("checkpoint").unwrap().is_none());
416+ drop(restored);
417+ config.delete().await.unwrap();
418+}
Added file: src/mentions_model.rs+127 additions
@@ -0,0 +1,127 @@
1+use ruma::{OwnedEventId, OwnedRoomId, OwnedUserId};
2+#[cfg(feature = "serde")]
3+use serde::{Deserialize, Serialize};
4+
5+use crate::{EventContext, TimelineItem};
6+
7+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
8+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
9+pub struct MentionKey {
10+ pub room_id: OwnedRoomId,
11+ pub event_id: OwnedEventId,
12+}
13+
14+#[derive(Clone, Debug, Eq, PartialEq)]
15+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
16+pub enum MentionReason {
17+ Direct,
18+ Reply,
19+ Room,
20+}
21+
22+#[derive(Clone, Debug, Eq, PartialEq)]
23+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
24+pub enum MentionCaptureState {
25+ Pending,
26+ Partial,
27+ Ready,
28+ Unavailable,
29+ Failed { message: String, retry_at_ms: u64 },
30+}
31+
32+#[derive(Clone, Debug, Eq, PartialEq)]
33+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
34+pub struct MentionEntry {
35+ pub key: MentionKey,
36+ pub timestamp_ms: u64,
37+ pub discovered_at_ms: u64,
38+ pub discovery_revision: u64,
39+ pub revision: u64,
40+ pub event: TimelineItem,
41+ pub reasons: Vec<MentionReason>,
42+ pub read: bool,
43+ pub capture: MentionCaptureState,
44+ pub room_name: Option<String>,
45+ pub notification_suppressed: bool,
46+}
47+
48+#[derive(Clone, Debug, Eq, PartialEq)]
49+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
50+pub struct MentionCoverage {
51+ pub earliest_timestamp_ms: Option<u64>,
52+ pub historical_supported: Option<bool>,
53+ pub recovering: bool,
54+ pub incomplete_history: bool,
55+ pub gaps: Vec<MentionGap>,
56+ pub truncated: bool,
57+ pub unresolved_candidates: usize,
58+}
59+
60+impl Default for MentionCoverage {
61+ fn default() -> Self {
62+ Self {
63+ earliest_timestamp_ms: None,
64+ historical_supported: None,
65+ recovering: false,
66+ incomplete_history: true,
67+ gaps: Vec::new(),
68+ truncated: false,
69+ unresolved_candidates: 0,
70+ }
71+ }
72+}
73+
74+#[derive(Clone, Debug, Eq, PartialEq)]
75+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
76+pub struct MentionGap {
77+ pub room_id: Option<OwnedRoomId>,
78+ pub reason: String,
79+ pub token: Option<String>,
80+}
81+
82+#[derive(Clone, Debug, Eq, PartialEq)]
83+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
84+pub struct MentionsBoundary {
85+ pub account: OwnedUserId,
86+ pub revision: u64,
87+}
88+
89+#[derive(Clone, Debug, Eq, PartialEq)]
90+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
91+pub struct MentionsPage {
92+ pub entries: Vec<MentionEntry>,
93+ pub next_cursor: Option<String>,
94+ pub end_reached: bool,
95+ pub coverage: MentionCoverage,
96+ pub unread_count: usize,
97+ pub revision: u64,
98+ pub boundary: MentionsBoundary,
99+}
100+
101+#[derive(Clone, Debug, Eq, PartialEq)]
102+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
103+pub struct SavedMentionContext {
104+ pub key: MentionKey,
105+ pub state: MentionCaptureState,
106+ pub context: Option<EventContext>,
107+ pub revision: u64,
108+}
109+
110+#[derive(Clone, Debug, Eq, PartialEq)]
111+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
112+pub struct MentionsUpdate {
113+ pub snapshot: MentionsPage,
114+ pub upserts: Vec<MentionEntry>,
115+ pub removed: Vec<MentionKey>,
116+}
117+
118+#[derive(Clone, Debug, Eq, PartialEq)]
119+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
120+pub enum MentionsCommand {
121+ Page { cursor: Option<String>, limit: u16 },
122+ Context(MentionKey),
123+ RetryContext(MentionKey),
124+ SetRead { keys: Vec<MentionKey>, read: bool },
125+ ReadThrough(MentionsBoundary),
126+ Recover,
127+}
Modified file: src/mirror.rs+24 additions
@@ -45,6 +45,7 @@ pub struct State {
4545 pub encryption: Option<EncryptionState>,
4646 pub verification: Option<VerificationState>,
4747 pub verification_target: Option<VerificationTarget>,
48+ pub mentions: Option<crate::MentionsPage>,
4849 pub rooms: Vector<RoomSummary>,
4950 pub image_packs: ImagePacks,
5051 pub timelines: imbl::HashMap<TimelineId, TimelineState>,
@@ -121,6 +122,7 @@ pub enum StateChange {
121122 Encryption(EncryptionState),
122123 Verification(VerificationState),
123124 VerificationTarget(VerificationTarget),
125+ Mentions(crate::MentionsUpdate),
124126 /// room modes arrive with the `Rooms` that follows
125127 NotificationSettings,
126128 Rooms(Vector<RoomSummary>),
@@ -200,6 +202,7 @@ impl Mirror {
200202 encryption: handle.encryption_state(),
201203 verification: None,
202204 verification_target: None,
205+ mentions: None,
203206 rooms: Vector::new(),
204207 image_packs: ImagePacks::default(),
205208 timelines: imbl::HashMap::new(),
@@ -262,6 +265,7 @@ impl Mirror {
262265 Some(StateChange::SessionSaved)
263266 }),
264267 Event::SessionAccessRejected { soft_logout } => self.publish(|state| {
268+ state.mentions = None;
265269 state.session = match soft_logout {
266270 Some(device_id) => SessionStatus::SoftLoggedOut { device_id },
267271 None => SessionStatus::AccessRevoked,
@@ -269,6 +273,7 @@ impl Mirror {
269273 Some(StateChange::Session(state.session.clone()))
270274 }),
271275 Event::LoggedOut => self.publish(|state| {
276+ state.mentions = None;
272277 state.session = SessionStatus::LoggedOut;
273278 Some(StateChange::Session(SessionStatus::LoggedOut))
274279 }),
@@ -290,6 +295,25 @@ impl Mirror {
290295 self.publish(|_| Some(StateChange::NotificationSettings));
291296 }
292297 Event::RoomsChanged => self.refetch_rooms(),
298+ Event::MentionsChanged(update) => self.publish(|state| {
299+ if state.session != SessionStatus::Active
300+ || state
301+ .user_id
302+ .as_ref()
303+ .is_some_and(|account| account != &update.snapshot.boundary.account)
304+ {
305+ return None;
306+ }
307+ if state
308+ .mentions
309+ .as_ref()
310+ .map_or(false, |current| update.snapshot.revision < current.revision)
311+ {
312+ return None;
313+ }
314+ state.mentions = Some(update.snapshot.clone());
315+ Some(StateChange::Mentions(update))
316+ }),
293317 Event::ImagePacksChanged => self.refetch_image_packs(),
294318 Event::EncryptionChanged(encryption) => self.publish(|state| {
295319 state.encryption = Some(encryption.clone());
Modified file: src/model.rs+4 additions
@@ -24,6 +24,7 @@ pub enum Command {
2424 Room(RoomCommand),
2525 Timeline(TimelineCommand),
2626 EventContext(EventContextCommand),
27+ Mentions(crate::MentionsCommand),
2728 Media(MediaCommand),
2829 Encryption(EncryptionCommand),
2930 Notification(NotificationCommand),
@@ -665,6 +666,8 @@ pub enum CommandOutput {
665666 },
666667 EventContext(EventContext),
667668 EventContextPage(EventContextPage),
669+ MentionsPage(crate::MentionsPage),
670+ MentionContext(crate::SavedMentionContext),
668671 }
669672
670673 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -885,6 +888,7 @@ pub enum Event {
885888 NotificationSettingsChanged,
886889 VerificationTargetChanged(VerificationTarget),
887890 RoomsChanged,
891+ MentionsChanged(crate::MentionsUpdate),
888892 /// an image pack came, went or changed somewhere. ask `image_packs` again
889893 ImagePacksChanged,
890894 /// replaces whatever you had for this timeline. every diff after it applies to it
Modified file: src/normalise/media.rs+7 additions−2 deletions
@@ -44,6 +44,7 @@ impl MediaScope {
4444 }
4545
4646 #[derive(Clone)]
47+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
4748 pub(crate) struct MediaRegistration {
4849 pub(crate) id: MediaId,
4950 pub(crate) kind: MediaKind,
@@ -52,6 +53,7 @@ pub(crate) struct MediaRegistration {
5253 }
5354
5455 #[derive(Clone)]
56+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
5557 pub(crate) struct MediaSourceInfo {
5658 pub(crate) source: MediaSource,
5759 pub(crate) mimetype: Option<String>,
@@ -208,7 +210,7 @@ pub(super) fn media_id(identity: &str, suffix: &str) -> MediaId {
208210 MediaId::new(format!("{identity}#{suffix}"))
209211 }
210212
211-fn media_sticker_registration(
213+pub(super) fn media_sticker_registration(
212214 identity: &str,
213215 content: &StickerEventContent,
214216 ) -> Option<MediaRegistration> {
@@ -227,7 +229,10 @@ fn media_sticker_registration(
227229 })
228230 }
229231
230-fn media_registration(identity: &str, message: &MessageType) -> Option<MediaRegistration> {
232+pub(super) fn media_registration(
233+ identity: &str,
234+ message: &MessageType,
235+) -> Option<MediaRegistration> {
231236 match message {
232237 MessageType::Image(content) => media_registration_with_info(
233238 identity,
Modified file: src/normalise/mod.rs+2 additions
@@ -2,6 +2,7 @@ mod image_pack;
22 mod link_preview;
33 mod media;
44 mod member;
5+mod observed;
56 mod room;
67 mod timeline;
78
@@ -11,6 +12,7 @@ pub(crate) use image_pack::{
1112 pub(crate) use link_preview::{ROOM_PREVIEW_URLS, server_preview, url_previews_allowed};
1213 pub(crate) use media::{MediaRegistration, MediaScope, MediaSourceInfo};
1314 pub(crate) use member::member_summary;
15+pub(crate) use observed::normalise_observed;
1416 pub(crate) use room::notification_mode;
1517 pub(crate) use room::{room_summaries, space_child, stored_space_order};
1618 pub(crate) use timeline::{message_kind, normalise_timeline, normalise_timeline_diffs};
Added file: src/normalise/observed.rs+270 additions
@@ -0,0 +1,270 @@
1+use ruma::{
2+ OwnedEventId, UserId,
3+ events::{
4+ poll::{start::PollStartEventContent, unstable_start::UnstablePollStartEventContent},
5+ room::message::{
6+ MessageType, Relation, RoomMessageEventContent, RoomMessageEventContentWithoutRelation,
7+ },
8+ sticker::StickerEventContent,
9+ },
10+};
11+use serde_json::Value;
12+
13+use super::{
14+ link_preview::sender_previews,
15+ media::{
16+ MediaRegistration, media_descriptor_for_sticker, media_registration,
17+ media_sticker_registration,
18+ },
19+ timeline::formatted_body,
20+};
21+use crate::{
22+ Message, MessageMentions, ReplyDetails, ReplyReference, SendState, Sender, TimelineContent,
23+ TimelineEvent, TimelineItem, TimelineItemId, UtdCause,
24+};
25+
26+pub(crate) fn normalise_observed(
27+ value: &Value,
28+ current_user_id: Option<&UserId>,
29+ identity: &str,
30+) -> Option<(TimelineItem, Vec<MediaRegistration>)> {
31+ let event_id: OwnedEventId = serde_json::from_value(value.get("event_id")?.clone()).ok()?;
32+ let sender_id: ruma::OwnedUserId = serde_json::from_value(value.get("sender")?.clone()).ok()?;
33+ let timestamp_ms = value.get("origin_server_ts")?.as_u64()?;
34+ let event_type = value.get("type")?.as_str()?.to_owned();
35+ let own = current_user_id.is_some_and(|user_id| user_id.as_str() == sender_id.as_str());
36+ let redacted = value
37+ .get("unsigned")
38+ .and_then(|unsigned| unsigned.get("redacted_because"))
39+ .is_some();
40+ let mut media = Vec::new();
41+ let (content, reply_to, thread_root) = if redacted {
42+ (TimelineContent::Redacted, None, None)
43+ } else if event_type == "m.room.message" {
44+ match value
45+ .get("content")
46+ .cloned()
47+ .and_then(|content| serde_json::from_value::<RoomMessageEventContent>(content).ok())
48+ {
49+ Some(mut message) => {
50+ let (reply_to, thread_root) = relation_refs(message.relates_to.as_ref());
51+ let relation_replacement = match &message.relates_to {
52+ Some(Relation::Replacement(replacement)) => {
53+ Some(replacement.new_content.clone())
54+ }
55+ _ => None,
56+ };
57+ let bundled_replacement = value
58+ .get("unsigned")
59+ .and_then(|unsigned| unsigned.get("m.relations"))
60+ .and_then(|relations| relations.get("m.replace"))
61+ .and_then(|edit| {
62+ let edit_sender = edit.get("sender")?.as_str()?;
63+ if edit_sender != sender_id.as_str() {
64+ return None;
65+ }
66+ edit.get("content")?.get("m.new_content").cloned()
67+ })
68+ .and_then(|content| {
69+ serde_json::from_value::<RoomMessageEventContentWithoutRelation>(content)
70+ .ok()
71+ });
72+ let edited = relation_replacement.is_some()
73+ || bundled_replacement.is_some()
74+ || value
75+ .get("unsigned")
76+ .and_then(|unsigned| unsigned.get("macltok_edited"))
77+ .and_then(Value::as_bool)
78+ .unwrap_or(false);
79+ if let Some(replacement) = relation_replacement.or(bundled_replacement) {
80+ message.apply_replacement(replacement);
81+ }
82+ let message_type = &message.msgtype;
83+ media.extend(media_registration(identity, message_type));
84+ let previews = sender_previews(Some(identity), message_type);
85+ media.extend(
86+ previews
87+ .iter()
88+ .filter_map(|(_, registration)| registration.clone()),
89+ );
90+ (
91+ TimelineContent::Message(Message {
92+ kind: super::message_kind(message_type),
93+ body: message.body().to_owned(),
94+ geo_uri: match message_type {
95+ MessageType::Location(location) => Some(location.geo_uri.clone()),
96+ _ => None,
97+ },
98+ formatted_body: formatted_body(message_type),
99+ mentions: message.mentions.as_ref().map(|mentions| MessageMentions {
100+ user_ids: mentions.user_ids.iter().cloned().collect(),
101+ room: mentions.room,
102+ }),
103+ edited,
104+ media: super::media::media_from_message(identity, message_type),
105+ link_previews: previews.into_iter().map(|(preview, _)| preview).collect(),
106+ }),
107+ reply_to,
108+ thread_root,
109+ )
110+ }
111+ None => (TimelineContent::Other { event_type }, None, None),
112+ }
113+ } else if event_type == "m.room.encrypted" {
114+ (
115+ TimelineContent::UnableToDecrypt(UtdCause::Unknown),
116+ None,
117+ None,
118+ )
119+ } else if event_type == "m.sticker" {
120+ match value
121+ .get("content")
122+ .cloned()
123+ .and_then(|content| serde_json::from_value::<StickerEventContent>(content).ok())
124+ {
125+ Some(sticker) => {
126+ let (reply_to, thread_root) = relation_refs(sticker.relates_to.as_ref());
127+ media.extend(media_sticker_registration(identity, &sticker));
128+ (
129+ TimelineContent::Sticker {
130+ body: sticker.body.clone(),
131+ media: media_descriptor_for_sticker(identity, &sticker),
132+ },
133+ reply_to,
134+ thread_root,
135+ )
136+ }
137+ None => (TimelineContent::Other { event_type }, None, None),
138+ }
139+ } else if event_type == "m.poll.start" || event_type == "org.matrix.msc3381.poll.start" {
140+ let poll_content = value.get("content");
141+ let question = poll_content.and_then(|content| poll_question(content, &event_type));
142+ match question {
143+ Some(question) => {
144+ let (reply_to, thread_root) =
145+ poll_content.map(raw_relation_refs).unwrap_or_default();
146+ (TimelineContent::Poll { question }, reply_to, thread_root)
147+ }
148+ None => (TimelineContent::Other { event_type }, None, None),
149+ }
150+ } else if let Some(state_key) = value.get("state_key").and_then(Value::as_str) {
151+ (
152+ TimelineContent::OtherState {
153+ event_type,
154+ state_key: state_key.to_owned(),
155+ },
156+ None,
157+ None,
158+ )
159+ } else {
160+ (TimelineContent::Other { event_type }, None, None)
161+ };
162+ let event = TimelineEvent {
163+ item_id: TimelineItemId::new(identity.to_owned()),
164+ event_id: Some(event_id),
165+ transaction_id: None,
166+ sender: Sender {
167+ user_id: sender_id,
168+ display_name: None,
169+ avatar: None,
170+ name_ambiguous: false,
171+ },
172+ timestamp_ms,
173+ own,
174+ send_state: SendState::Sent,
175+ content,
176+ reply_to: reply_to.map(|event_id| ReplyReference {
177+ event_id,
178+ details: ReplyDetails::Pending,
179+ }),
180+ thread_root,
181+ thread_summary: None,
182+ reactions: Vec::new(),
183+ read_by: Vec::new(),
184+ editable: false,
185+ };
186+ Some((TimelineItem::Event(Box::new(event)), media))
187+}
188+
189+fn relation_refs<C>(
190+ relation: Option<&Relation<C>>,
191+) -> (Option<OwnedEventId>, Option<OwnedEventId>) {
192+ match relation {
193+ Some(Relation::Reply(reply)) => (Some(reply.in_reply_to.event_id.clone()), None),
194+ Some(Relation::Thread(thread)) => (
195+ thread
196+ .in_reply_to
197+ .as_ref()
198+ .map(|reply| reply.event_id.clone()),
199+ Some(thread.event_id.clone()),
200+ ),
201+ _ => (None, None),
202+ }
203+}
204+
205+fn raw_relation_refs(content: &Value) -> (Option<OwnedEventId>, Option<OwnedEventId>) {
206+ let Some(relation) = content.get("m.relates_to") else {
207+ return (None, None);
208+ };
209+ let reply_to = relation
210+ .get("m.in_reply_to")
211+ .and_then(|reply| reply.get("event_id"))
212+ .and_then(Value::as_str)
213+ .and_then(|event_id| event_id.parse().ok());
214+ let thread_root = (relation.get("rel_type").and_then(Value::as_str) == Some("m.thread"))
215+ .then(|| relation.get("event_id").and_then(Value::as_str))
216+ .flatten()
217+ .and_then(|event_id| event_id.parse().ok());
218+ (reply_to, thread_root)
219+}
220+
221+fn poll_question(content: &Value, event_type: &str) -> Option<String> {
222+ let typed_question = match event_type {
223+ "m.poll.start" => serde_json::from_value::<PollStartEventContent>(content.clone())
224+ .ok()
225+ .and_then(|poll| poll.poll.question.text.find_plain().map(str::to_owned)),
226+ "org.matrix.msc3381.poll.start" => {
227+ serde_json::from_value::<UnstablePollStartEventContent>(content.clone())
228+ .ok()
229+ .map(|poll| poll.poll_start().question.text.clone())
230+ }
231+ _ => None,
232+ };
233+ typed_question
234+ .or_else(|| {
235+ content
236+ .get("m.poll")
237+ .and_then(|poll| poll.get("question"))
238+ .and_then(|question| question.get("m.text"))
239+ .and_then(generic_text)
240+ })
241+ .or_else(|| {
242+ content
243+ .get("org.matrix.msc3381.poll.start")
244+ .and_then(|poll| poll.get("question"))
245+ .and_then(|question| question.get("org.matrix.msc1767.text"))
246+ .and_then(generic_text)
247+ })
248+ .or_else(|| {
249+ ["org.matrix.msc1767.text", "m.text", "question", "body"]
250+ .iter()
251+ .find_map(|key| content.get(*key).and_then(generic_text))
252+ })
253+}
254+
255+fn generic_text(value: &Value) -> Option<String> {
256+ match value {
257+ Value::String(text) => Some(text.clone()),
258+ Value::Array(representations) => representations
259+ .iter()
260+ .find(|representation| {
261+ representation.get("mimetype").and_then(Value::as_str) == Some("text/plain")
262+ })
263+ .or_else(|| representations.first())
264+ .and_then(|representation| representation.get("body"))
265+ .and_then(Value::as_str)
266+ .map(str::to_owned),
267+ Value::Object(_) => value.get("body").and_then(Value::as_str).map(str::to_owned),
268+ _ => None,
269+ }
270+}
Modified file: src/normalise/timeline.rs+1 additions−1 deletions
@@ -436,7 +436,7 @@ fn reactions(event: &EventTimelineItem, current_user_id: Option<&UserId>) -> Vec
436436 .collect()
437437 }
438438
439-fn formatted_body(message: &MessageType) -> Option<String> {
439+pub(super) fn formatted_body(message: &MessageType) -> Option<String> {
440440 let formatted = match message {
441441 MessageType::Text(content) => content.formatted.as_ref(),
442442 MessageType::Emote(content) => content.formatted.as_ref(),
Modified file: src/session_store.rs+3 additions−1 deletions
@@ -473,11 +473,12 @@ pub(crate) fn delete_sqlite_store(path: &Path) -> std::io::Result<()> {
473473 use std::{fs, io::ErrorKind};
474474
475475 // matrix-sdk-sqlite 0.18's database names, which it keeps private
476- const DATABASES: [&str; 4] = [
476+ const DATABASES: [&str; 5] = [
477477 "matrix-sdk-state.sqlite3",
478478 "matrix-sdk-crypto.sqlite3",
479479 "matrix-sdk-event-cache.sqlite3",
480480 "matrix-sdk-media.sqlite3",
481+ "macltok-mentions.sqlite3",
481482 ];
482483 for database in DATABASES {
483484 for suffix in ["", "-wal", "-shm", "-journal"] {
@@ -507,6 +508,7 @@ pub(crate) async fn delete_indexeddb_store(name: &str) -> Result<(), DeleteError
507508 "::matrix-sdk-crypto-meta",
508509 "::event_cache",
509510 "::media",
511+ "::macltok-mentions",
510512 ] {
511513 delete_database(&format!("{name}{suffix}")).await?;
512514 }
Modified file: src/transport.rs+77 additions−8 deletions
@@ -29,14 +29,15 @@ use tokio::sync::{OnceCell, mpsc, oneshot, watch};
2929 use crate::{
3030 AccountCommand, AttachmentUpload, Command, CommandOutput, ConnectionState, CreateRoomOptions,
3131 EncryptionCommand, EncryptionState, Error, ErrorKind, Event, EventContext, EventContextCommand,
32- EventContextPage, EventRef, ImagePackSource,
33- ImagePacks, InteractiveAuth, InteractiveAuthOutcome, LifecycleCommand, LinkPreview, Media,
34- MediaCommand, MediaId, MediaLocation, MediaSize, MemberFilter, MemberSummary, MessageContent,
35- MessageEditContent, NotificationCommand, OwnDevice, PackCommand, PackImageInfo, PackUsage,
36- Patch, PowerLevelChanges, Profile, PushNotification, PushNotificationEvent,
37- PushNotificationStatus, RoomCommand, RoomListFilter, RoomNotificationMode,
38- RoomNotificationSettings, RoomPowerLevels, RoomSummary, SpaceChild, TimelineCommand,
39- TimelineId, VerificationAction, VerificationTarget,
32+ EventContextPage, EventRef, ImagePackSource, ImagePacks, InteractiveAuth,
33+ InteractiveAuthOutcome, LifecycleCommand, LinkPreview, Media, MediaCommand, MediaId,
34+ MediaLocation, MediaSize, MemberFilter, MemberSummary, MentionKey, MentionsBoundary,
35+ MentionsCommand, MentionsPage, MessageContent, MessageEditContent, NotificationCommand,
36+ OwnDevice, PackCommand, PackImageInfo, PackUsage, Patch, PowerLevelChanges, Profile,
37+ PushNotification, PushNotificationEvent, PushNotificationStatus, RoomCommand, RoomListFilter,
38+ RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels, RoomSummary,
39+ SavedMentionContext, SpaceChild, TimelineCommand, TimelineId, VerificationAction,
40+ VerificationTarget,
4041 encryption::encryption_state,
4142 engine::{MediaService, StoreLock},
4243 model::{PushNotificationRoom, PushNotificationSender},
@@ -385,6 +386,74 @@ impl ClientHandle {
385386 }
386387 }
387388
389+ pub async fn mentions_page(
390+ &self,
391+ cursor: Option<String>,
392+ limit: u32,
393+ ) -> Result<MentionsPage, Error> {
394+ match self
395+ .execute(Command::Mentions(MentionsCommand::Page {
396+ cursor,
397+ limit: limit.min(50) as u16,
398+ }))
399+ .await?
400+ {
401+ CommandOutput::MentionsPage(page) => Ok(page),
402+ output => Err(unexpected("mentions page", &output)),
403+ }
404+ }
405+
406+ pub async fn mention_context(
407+ &self,
408+ room_id: &RoomId,
409+ event_id: &EventId,
410+ ) -> Result<SavedMentionContext, Error> {
411+ self.mention_context_command(MentionsCommand::Context(MentionKey {
412+ room_id: room_id.to_owned(),
413+ event_id: event_id.to_owned(),
414+ }))
415+ .await
416+ }
417+
418+ pub async fn retry_mention_context(
419+ &self,
420+ room_id: &RoomId,
421+ event_id: &EventId,
422+ ) -> Result<SavedMentionContext, Error> {
423+ self.mention_context_command(MentionsCommand::RetryContext(MentionKey {
424+ room_id: room_id.to_owned(),
425+ event_id: event_id.to_owned(),
426+ }))
427+ .await
428+ }
429+
430+ async fn mention_context_command(
431+ &self,
432+ command: MentionsCommand,
433+ ) -> Result<SavedMentionContext, Error> {
434+ match self.execute(Command::Mentions(command)).await? {
435+ CommandOutput::MentionContext(context) => Ok(context),
436+ output => Err(unexpected("mention context", &output)),
437+ }
438+ }
439+
440+ pub async fn set_mentions_read(&self, keys: Vec<MentionKey>, read: bool) -> Result<(), Error> {
441+ self.run(Command::Mentions(MentionsCommand::SetRead { keys, read }))
442+ .await
443+ }
444+
445+ pub async fn mark_mentions_read_through(
446+ &self,
447+ boundary: MentionsBoundary,
448+ ) -> Result<(), Error> {
449+ self.run(Command::Mentions(MentionsCommand::ReadThrough(boundary)))
450+ .await
451+ }
452+
453+ pub async fn recover_mentions(&self) -> Result<(), Error> {
454+ self.run(Command::Mentions(MentionsCommand::Recover)).await
455+ }
456+
388457 pub async fn send_message(
389458 &self,
390459 timeline_id: TimelineId,
Added file: tests/mentions_checkpoint.rs+155 additions
@@ -0,0 +1,155 @@
1+use std::{sync::Arc, time::Duration};
2+
3+use matrix_sdk::{
4+ Client, Room,
5+ ruma::{api::client::sync::sync_events::v5, events::AnySyncTimelineEvent, room_id, serde::Raw},
6+ sliding_sync::{SlidingSyncList, SlidingSyncMode},
7+ test_utils::mocks::MatrixMockServer,
8+};
9+use serde_json::{Value, json};
10+use tokio::sync::{Notify, Semaphore};
11+
12+async fn saved_position(client: &Client) -> Option<String> {
13+ let machine = client.olm_machine_for_testing().await;
14+ let key = format!(
15+ "sliding_sync_store::mentions-spike::{}::instance",
16+ client.user_id().unwrap()
17+ );
18+ let saved = machine
19+ .as_ref()
20+ .unwrap()
21+ .store()
22+ .get_custom_value(&key)
23+ .await
24+ .unwrap()?;
25+ serde_json::from_slice::<Value>(&saved).unwrap()["pos"]
26+ .as_str()
27+ .map(ToOwned::to_owned)
28+}
29+
30+async fn server_with_mention() -> (MatrixMockServer, Client) {
31+ let server = MatrixMockServer::new().await;
32+ let client = server.client_builder().build().await;
33+ let mut response = v5::Response::new("position-1".to_owned());
34+ response.rooms.insert(
35+ room_id!("!checkpoint:localhost").to_owned(),
36+ serde_json::from_value(json!({
37+ "initial": true,
38+ "required_state": [],
39+ "timeline": [{
40+ "type": "m.room.message",
41+ "event_id": "$checkpoint-mention",
42+ "sender": "@sender:localhost",
43+ "origin_server_ts": 1,
44+ "content": {
45+ "msgtype": "m.text",
46+ "body": "checkpoint candidate",
47+ "m.mentions": { "user_ids": [client.user_id().unwrap()] }
48+ }
49+ }]
50+ }))
51+ .unwrap(),
52+ );
53+ server.mock_sliding_sync().ok(response).mount().await;
54+ (server, client)
55+}
56+
57+async fn new_sync(client: &Client) -> matrix_sdk::sliding_sync::SlidingSync {
58+ client
59+ .sliding_sync("mentions-spike")
60+ .unwrap()
61+ .share_pos()
62+ .add_list(
63+ SlidingSyncList::builder("mentions")
64+ .sync_mode(SlidingSyncMode::new_selective().add_range(0..=0)),
65+ )
66+ .build()
67+ .await
68+ .unwrap()
69+}
70+
71+#[tokio::test]
72+async fn awaited_event_handler_fences_position_even_after_caller_cancellation() {
73+ let (server, client) = server_with_mention().await;
74+
75+ let entered = Arc::new(Notify::new());
76+ let committed = Arc::new(Semaphore::new(0));
77+ let handler = client.add_event_handler({
78+ let entered = entered.clone();
79+ let committed = committed.clone();
80+ move |event: Raw<AnySyncTimelineEvent>, room: Room| {
81+ let entered = entered.clone();
82+ let committed = committed.clone();
83+ async move {
84+ assert_eq!(room.room_id().as_str(), "!checkpoint:localhost");
85+ assert_eq!(
86+ event.get_field::<String>("event_id").unwrap().as_deref(),
87+ Some("$checkpoint-mention")
88+ );
89+ entered.notify_one();
90+ committed.acquire().await.unwrap().forget();
91+ }
92+ }
93+ });
94+ let sync = new_sync(&client).await;
95+ let processing = tokio::spawn(async move { sync.sync_once().await });
96+ tokio::time::timeout(Duration::from_secs(5), entered.notified())
97+ .await
98+ .unwrap();
99+ assert!(!processing.is_finished());
100+ assert!(saved_position(&client).await.is_none());
101+ processing.abort();
102+ assert!(processing.await.unwrap_err().is_cancelled());
103+ assert!(saved_position(&client).await.is_none());
104+
105+ committed.add_permits(1);
106+ tokio::time::timeout(Duration::from_secs(5), async {
107+ while saved_position(&client).await.as_deref() != Some("position-1") {
108+ tokio::task::yield_now().await;
109+ }
110+ })
111+ .await
112+ .unwrap();
113+ client.remove_event_handler(handler);
114+
115+ let restored = new_sync(&client).await;
116+ tokio::time::timeout(Duration::from_secs(5), restored.sync_once())
117+ .await
118+ .unwrap()
119+ .unwrap();
120+ let requests: Vec<_> = server
121+ .received_requests()
122+ .await
123+ .unwrap()
124+ .into_iter()
125+ .filter_map(|request| {
126+ let body = serde_json::from_slice::<Value>(&request.body).ok()?;
127+ let pos = request
128+ .url
129+ .query_pairs()
130+ .find(|(key, _)| key == "pos")
131+ .map(|(_, value)| value.into_owned());
132+ Some((body, pos))
133+ })
134+ .filter(|(body, _)| body["conn_id"] == "mentions-spike")
135+ .collect();
136+ assert_eq!(requests.len(), 2);
137+ assert!(requests[0].1.is_none());
138+ assert_eq!(requests[1].1.as_deref(), Some("position-1"));
139+}
140+
141+#[tokio::test]
142+async fn event_handler_error_does_not_prevent_position_persistence() {
143+ let (_server, client) = server_with_mention().await;
144+ let handler =
145+ client.add_event_handler(|_event: Raw<AnySyncTimelineEvent>, _room: Room| async {
146+ Err::<(), _>("the candidate transaction failed")
147+ });
148+ let sync = new_sync(&client).await;
149+ tokio::time::timeout(Duration::from_secs(5), sync.sync_once())
150+ .await
151+ .unwrap()
152+ .unwrap();
153+ assert_eq!(saved_position(&client).await.as_deref(), Some("position-1"));
154+ client.remove_event_handler(handler);
155+}
Added file: tests/mentions_decryption.rs+128 additions
@@ -0,0 +1,128 @@
1+use std::time::Duration;
2+
3+use futures_util::StreamExt;
4+use matrix_sdk::{
5+ deserialized_responses::TimelineEventKind, event_cache::RedecryptorReport,
6+ test_utils::mocks::MatrixMockServer,
7+};
8+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};
9+use ruma::{RoomVersionId, device_id, event_id, room_id, user_id};
10+use serde_json::json;
11+
12+#[tokio::test]
13+async fn late_key_exposes_a_mention_without_a_ui_timeline() {
14+ let server = MatrixMockServer::new().await;
15+ server.mock_crypto_endpoints_preset().await;
16+ let alice_id = user_id!("@alice:localhost");
17+ let bob_id = user_id!("@bob:localhost");
18+ let room_id = room_id!("!mentions-encrypted:localhost");
19+ let event_id = event_id!("$late-mention");
20+ let alice = server
21+ .client_builder_for_crypto_end_to_end(alice_id, device_id!("ALICEDEVICE"))
22+ .build()
23+ .await;
24+ let bob = server
25+ .client_builder_for_crypto_end_to_end(bob_id, device_id!("BOBDEVICE"))
26+ .build()
27+ .await;
28+ bob.event_cache().subscribe().unwrap();
29+ server.exchange_e2ee_identities(&alice, &bob).await;
30+
31+ let factory = EventFactory::new().room(room_id).sender(alice_id);
32+ let joined = JoinedRoomBuilder::new(room_id)
33+ .add_state_event(factory.create(alice_id, RoomVersionId::V1))
34+ .add_state_event(factory.room_encryption())
35+ .add_state_event(factory.member(alice_id))
36+ .add_state_event(factory.member(bob_id));
37+ server.sync_room(&alice, joined.clone()).await;
38+ server.sync_room(&bob, joined).await;
39+
40+ let room = alice.get_room(room_id).unwrap();
41+ let (encrypted_event, send) = server.mock_room_send().ok_with_capture(event_id, alice_id);
42+ let _send = send.mock_once().mount_as_scoped().await;
43+ let (_keys, room_key) = server.mock_capture_put_to_device(alice_id).await;
44+ server
45+ .mock_get_members()
46+ .ok(vec![
47+ factory.member(alice_id).into_raw(),
48+ factory.member(bob_id).into_raw(),
49+ ])
50+ .mock_once()
51+ .mount()
52+ .await;
53+ room.send_raw(
54+ "m.room.message",
55+ json!({
56+ "msgtype": "m.text",
57+ "body": "a private mention",
58+ "m.mentions": { "user_ids": [bob_id] }
59+ }),
60+ )
61+ .await
62+ .unwrap();
63+ let encrypted_event = encrypted_event.await.unwrap();
64+ let room_key = room_key.await;
65+
66+ let mut sync_updates = bob.subscribe_to_all_room_updates();
67+ let mut cache_updates = bob.event_cache().subscribe_to_room_generic_updates();
68+ let mut reports = bob.event_cache().subscribe_to_decryption_reports();
69+ server
70+ .sync_room(
71+ &bob,
72+ JoinedRoomBuilder::new(room_id).add_timeline_event(encrypted_event),
73+ )
74+ .await;
75+ let update = sync_updates.recv().await.unwrap();
76+ let event = &update.joined[room_id].timeline.events[0];
77+ assert!(matches!(
78+ event.kind,
79+ TimelineEventKind::UnableToDecrypt { .. }
80+ ));
81+ tokio::time::timeout(Duration::from_secs(5), async {
82+ while cache_updates.recv().await.unwrap().room_id != room_id {}
83+ })
84+ .await
85+ .unwrap();
86+
87+ server
88+ .mock_sync()
89+ .ok_and_run(&bob, |sync| {
90+ sync.add_to_device_event(room_key.deserialize_as().unwrap());
91+ })
92+ .await;
93+ tokio::time::timeout(Duration::from_secs(5), async {
94+ while let Some(report) = reports.next().await {
95+ if let Ok(RedecryptorReport::ResolvedUtds {
96+ room_id: resolved,
97+ events,
98+ }) = report
99+ {
100+ if resolved == room_id && events.contains(event_id) {
101+ return;
102+ }
103+ }
104+ }
105+ panic!("decryption reports ended before the mention was decrypted");
106+ })
107+ .await
108+ .unwrap();
109+
110+ let room = bob.get_room(room_id).unwrap();
111+ let (cache, _cache_handles) = room.event_cache().await.unwrap();
112+ let decrypted = cache.find_event(event_id).await.unwrap().unwrap();
113+ assert!(matches!(decrypted.kind, TimelineEventKind::Decrypted(_)));
114+ let content = decrypted
115+ .raw()
116+ .get_field::<serde_json::Value>("content")
117+ .unwrap()
118+ .unwrap();
119+ assert_eq!(content["body"], "a private mention");
120+ assert_eq!(content["m.mentions"]["user_ids"], json!([bob_id]));
121+ let push_actions = room
122+ .event_push_actions(decrypted.raw())
123+ .await
124+ .unwrap()
125+ .unwrap();
126+ assert!(push_actions.iter().any(|action| action.should_notify()));
127+ assert_eq!(cache.events().await.unwrap().len(), 1);
128+}
Added file: tests/mentions_discovery.rs+232 additions
@@ -0,0 +1,232 @@
1+use matrix_sdk::{
2+ Room,
3+ ruma::{
4+ UInt,
5+ api::client::push::get_notifications::v3,
6+ event_id,
7+ events::{AnySyncTimelineEvent, Mentions},
8+ owned_user_id,
9+ push::{Action, EventMatchConditionData, NewConditionalPushRule, PushCondition, Ruleset},
10+ room_id,
11+ serde::Raw,
12+ },
13+ test_utils::mocks::MatrixMockServer,
14+};
15+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};
16+use serde_json::json;
17+use std::time::Duration;
18+use wiremock::{
19+ Mock, ResponseTemplate,
20+ matchers::{method, path, query_param},
21+};
22+
23+fn explicit_mention(
24+ room_id: &matrix_sdk::ruma::RoomId,
25+ event_id: &matrix_sdk::ruma::EventId,
26+ sender: &matrix_sdk::ruma::UserId,
27+ mentioned: &matrix_sdk::ruma::UserId,
28+) -> Raw<AnySyncTimelineEvent> {
29+ EventFactory::new()
30+ .room(room_id)
31+ .text_msg("hello")
32+ .mentions(Mentions::with_user_ids([mentioned.to_owned()]))
33+ .event_id(event_id)
34+ .sender(sender)
35+ .into_raw_sync()
36+}
37+
38+fn assert_mention_actions(actions: &[Action]) {
39+ assert!(
40+ actions
41+ .iter()
42+ .any(|action| matches!(action, Action::Notify))
43+ );
44+ assert!(actions.iter().any(Action::is_highlight));
45+}
46+
47+async fn sync_room_with_mention(
48+ server: &MatrixMockServer,
49+ client: &matrix_sdk::Client,
50+ room_id: &matrix_sdk::ruma::RoomId,
51+ event_id: &matrix_sdk::ruma::EventId,
52+) {
53+ let user_id = client.user_id().unwrap();
54+ let factory = EventFactory::new().room(room_id);
55+ let sender = owned_user_id!("@alice:localhost");
56+ let event = explicit_mention(room_id, event_id, &sender, user_id);
57+ let rules = Ruleset::server_default(user_id);
58+
59+ server
60+ .mock_sync()
61+ .ok_and_run(client, |builder| {
62+ builder.add_global_account_data(factory.push_rules(rules));
63+ builder.add_joined_room(
64+ JoinedRoomBuilder::new(room_id)
65+ .add_state_event(factory.member(user_id))
66+ .add_timeline_event(event),
67+ );
68+ })
69+ .await;
70+}
71+
72+async fn actions_for(room: &Room, event: &Raw<AnySyncTimelineEvent>) -> Option<Vec<Action>> {
73+ room.event_push_actions(event).await.unwrap()
74+}
75+
76+#[tokio::test]
77+async fn joined_room_update_exposes_mention_actions_without_opening_a_ui_timeline() {
78+ let server = MatrixMockServer::new().await;
79+ let client = server.client_builder().build().await;
80+ let room_id = room_id!("!mentions:localhost");
81+ let event_id = event_id!("$mention:localhost");
82+ let mut updates = client.subscribe_to_all_room_updates();
83+
84+ sync_room_with_mention(&server, &client, room_id, event_id).await;
85+
86+ let room_updates = tokio::time::timeout(Duration::from_secs(2), updates.recv())
87+ .await
88+ .unwrap()
89+ .unwrap();
90+ let joined = room_updates
91+ .joined
92+ .get(room_id)
93+ .expect("the joined room update was absent");
94+ let event = joined
95+ .timeline
96+ .events
97+ .iter()
98+ .find(|event| event.event_id().as_deref().map(|id| id.as_str()) == Some(event_id.as_str()))
99+ .expect("the explicit mention was absent from the room update");
100+ let event_json: serde_json::Value =
101+ serde_json::from_str(event.kind.raw().json().get()).unwrap();
102+
103+ assert_eq!(
104+ event_json["content"]["m.mentions"]["user_ids"],
105+ json!([client.user_id().unwrap().as_str()])
106+ );
107+ assert_mention_actions(
108+ event
109+ .push_actions()
110+ .expect("push actions were not computed"),
111+ );
112+}
113+
114+#[tokio::test]
115+async fn event_cache_generic_update_finds_the_mention_without_opening_a_ui_timeline() {
116+ let server = MatrixMockServer::new().await;
117+ let client = server.client_builder().build().await;
118+ let room_id = room_id!("!cached-mentions:localhost");
119+ let event_id = event_id!("$cached-mention:localhost");
120+ client.event_cache().subscribe().unwrap();
121+ let mut updates = client.event_cache().subscribe_to_room_generic_updates();
122+
123+ sync_room_with_mention(&server, &client, room_id, event_id).await;
124+
125+ let update = tokio::time::timeout(Duration::from_secs(2), updates.recv())
126+ .await
127+ .unwrap()
128+ .unwrap();
129+ assert_eq!(update.room_id, room_id);
130+ let room = client.get_room(room_id).unwrap();
131+ let (event_cache, _drop_handles) = room.event_cache().await.unwrap();
132+ let event = event_cache.find_event(event_id).await.unwrap().unwrap();
133+ assert_eq!(
134+ event.event_id().as_deref().map(|id| id.as_str()),
135+ Some(event_id.as_str())
136+ );
137+ assert_mention_actions(
138+ event
139+ .push_actions()
140+ .expect("cached push actions were absent"),
141+ );
142+}
143+
144+#[tokio::test]
145+async fn ignored_sender_and_muted_room_actions_remain_sdk_push_rule_results() {
146+ let server = MatrixMockServer::new().await;
147+ let client = server.client_builder().build().await;
148+ let ignored_room_id = room_id!("!ignored-sender:localhost");
149+ let muted_room_id = room_id!("!muted-room:localhost");
150+ let ignored_sender = owned_user_id!("@ignored:localhost");
151+ let own_user = client.user_id().unwrap();
152+ let factory = EventFactory::new();
153+ let ignored_event_id = event_id!("$ignored-mention:localhost");
154+ let muted_event_id = event_id!("$muted-message:localhost");
155+ let ignored_event =
156+ explicit_mention(ignored_room_id, ignored_event_id, &ignored_sender, own_user);
157+ let sender = owned_user_id!("@alice:localhost");
158+ let muted_event = explicit_mention(muted_room_id, muted_event_id, &sender, own_user);
159+ let mut rules = Ruleset::server_default(own_user);
160+ rules.override_.shift_insert(
161+ 1,
162+ NewConditionalPushRule::new(
163+ "mute-room".to_owned(),
164+ vec![PushCondition::EventMatch(EventMatchConditionData::new(
165+ "room_id".to_owned(),
166+ muted_room_id.to_string(),
167+ ))],
168+ Vec::new(),
169+ )
170+ .into(),
171+ );
172+
173+ server
174+ .mock_sync()
175+ .ok_and_run(&client, |builder| {
176+ builder.add_global_account_data(factory.ignored_user_list([ignored_sender.clone()]));
177+ builder.add_global_account_data(factory.push_rules(rules));
178+ builder.add_joined_room(
179+ JoinedRoomBuilder::new(ignored_room_id)
180+ .add_state_event(EventFactory::new().room(ignored_room_id).member(own_user))
181+ .add_timeline_event(ignored_event.clone()),
182+ );
183+ builder.add_joined_room(
184+ JoinedRoomBuilder::new(muted_room_id)
185+ .add_state_event(EventFactory::new().room(muted_room_id).member(own_user))
186+ .add_timeline_event(muted_event.clone()),
187+ );
188+ })
189+ .await;
190+
191+ assert!(client.is_user_ignored(&ignored_sender).await);
192+ let ignored_room = client.get_room(ignored_room_id).unwrap();
193+ let ignored_actions = actions_for(&ignored_room, &ignored_event)
194+ .await
195+ .expect("the ignored sender's push actions could not be computed");
196+ assert_mention_actions(&ignored_actions);
197+
198+ let muted_room = client.get_room(muted_room_id).unwrap();
199+ let muted_actions = actions_for(&muted_room, &muted_event)
200+ .await
201+ .expect("the muted room's push actions could not be computed");
202+ assert!(muted_actions.is_empty());
203+}
204+
205+#[tokio::test]
206+async fn notifications_v3_request_round_trips_opaque_pagination() {
207+ let server = MatrixMockServer::new().await;
208+ let client = server.client_builder().build().await;
209+ let cursor = "opaque / cursor?x=1&y=2";
210+ let next_cursor = "next / cursor?x=3&y=4";
211+
212+ Mock::given(method("GET"))
213+ .and(path("/_matrix/client/v3/notifications"))
214+ .and(query_param("from", cursor))
215+ .and(query_param("limit", "8"))
216+ .and(query_param("only", "highlight"))
217+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
218+ "next_token": next_cursor,
219+ "notifications": [],
220+ })))
221+ .expect(1)
222+ .mount(server.server())
223+ .await;
224+
225+ let mut request = v3::Request::new();
226+ request.from = Some(cursor.to_owned());
227+ request.limit = Some(UInt::new(8).unwrap());
228+ request.only = Some("highlight".to_owned());
229+ let response = client.send(request).await.unwrap();
230+
231+ assert_eq!(response.next_token.as_deref(), Some(next_cursor));
232+}
Added file: tests/mentions_inbox.rs+882 additions
@@ -0,0 +1,882 @@
1+mod support;
2+
3+use std::{
4+ collections::{BTreeMap, HashSet},
5+ sync::{
6+ Arc, Mutex,
7+ atomic::{AtomicUsize, Ordering},
8+ },
9+ time::Duration,
10+};
11+
12+#[cfg(feature = "sqlite")]
13+use macltok::{EncryptedSessionStore, PasswordLogin, Restore, Session, SessionStore, StoreConfig};
14+use macltok::{Engine, EngineConfig, Event, MentionsPage};
15+use matrix_sdk::{
16+ Client,
17+ ruma::{OwnedRoomId, OwnedUserId, event_id, events::Mentions, room_id, user_id},
18+ test_utils::mocks::{MatrixMockServer, RoomContextResponseTemplate},
19+};
20+use matrix_sdk_test::{ALICE, JoinedRoomBuilder, event_factory::EventFactory};
21+use ruma::push::{EventMatchConditionData, NewConditionalPushRule, PushCondition, Ruleset};
22+use serde_json::{Value, json};
23+#[cfg(feature = "sqlite")]
24+use support::receive_until;
25+use support::shut_down;
26+use wiremock::{
27+ Mock, Request, Respond, ResponseTemplate,
28+ matchers::{method, path, path_regex, query_param},
29+};
30+
31+#[cfg(feature = "serde")]
32+use macltok::{MentionCoverage, MentionsBoundary, MentionsCommand};
33+
34+const SLIDING_SYNC_PATH: &str = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync";
35+
36+struct SlidingFeed {
37+ rooms: Arc<Mutex<BTreeMap<OwnedRoomId, Vec<Value>>>>,
38+ required_state: Arc<Mutex<BTreeMap<OwnedRoomId, Vec<Value>>>>,
39+ requests: Arc<AtomicUsize>,
40+}
41+
42+impl SlidingFeed {
43+ fn new(room_id: OwnedRoomId, events: Vec<Value>) -> Self {
44+ Self {
45+ rooms: Arc::new(Mutex::new(BTreeMap::from([(room_id, events)]))),
46+ required_state: Arc::new(Mutex::new(BTreeMap::new())),
47+ requests: Arc::new(AtomicUsize::new(0)),
48+ }
49+ }
50+
51+ fn replace(&self, room_id: &ruma::RoomId, events: Vec<Value>) {
52+ self.rooms
53+ .lock()
54+ .unwrap()
55+ .insert(room_id.to_owned(), events);
56+ }
57+
58+ fn add_room(&self, room_id: &ruma::RoomId, events: Vec<Value>) {
59+ self.rooms
60+ .lock()
61+ .unwrap()
62+ .insert(room_id.to_owned(), events);
63+ }
64+
65+ fn set_required_state(&self, room_id: &ruma::RoomId, events: Vec<Value>) {
66+ self.required_state
67+ .lock()
68+ .unwrap()
69+ .insert(room_id.to_owned(), events);
70+ }
71+
72+ async fn mount(&self, server: &MatrixMockServer) {
73+ Mock::given(method("POST"))
74+ .and(path(SLIDING_SYNC_PATH))
75+ .respond_with(self.clone())
76+ .mount(server.server())
77+ .await;
78+ }
79+}
80+
81+impl Clone for SlidingFeed {
82+ fn clone(&self) -> Self {
83+ Self {
84+ rooms: self.rooms.clone(),
85+ required_state: self.required_state.clone(),
86+ requests: self.requests.clone(),
87+ }
88+ }
89+}
90+
91+impl Respond for SlidingFeed {
92+ fn respond(&self, _request: &Request) -> ResponseTemplate {
93+ let request = self.requests.fetch_add(1, Ordering::AcqRel);
94+ let required_state = self.required_state.lock().unwrap().clone();
95+ let rooms: serde_json::Map<String, Value> = self
96+ .rooms
97+ .lock()
98+ .unwrap()
99+ .iter()
100+ .map(|(room_id, events)| {
101+ (
102+ room_id.to_string(),
103+ json!({
104+ "initial": request == 0,
105+ "required_state": required_state
106+ .get(room_id)
107+ .cloned()
108+ .unwrap_or_default(),
109+ "timeline": events,
110+ }),
111+ )
112+ })
113+ .collect();
114+ ResponseTemplate::new(200)
115+ .set_body_json(json!({
116+ "txn_id": null,
117+ "pos": format!("mentions-{request}"),
118+ "lists": {},
119+ "rooms": rooms,
120+ "extensions": {},
121+ }))
122+ .set_delay(Duration::from_millis(20))
123+ }
124+}
125+
126+async fn joined_client(server: &MatrixMockServer, room_id: &ruma::RoomId) -> (Client, OwnedUserId) {
127+ let client = server.client_builder().build().await;
128+ let own_user = client.user_id().unwrap().to_owned();
129+ let room_id = room_id.to_owned();
130+ server
131+ .mock_sync()
132+ .ok_and_run(&client, {
133+ let own_user = own_user.clone();
134+ move |builder| {
135+ let factory = EventFactory::new().room(&room_id);
136+ builder.add_global_account_data(
137+ factory.push_rules(Ruleset::server_default(&own_user)),
138+ );
139+ builder.add_joined_room(
140+ JoinedRoomBuilder::new(&room_id).add_state_event(factory.member(&own_user)),
141+ );
142+ }
143+ })
144+ .await;
145+ (client, own_user)
146+}
147+
148+fn message(
149+ event_id: &str,
150+ sender: &str,
151+ timestamp: u64,
152+ body: &str,
153+ mentioned: Option<&str>,
154+) -> Value {
155+ let mut content = json!({ "msgtype": "m.text", "body": body });
156+ if let Some(mentioned) = mentioned {
157+ content["m.mentions"] = json!({ "user_ids": [mentioned] });
158+ }
159+ json!({
160+ "type": "m.room.message",
161+ "event_id": event_id,
162+ "sender": sender,
163+ "origin_server_ts": timestamp,
164+ "content": content,
165+ })
166+}
167+
168+fn room_mention(event_id: &str, sender: &str, timestamp: u64) -> Value {
169+ let mut event = message(event_id, sender, timestamp, "room mention", None);
170+ event["content"]["m.mentions"] = json!({ "room": true });
171+ event
172+}
173+
174+async fn mentions_page(
175+ events: &mut macltok::EventStream,
176+ room_id: &ruma::RoomId,
177+ predicate: impl Fn(&MentionsPage) -> bool,
178+) -> MentionsPage {
179+ mentions_page_with_diagnostics(events, room_id, predicate, None).await
180+}
181+
182+async fn mentions_page_with_diagnostics(
183+ events: &mut macltok::EventStream,
184+ room_id: &ruma::RoomId,
185+ predicate: impl Fn(&MentionsPage) -> bool,
186+ diagnostic_server: Option<&MatrixMockServer>,
187+) -> MentionsPage {
188+ let mut last_page = None;
189+ let result = tokio::time::timeout(Duration::from_secs(5), async {
190+ while let Some(event) = events.recv().await {
191+ match event {
192+ Event::TimelineReset { timeline_id, .. } if timeline_id.room_id() == room_id => {
193+ panic!("the unopened room produced a timeline reset");
194+ }
195+ Event::MentionsChanged(update) => {
196+ let snapshot = update.snapshot;
197+ let matched = predicate(&snapshot);
198+ last_page = Some(snapshot.clone());
199+ if matched {
200+ return snapshot;
201+ }
202+ }
203+ _ => {}
204+ }
205+ }
206+ panic!("event stream closed before the expected mentions page");
207+ })
208+ .await;
209+
210+ match result {
211+ Ok(page) => page,
212+ Err(_) => {
213+ let last_state = last_page.map(|page: MentionsPage| {
214+ (
215+ page.revision,
216+ page.unread_count,
217+ page.entries
218+ .iter()
219+ .map(|entry| {
220+ (
221+ entry.key.room_id.to_string(),
222+ entry.key.event_id.to_string(),
223+ entry.capture.clone(),
224+ entry.revision,
225+ )
226+ })
227+ .collect::<Vec<_>>(),
228+ )
229+ });
230+ let requests = if let Some(server) = diagnostic_server {
231+ server.server().received_requests().await.map(|requests| {
232+ requests
233+ .iter()
234+ .filter(|request| {
235+ let path = request.url.path();
236+ path.contains(room_id.as_str())
237+ && (path.contains("/context/") || path.ends_with("/messages"))
238+ })
239+ .map(|request| format!("{} {}", request.method, request.url))
240+ .collect::<Vec<_>>()
241+ })
242+ } else {
243+ None
244+ };
245+ panic!(
246+ "expected mentions update did not arrive, last state {last_state:?}, requests {requests:?}"
247+ );
248+ }
249+ }
250+}
251+
252+fn entry_ids(page: &MentionsPage) -> HashSet<String> {
253+ page.entries
254+ .iter()
255+ .map(|entry| entry.key.event_id.to_string())
256+ .collect()
257+}
258+
259+async fn wait_for_requests(feed: &SlidingFeed, minimum: usize) {
260+ tokio::time::timeout(Duration::from_secs(5), async {
261+ while feed.requests.load(Ordering::Acquire) < minimum {
262+ tokio::task::yield_now().await;
263+ }
264+ })
265+ .await
266+ .expect("the sliding sync feed did not receive another request");
267+}
268+
269+async fn mount_messages_page(
270+ server: &MatrixMockServer,
271+ room_id: &ruma::RoomId,
272+ direction: &str,
273+ token: &str,
274+ limit: u16,
275+) {
276+ Mock::given(method("GET"))
277+ .and(path_regex(format!(
278+ r"^/_matrix/client/v3/rooms/{room_id}/messages$"
279+ )))
280+ .and(query_param("dir", direction))
281+ .and(query_param("from", token))
282+ .and(move |request: &Request| {
283+ request
284+ .url
285+ .query_pairs()
286+ .find(|(key, _)| key == "limit")
287+ .map_or(limit == 10, |(_, value)| value == limit.to_string())
288+ })
289+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
290+ "start": token,
291+ "end": format!("{token}-next"),
292+ "chunk": [],
293+ "state": [],
294+ })))
295+ .expect(1)
296+ .mount(server.server())
297+ .await;
298+}
299+
300+#[tokio::test]
301+async fn unopened_room_mentions_are_deduplicated_and_read_through_uses_a_revision_boundary() {
302+ let server = MatrixMockServer::new().await;
303+ let room_id = room_id!("!mentions-inbox:localhost");
304+ let (client, own_user) = joined_client(&server, room_id).await;
305+ let sender = user_id!("@alice:localhost");
306+ let initial = vec![
307+ message(
308+ "$mention",
309+ sender.as_str(),
310+ 2_000,
311+ "direct mention",
312+ Some(own_user.as_str()),
313+ ),
314+ message(
315+ "$own-mention",
316+ own_user.as_str(),
317+ 2_100,
318+ "own event",
319+ Some(own_user.as_str()),
320+ ),
321+ message(
322+ "$unrelated",
323+ sender.as_str(),
324+ 2_200,
325+ "ordinary message",
326+ None,
327+ ),
328+ ];
329+ let feed = SlidingFeed::new(room_id.to_owned(), initial.clone());
330+ feed.mount(&server).await;
331+
332+ let (engine, handle, mut events) = Engine::from_client(
333+ client,
334+ EngineConfig::default().with_initial_history_limit(0),
335+ )
336+ .await
337+ .unwrap();
338+ let running = tokio::spawn(engine.run());
339+
340+ let first = mentions_page(&mut events, room_id, |page| !page.entries.is_empty()).await;
341+ assert_eq!(entry_ids(&first), HashSet::from(["$mention".to_owned()]));
342+ assert_eq!(first.unread_count, 1);
343+ let first_boundary = first.boundary.clone();
344+
345+ wait_for_requests(&feed, 3).await;
346+ let replayed = handle.mentions_page(None, 50).await.unwrap();
347+ assert_eq!(entry_ids(&replayed), HashSet::from(["$mention".to_owned()]));
348+
349+ let key = replayed.entries[0].key.clone();
350+ handle
351+ .set_mentions_read(vec![key.clone()], true)
352+ .await
353+ .unwrap();
354+ let read = handle.mentions_page(None, 50).await.unwrap();
355+ assert_eq!(read.unread_count, 0);
356+
357+ feed.replace(
358+ room_id,
359+ [
360+ initial,
361+ vec![message(
362+ "$older-mention",
363+ sender.as_str(),
364+ 1_000,
365+ "older direct mention",
366+ Some(own_user.as_str()),
367+ )],
368+ ]
369+ .concat(),
370+ );
371+ let updated = mentions_page(&mut events, room_id, |page| page.entries.len() == 2).await;
372+ assert_eq!(
373+ entry_ids(&updated),
374+ HashSet::from(["$mention".to_owned(), "$older-mention".to_owned(),])
375+ );
376+
377+ handle
378+ .mark_mentions_read_through(first_boundary)
379+ .await
380+ .unwrap();
381+ let after_boundary = handle.mentions_page(None, 50).await.unwrap();
382+ let read_states: Vec<_> = after_boundary
383+ .entries
384+ .iter()
385+ .map(|entry| (entry.key.event_id.to_string(), entry.read))
386+ .collect();
387+ assert!(read_states.contains(&("$mention".to_owned(), true)));
388+ assert!(read_states.contains(&("$older-mention".to_owned(), false)));
389+ assert_eq!(after_boundary.unread_count, 1);
390+
391+ shut_down(&handle, &mut events, running).await;
392+}
393+
394+#[tokio::test]
395+async fn edits_can_remove_and_restore_a_mention_until_redaction_removes_its_target() {
396+ let server = MatrixMockServer::new().await;
397+ let room_id = room_id!("!mentions-edits:localhost");
398+ let (client, own_user) = joined_client(&server, room_id).await;
399+ let sender = user_id!("@alice:localhost");
400+ let original = message(
401+ "$edited-target",
402+ sender.as_str(),
403+ 1_000,
404+ "original mention",
405+ Some(own_user.as_str()),
406+ );
407+ let removal = json!({
408+ "type": "m.room.message",
409+ "event_id": "$remove-edit",
410+ "sender": sender,
411+ "origin_server_ts": 2_000,
412+ "content": {
413+ "msgtype": "m.text",
414+ "body": "* edited without a mention",
415+ "m.relates_to": { "rel_type": "m.replace", "event_id": "$edited-target" },
416+ "m.new_content": { "msgtype": "m.text", "body": "edited without a mention" },
417+ },
418+ });
419+ let addition = json!({
420+ "type": "m.room.message",
421+ "event_id": "$add-edit",
422+ "sender": sender,
423+ "origin_server_ts": 3_000,
424+ "content": {
425+ "msgtype": "m.text",
426+ "body": "* edited to mention you",
427+ "m.relates_to": { "rel_type": "m.replace", "event_id": "$edited-target" },
428+ "m.new_content": {
429+ "msgtype": "m.text",
430+ "body": "edited to mention you",
431+ "m.mentions": { "user_ids": [own_user.as_str()] },
432+ },
433+ },
434+ });
435+ let redaction = json!({
436+ "type": "m.room.redaction",
437+ "event_id": "$redaction",
438+ "sender": sender,
439+ "origin_server_ts": 4_000,
440+ "redacts": "$edited-target",
441+ "content": {},
442+ });
443+ let feed = SlidingFeed::new(room_id.to_owned(), vec![original.clone()]);
444+ feed.mount(&server).await;
445+
446+ let (engine, handle, mut events) = Engine::from_client(
447+ client,
448+ EngineConfig::default().with_initial_history_limit(0),
449+ )
450+ .await
451+ .unwrap();
452+ let running = tokio::spawn(engine.run());
453+
454+ let discovered = mentions_page(&mut events, room_id, |page| !page.entries.is_empty()).await;
455+ assert_eq!(
456+ entry_ids(&discovered),
457+ HashSet::from(["$edited-target".to_owned()])
458+ );
459+
460+ feed.replace(room_id, vec![original.clone(), removal.clone()]);
461+ let removed = mentions_page(&mut events, room_id, |page| {
462+ page.revision > discovered.revision && page.entries.is_empty()
463+ })
464+ .await;
465+ assert!(removed.entries.is_empty());
466+ assert_eq!(
467+ handle
468+ .mention_context(room_id, ruma::event_id!("$edited-target"))
469+ .await
470+ .unwrap_err()
471+ .kind,
472+ macltok::ErrorKind::EventNotFound
473+ );
474+
475+ feed.replace(room_id, vec![original.clone(), removal, addition]);
476+ let restored = mentions_page(&mut events, room_id, |page| {
477+ page.revision > removed.revision && entry_ids(page).contains("$edited-target")
478+ })
479+ .await;
480+ assert_eq!(
481+ entry_ids(&restored),
482+ HashSet::from(["$edited-target".to_owned()])
483+ );
484+
485+ feed.replace(room_id, vec![original, redaction]);
486+ let redacted = mentions_page(&mut events, room_id, |page| {
487+ page.revision > restored.revision && page.entries.is_empty()
488+ })
489+ .await;
490+ assert!(redacted.entries.is_empty());
491+ assert_eq!(
492+ handle
493+ .mention_context(room_id, ruma::event_id!("$edited-target"))
494+ .await
495+ .unwrap_err()
496+ .kind,
497+ macltok::ErrorKind::EventNotFound
498+ );
499+
500+ shut_down(&handle, &mut events, running).await;
501+}
502+
503+#[tokio::test]
504+async fn room_mentions_require_power_while_muted_direct_mentions_remain_and_ignored_senders_drop() {
505+ let server = MatrixMockServer::new().await;
506+ let client = server.client_builder().build().await;
507+ let own_user = client.user_id().unwrap().to_owned();
508+ let room_mention_room = room_id!("!room-mention-policy:localhost");
509+ let muted_room = room_id!("!muted-mention-policy:localhost");
510+ let ignored_sender = user_id!("@ignored:localhost");
511+ let sender = user_id!("@alice:localhost");
512+ let mut rules = Ruleset::server_default(&own_user);
513+ rules.override_.shift_insert(
514+ 1,
515+ NewConditionalPushRule::new(
516+ "mute-inbox-room".to_owned(),
517+ vec![PushCondition::EventMatch(EventMatchConditionData::new(
518+ "room_id".to_owned(),
519+ muted_room.to_string(),
520+ ))],
521+ Vec::new(),
522+ )
523+ .into(),
524+ );
525+ server
526+ .mock_sync()
527+ .ok_and_run(&client, {
528+ let own_user = own_user.clone();
529+ let ignored_sender = ignored_sender.to_owned();
530+ move |builder| {
531+ let factory = EventFactory::new();
532+ builder.add_global_account_data(factory.push_rules(rules));
533+ builder.add_global_account_data(factory.ignored_user_list([ignored_sender]));
534+ builder.add_joined_room(
535+ JoinedRoomBuilder::new(room_mention_room)
536+ .add_state_event(
537+ EventFactory::new()
538+ .room(room_mention_room)
539+ .member(&own_user),
540+ )
541+ .add_state_event(
542+ EventFactory::new()
543+ .room(room_mention_room)
544+ .default_power_levels()
545+ .sender(&own_user),
546+ ),
547+ );
548+ builder.add_joined_room(
549+ JoinedRoomBuilder::new(muted_room)
550+ .add_state_event(EventFactory::new().room(muted_room).member(&own_user))
551+ .add_state_event(
552+ EventFactory::new()
553+ .room(muted_room)
554+ .default_power_levels()
555+ .sender(&own_user),
556+ ),
557+ );
558+ }
559+ })
560+ .await;
561+
562+ let feed = SlidingFeed::new(
563+ room_mention_room.to_owned(),
564+ vec![room_mention(
565+ "$unauthorized-room-mention",
566+ sender.as_str(),
567+ 3_000,
568+ )],
569+ );
570+ feed.mount(&server).await;
571+ let muted_events = vec![message(
572+ "$muted-direct-mention",
573+ sender.as_str(),
574+ 2_000,
575+ "muted direct mention",
576+ Some(own_user.as_str()),
577+ )];
578+ let ignored = message(
579+ "$ignored-direct-mention",
580+ ignored_sender.as_str(),
581+ 1_000,
582+ "ignored direct mention",
583+ Some(own_user.as_str()),
584+ );
585+ let mut muted_events = muted_events;
586+ muted_events.push(ignored);
587+ feed.add_room(muted_room, muted_events);
588+
589+ let (engine, handle, mut events) = Engine::from_client(
590+ client,
591+ EngineConfig::default().with_initial_history_limit(0),
592+ )
593+ .await
594+ .unwrap();
595+ let running = tokio::spawn(engine.run());
596+
597+ let page = mentions_page(&mut events, room_mention_room, |page| {
598+ page.entries
599+ .iter()
600+ .any(|entry| entry.key.event_id.as_str() == "$muted-direct-mention")
601+ })
602+ .await;
603+ assert_eq!(
604+ entry_ids(&page),
605+ HashSet::from(["$muted-direct-mention".to_owned()])
606+ );
607+ assert!(page.entries[0].notification_suppressed);
608+ assert!(!entry_ids(&page).contains("$unauthorized-room-mention"));
609+ assert!(!entry_ids(&page).contains("$ignored-direct-mention"));
610+ let room_ids: HashSet<_> = page
611+ .entries
612+ .iter()
613+ .map(|entry| entry.key.room_id.clone())
614+ .collect();
615+ assert!(room_ids.contains(&muted_room.to_owned()));
616+
617+ shut_down(&handle, &mut events, running).await;
618+}
619+
620+#[tokio::test]
621+async fn event_context_uses_independent_direction_limits_and_saved_mention_context_is_offline() {
622+ let server = MatrixMockServer::new().await;
623+ let room_id = room_id!("!mentions-context:localhost");
624+ let (client, own_user) = joined_client(&server, room_id).await;
625+ let mention_id = event_id!("$mention-context");
626+ let context_id = event_id!("$manual-context");
627+ let mention_event = EventFactory::new()
628+ .room(room_id)
629+ .text_msg("mention with saved context")
630+ .mentions(Mentions::with_user_ids([own_user.clone()]))
631+ .event_id(mention_id)
632+ .sender(&ALICE)
633+ .into_event();
634+ let context_event = EventFactory::new()
635+ .room(room_id)
636+ .text_msg("manually fetched context")
637+ .event_id(context_id)
638+ .sender(&ALICE)
639+ .into_event();
640+
641+ server
642+ .mock_room_event_context()
643+ .room(room_id.to_owned())
644+ .match_event_id()
645+ .ok(RoomContextResponseTemplate::new(mention_event.clone())
646+ .start("mention-before")
647+ .end("mention-after"))
648+ .expect(1)
649+ .mount()
650+ .await;
651+ server
652+ .mock_room_event_context()
653+ .room(room_id.to_owned())
654+ .match_event_id()
655+ .ok(RoomContextResponseTemplate::new(context_event)
656+ .start("manual-before")
657+ .end("manual-after"))
658+ .expect(1)
659+ .mount()
660+ .await;
661+ mount_messages_page(&server, room_id, "b", "mention-before", 10).await;
662+ mount_messages_page(&server, room_id, "f", "mention-after", 10).await;
663+ mount_messages_page(&server, room_id, "b", "manual-before", 2).await;
664+ mount_messages_page(&server, room_id, "f", "manual-after", 3).await;
665+
666+ let feed = SlidingFeed::new(
667+ room_id.to_owned(),
668+ vec![message(
669+ mention_id.as_str(),
670+ ALICE.as_str(),
671+ 5_000,
672+ "mention with saved context",
673+ Some(own_user.as_str()),
674+ )],
675+ );
676+ feed.mount(&server).await;
677+ let (engine, handle, mut events) = Engine::from_client(
678+ client,
679+ EngineConfig::default().with_initial_history_limit(0),
680+ )
681+ .await
682+ .unwrap();
683+ let running = tokio::spawn(engine.run());
684+
685+ let captured = mentions_page_with_diagnostics(
686+ &mut events,
687+ room_id,
688+ |page| {
689+ page.entries.iter().any(|entry| {
690+ entry.key.event_id == mention_id
691+ && matches!(&entry.capture, macltok::MentionCaptureState::Partial)
692+ })
693+ },
694+ Some(&server),
695+ )
696+ .await;
697+ let saved = handle.mention_context(room_id, mention_id).await.unwrap();
698+ assert!(saved.context.is_some());
699+ assert!(matches!(saved.state, macltok::MentionCaptureState::Partial));
700+
701+ let before_saved_request_count = server
702+ .server()
703+ .received_requests()
704+ .await
705+ .unwrap()
706+ .iter()
707+ .filter(|request| request.url.path().contains("/context/$mention-context"))
708+ .count();
709+ let _ = handle.mention_context(room_id, mention_id).await.unwrap();
710+ let after_saved_request_count = server
711+ .server()
712+ .received_requests()
713+ .await
714+ .unwrap()
715+ .iter()
716+ .filter(|request| request.url.path().contains("/context/$mention-context"))
717+ .count();
718+ assert_eq!(after_saved_request_count, before_saved_request_count);
719+
720+ let context = handle
721+ .event_context(room_id, context_id, 2, 3)
722+ .await
723+ .unwrap();
724+ assert_eq!(context.start_token.as_deref(), Some("manual-before-next"));
725+ assert_eq!(context.end_token.as_deref(), Some("manual-after-next"));
726+
727+ let requests = server.server().received_requests().await.unwrap();
728+ let context_limits: Vec<_> = requests
729+ .iter()
730+ .filter(|request| request.url.path().contains("/context/"))
731+ .map(|request| {
732+ request
733+ .url
734+ .query_pairs()
735+ .find(|(key, _)| key == "limit")
736+ .map(|(_, value)| value.into_owned())
737+ .unwrap()
738+ })
739+ .collect();
740+ assert_eq!(context_limits, ["0", "0"]);
741+ let queries: Vec<_> = requests
742+ .iter()
743+ .filter(|request| request.url.path().ends_with("/messages"))
744+ .map(|request| {
745+ let query: std::collections::HashMap<_, _> =
746+ request.url.query_pairs().into_owned().collect();
747+ (
748+ query.get("dir").cloned().unwrap(),
749+ query.get("from").cloned().unwrap(),
750+ query
751+ .get("limit")
752+ .cloned()
753+ .unwrap_or_else(|| "10".to_owned()),
754+ )
755+ })
756+ .collect();
757+ assert!(queries.contains(&("b".to_owned(), "mention-before".to_owned(), "10".to_owned())));
758+ assert!(queries.contains(&("f".to_owned(), "mention-after".to_owned(), "10".to_owned())));
759+ assert!(queries.contains(&("b".to_owned(), "manual-before".to_owned(), "2".to_owned())));
760+ assert!(queries.contains(&("f".to_owned(), "manual-after".to_owned(), "3".to_owned())));
761+ assert_eq!(captured.unread_count, 1);
762+
763+ shut_down(&handle, &mut events, running).await;
764+ server.server().verify().await;
765+}
766+
767+#[cfg(feature = "sqlite")]
768+#[tokio::test]
769+async fn sqlite_mentions_survive_engine_shutdown_and_session_resume() {
770+ static NEXT_TEMPORARY_PATH: AtomicUsize = AtomicUsize::new(0);
771+
772+ let server = MatrixMockServer::new().await;
773+ server
774+ .mock_versions()
775+ .with_simplified_sliding_sync()
776+ .ok()
777+ .mount()
778+ .await;
779+ server.mock_login().ok().mount().await;
780+
781+ let root = std::env::temp_dir().join(format!(
782+ "macltok-mentions-inbox-{}-{}",
783+ std::process::id(),
784+ NEXT_TEMPORARY_PATH.fetch_add(1, Ordering::Relaxed)
785+ ));
786+ let store_path = root.join("store");
787+ let sessions_path = root.join("records").join("session");
788+ let sessions = Arc::new(EncryptedSessionStore::file(
789+ &sessions_path,
790+ StoreConfig::sqlite(&store_path, "mentions inbox test passphrase"),
791+ ));
792+ let room_id = room_id!("!mentions-resume:matrix.org");
793+ let own_user = user_id!("@cheeky_monkey:matrix.org");
794+ let feed = SlidingFeed::new(
795+ room_id.to_owned(),
796+ vec![message(
797+ "$durable-mention",
798+ "@alice:matrix.org",
799+ 7_000,
800+ "persist this mention",
801+ Some(own_user.as_str()),
802+ )],
803+ );
804+ feed.set_required_state(
805+ room_id,
806+ vec![json!({
807+ "type": "m.room.member",
808+ "event_id": "$joined-member",
809+ "sender": own_user,
810+ "state_key": own_user,
811+ "origin_server_ts": 1,
812+ "content": { "membership": "join" },
813+ })],
814+ );
815+ feed.mount(&server).await;
816+
817+ let config = EngineConfig::default().with_initial_history_limit(0);
818+ let session = Session::login_password(
819+ PasswordLogin::new(server.uri(), "example", "wordpass").allow_insecure_loopback(true),
820+ sessions.clone(),
821+ )
822+ .await
823+ .unwrap();
824+ let (engine, handle, mut events) = session.build(config.clone()).await.unwrap();
825+ let running = tokio::spawn(engine.run());
826+ let first = mentions_page(&mut events, room_id, |page| !page.entries.is_empty()).await;
827+ assert_eq!(
828+ entry_ids(&first),
829+ HashSet::from(["$durable-mention".to_owned()])
830+ );
831+ shut_down(&handle, &mut events, running).await;
832+ drop(handle);
833+
834+ let Some(Restore::Resumed(session)) = Session::restore(sessions.clone()).await.unwrap() else {
835+ panic!("the saved session did not resume");
836+ };
837+ let (engine, handle, mut events) = session.build(config).await.unwrap();
838+ let running = tokio::spawn(engine.run());
839+ receive_until(&mut events, |event| {
840+ matches!(event, Event::Authenticated { .. })
841+ })
842+ .await;
843+ let resumed = handle.mentions_page(None, 50).await.unwrap();
844+ assert_eq!(
845+ entry_ids(&resumed),
846+ HashSet::from(["$durable-mention".to_owned()])
847+ );
848+ assert_eq!(resumed.unread_count, 1);
849+ shut_down(&handle, &mut events, running).await;
850+ drop(handle);
851+
852+ sessions.discard().await.unwrap();
853+ std::fs::remove_dir_all(root).unwrap();
854+}
855+
856+#[cfg(feature = "serde")]
857+#[test]
858+fn mentions_command_and_page_round_trip_through_serde() {
859+ let command = MentionsCommand::ReadThrough(MentionsBoundary {
860+ account: user_id!("@alice:example.org").to_owned(),
861+ revision: 7,
862+ });
863+ let encoded_command = serde_json::to_value(&command).unwrap();
864+ let decoded_command: MentionsCommand = serde_json::from_value(encoded_command).unwrap();
865+ assert_eq!(decoded_command, command);
866+
867+ let page = MentionsPage {
868+ entries: Vec::new(),
869+ next_cursor: Some("opaque cursor".to_owned()),
870+ end_reached: false,
871+ coverage: MentionCoverage::default(),
872+ unread_count: 0,
873+ revision: 7,
874+ boundary: MentionsBoundary {
875+ account: user_id!("@alice:example.org").to_owned(),
876+ revision: 7,
877+ },
878+ };
879+ let encoded_page = serde_json::to_value(&page).unwrap();
880+ let decoded_page: MentionsPage = serde_json::from_value(encoded_page).unwrap();
881+ assert_eq!(decoded_page, page);
882+}
Added file: tests/mentions_late_decryption.rs+207 additions
@@ -0,0 +1,207 @@
1+mod support;
2+
3+use std::sync::{
4+ Arc,
5+ atomic::{AtomicUsize, Ordering},
6+};
7+use std::time::Duration;
8+
9+use macltok::{Engine, EngineConfig, Event, EventStream, MentionsPage};
10+use matrix_sdk::{
11+ ruma::{RoomVersionId, device_id, event_id, room_id, user_id},
12+ test_utils::mocks::MatrixMockServer,
13+};
14+use matrix_sdk_test::{JoinedRoomBuilder, event_factory::EventFactory};
15+use serde_json::{Value, json};
16+use support::shut_down;
17+use wiremock::{
18+ Mock, Request, Respond, ResponseTemplate,
19+ matchers::{method, path},
20+};
21+
22+const SLIDING_SYNC_PATH: &str = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync";
23+
24+struct EncryptedSlidingFeed {
25+ room_id: String,
26+ event: Value,
27+ requests: Arc<AtomicUsize>,
28+}
29+
30+impl Clone for EncryptedSlidingFeed {
31+ fn clone(&self) -> Self {
32+ Self {
33+ room_id: self.room_id.clone(),
34+ event: self.event.clone(),
35+ requests: self.requests.clone(),
36+ }
37+ }
38+}
39+
40+impl EncryptedSlidingFeed {
41+ async fn mount(&self, server: &MatrixMockServer) {
42+ Mock::given(method("POST"))
43+ .and(path(SLIDING_SYNC_PATH))
44+ .respond_with(self.clone())
45+ .mount(server.server())
46+ .await;
47+ }
48+}
49+
50+impl Respond for EncryptedSlidingFeed {
51+ fn respond(&self, _request: &Request) -> ResponseTemplate {
52+ let request = self.requests.fetch_add(1, Ordering::AcqRel);
53+ ResponseTemplate::new(200)
54+ .set_delay(Duration::from_millis(20))
55+ .set_body_json(json!({
56+ "txn_id": null,
57+ "pos": format!("late-decryption-{request}"),
58+ "lists": {},
59+ "rooms": {
60+ (self.room_id.clone()): {
61+ "initial": request == 0,
62+ "required_state": [],
63+ "timeline": [self.event.clone()],
64+ }
65+ },
66+ "extensions": {},
67+ }))
68+ }
69+}
70+
71+async fn mentions_page(
72+ events: &mut EventStream,
73+ room_id: &ruma::RoomId,
74+ predicate: impl Fn(&MentionsPage) -> bool,
75+) -> MentionsPage {
76+ tokio::time::timeout(Duration::from_secs(5), async {
77+ while let Some(event) = events.recv().await {
78+ match event {
79+ Event::TimelineReset { timeline_id, .. } if timeline_id.room_id() == room_id => {
80+ panic!("the unopened room produced a timeline reset");
81+ }
82+ Event::MentionsChanged(update) if predicate(&update.snapshot) => {
83+ return update.snapshot;
84+ }
85+ _ => {}
86+ }
87+ }
88+ panic!("event stream closed before the expected mentions update");
89+ })
90+ .await
91+ .expect("expected mentions update did not arrive")
92+}
93+
94+async fn wait_for_requests(feed: &EncryptedSlidingFeed, minimum: usize) {
95+ tokio::time::timeout(Duration::from_secs(5), async {
96+ while feed.requests.load(Ordering::Acquire) < minimum {
97+ tokio::task::yield_now().await;
98+ }
99+ })
100+ .await
101+ .expect("the sliding sync feed did not receive another request");
102+}
103+
104+#[tokio::test]
105+async fn late_room_key_updates_the_production_mentions_inbox_once() {
106+ let server = MatrixMockServer::new().await;
107+ server.mock_crypto_endpoints_preset().await;
108+ let alice_id = user_id!("@alice:localhost");
109+ let bob_id = user_id!("@bob:localhost");
110+ let room_id = room_id!("!mentions-late-decryption:localhost");
111+ let event_id = event_id!("$late-production-mention");
112+ let alice = server
113+ .client_builder_for_crypto_end_to_end(alice_id, device_id!("ALICEDEVICE"))
114+ .build()
115+ .await;
116+ let bob = server
117+ .client_builder_for_crypto_end_to_end(bob_id, device_id!("BOBDEVICE"))
118+ .build()
119+ .await;
120+ bob.event_cache().subscribe().unwrap();
121+ server.exchange_e2ee_identities(&alice, &bob).await;
122+
123+ let factory = EventFactory::new().room(room_id).sender(alice_id);
124+ let joined = JoinedRoomBuilder::new(room_id)
125+ .add_state_event(factory.create(alice_id, RoomVersionId::V1))
126+ .add_state_event(factory.room_encryption())
127+ .add_state_event(factory.member(alice_id))
128+ .add_state_event(factory.member(bob_id));
129+ server.sync_room(&alice, joined.clone()).await;
130+ server.sync_room(&bob, joined).await;
131+
132+ let room = alice.get_room(room_id).unwrap();
133+ let (encrypted_event, send) = server.mock_room_send().ok_with_capture(event_id, alice_id);
134+ let _send = send.mock_once().mount_as_scoped().await;
135+ let (_keys, room_key) = server.mock_capture_put_to_device(alice_id).await;
136+ server
137+ .mock_get_members()
138+ .ok(vec![
139+ factory.member(alice_id).into_raw(),
140+ factory.member(bob_id).into_raw(),
141+ ])
142+ .mock_once()
143+ .mount()
144+ .await;
145+ room.send_raw(
146+ "m.room.message",
147+ json!({
148+ "msgtype": "m.text",
149+ "body": "a private mention",
150+ "m.mentions": { "user_ids": [bob_id] }
151+ }),
152+ )
153+ .await
154+ .unwrap();
155+ let encrypted_event = encrypted_event.await.unwrap();
156+ let room_key = room_key.await;
157+ let encrypted_event: Value = serde_json::from_str(encrypted_event.json().get()).unwrap();
158+
159+ let feed = EncryptedSlidingFeed {
160+ room_id: room_id.to_string(),
161+ event: encrypted_event,
162+ requests: Arc::new(AtomicUsize::new(0)),
163+ };
164+ feed.mount(&server).await;
165+
166+ let (engine, handle, mut events) = Engine::from_client(
167+ bob.clone(),
168+ EngineConfig::default().with_initial_history_limit(0),
169+ )
170+ .await
171+ .unwrap();
172+ let running = tokio::spawn(engine.run());
173+
174+ let unresolved = mentions_page(&mut events, room_id, |page| {
175+ page.coverage.unresolved_candidates == 1
176+ })
177+ .await;
178+ assert!(unresolved.entries.is_empty());
179+
180+ server
181+ .mock_sync()
182+ .ok_and_run(&bob, |sync| {
183+ sync.add_to_device_event(room_key.deserialize_as().unwrap());
184+ })
185+ .await;
186+
187+ let decrypted = mentions_page(&mut events, room_id, |page| {
188+ page.entries.len() == 1 && page.entries[0].key.event_id == event_id
189+ })
190+ .await;
191+ assert_eq!(decrypted.entries.len(), 1);
192+ assert_eq!(decrypted.entries[0].key.event_id, event_id);
193+ assert_eq!(decrypted.coverage.unresolved_candidates, 0);
194+
195+ let replay_start = feed.requests.load(Ordering::Acquire);
196+ wait_for_requests(&feed, replay_start + 3).await;
197+ let replayed = handle.mentions_page(None, 50).await.unwrap();
198+ assert_eq!(replayed.entries.len(), 1);
199+ assert_eq!(replayed.entries[0].key.event_id, event_id);
200+ while let Ok(event) = events.try_recv() {
201+ if let Event::TimelineReset { timeline_id, .. } = event {
202+ assert_ne!(timeline_id.room_id(), room_id);
203+ }
204+ }
205+
206+ shut_down(&handle, &mut events, running).await;
207+}
Modified file: tests/mirror.rs+97 additions−2 deletions
@@ -4,14 +4,15 @@ use std::time::Duration;
44
55 use futures_util::StreamExt;
66 use macltok::{
7- ClientHandle, Command, Engine, EngineConfig, Event, Mirror, MirrorHandle, StateChange,
7+ ClientHandle, Command, ConnectionState, Engine, EngineConfig, Event, MentionCoverage,
8+ MentionsBoundary, MentionsPage, MentionsUpdate, Mirror, MirrorHandle, StateChange,
89 Subscription, TimelineChange, TimelineCommand, TimelineDiff, TimelineId, TimelineItem, Update,
910 UploadOutcome,
1011 };
1112 use matrix_sdk::{
1213 ruma::{
1314 OwnedRoomId, OwnedTransactionId, TransactionId, api::client::sync::sync_events::v5,
14- event_id, room_id,
15+ event_id, room_id, user_id,
1516 },
1617 test_utils::mocks::MatrixMockServer,
1718 };
@@ -433,3 +434,97 @@ async fn an_upload_ends_on_finished() {
433434 .await;
434435 assert!(harness.mirror.snapshot().uploads.is_empty());
435436 }
437+
438+#[tokio::test]
439+async fn mentions_updates_require_the_current_account_and_revision() {
440+ let harness = start(EngineConfig::default(), 256).await;
441+ let mut updates = harness.mirror.subscribe();
442+ let account = tokio::time::timeout(Duration::from_secs(5), async {
443+ loop {
444+ let state = harness.mirror.snapshot();
445+ if state.mentions.is_some() {
446+ break state.user_id.clone().unwrap();
447+ }
448+ updates.next().await.unwrap();
449+ }
450+ })
451+ .await
452+ .expect("the initial mentions snapshot did not arrive");
453+ let mentions_update = |account, revision| {
454+ Event::MentionsChanged(MentionsUpdate {
455+ snapshot: MentionsPage {
456+ entries: Vec::new(),
457+ next_cursor: None,
458+ end_reached: true,
459+ coverage: MentionCoverage::default(),
460+ unread_count: 0,
461+ revision,
462+ boundary: MentionsBoundary { account, revision },
463+ },
464+ upserts: Vec::new(),
465+ removed: Vec::new(),
466+ })
467+ };
468+
469+ harness
470+ .inject
471+ .send(mentions_update(account.clone(), 100))
472+ .unwrap();
473+ next_matching(
474+ &mut updates,
475+ |change| matches!(change, StateChange::Mentions(update) if update.snapshot.revision == 100),
476+ )
477+ .await;
478+ assert_eq!(
479+ harness
480+ .mirror
481+ .snapshot()
482+ .mentions
483+ .as_ref()
484+ .unwrap()
485+ .revision,
486+ 100
487+ );
488+
489+ harness
490+ .inject
491+ .send(mentions_update(account.clone(), 99))
492+ .unwrap();
493+ harness
494+ .inject
495+ .send(mentions_update(
496+ user_id!("@other:localhost").to_owned(),
497+ 101,
498+ ))
499+ .unwrap();
500+ harness
501+ .inject
502+ .send(Event::ConnectionChanged(ConnectionState::Stopped))
503+ .unwrap();
504+ next_matching(&mut updates, |change| {
505+ matches!(change, StateChange::Connection(ConnectionState::Stopped))
506+ })
507+ .await;
508+ assert_eq!(
509+ harness
510+ .mirror
511+ .snapshot()
512+ .mentions
513+ .as_ref()
514+ .unwrap()
515+ .revision,
516+ 100
517+ );
518+
519+ harness.inject.send(Event::LoggedOut).unwrap();
520+ harness.inject.send(mentions_update(account, 102)).unwrap();
521+ harness
522+ .inject
523+ .send(Event::ConnectionChanged(ConnectionState::Stopped))
524+ .unwrap();
525+ next_matching(&mut updates, |change| {
526+ matches!(change, StateChange::Connection(ConnectionState::Stopped))
527+ })
528+ .await;
529+ assert!(harness.mirror.snapshot().mentions.is_none());
530+}

show truncated diff