macltok

public

Star0

Fork 0

Watch0

37 commits 3 branches 0 tags 0 releases3.0 MiB

Commit `17e042fbc331`

tarball

atagen <boss@atagen.co>2026-10-22 09:31 UTC

13 files changed+526-44

image_packs: hand out custom emote and sticker packs, and say when they change

MSC2545 packs live in account data and in room state the sdk's sliding sync never asks for, so the fork's extra required_state brings every room's im.ponies.room_emotes into the store, and a change arrives even across a timeline gap. Parsing is lenient per image, since pack contents are whatever someone's client wrote.

Modified file: Cargo.lock+1

@@ -1830,6 +1830,7 @@ dependencies = [

18301830 "imbl",

18311831 "js-sys",

18321832 "matrix-sdk",

1833+ "matrix-sdk-base",

18331834 "matrix-sdk-crypto",

18341835 "matrix-sdk-store-encryption",

18351836 "matrix-sdk-test",

Modified file: Cargo.toml+2 additions−1 deletions@@ -27,10 +27,11 @@ futures-util = { version = "0.3.31", default-features = false, features = ["std"
 http = "1"
 imbl = { version = "6.1.0", default-features = false }
 matrix-sdk = { version = "0.18.0", default-features = false, features = ["e2e-encryption"] }
+matrix-sdk-base = { version = "0.18.0", default-features = false }
 matrix-sdk-crypto = { version = "0.18.0", default-features = false }
 matrix-sdk-store-encryption = { version = "0.18.0", optional = true }
 matrix-sdk-ui = { version = "0.18.0", default-features = false }
-ruma = { version = "0.16.0", default-features = false }
+ruma = { version = "0.16.0", default-features = false, features = ["unstable-msc2545"] }
 serde = { version = "1.0.219", features = ["derive", "rc"], optional = true }
 thiserror = "2.0.16"
 mime = "0.3.17"

Modified file: docs/design-decisions.md+21 additions@@ -319,6 +319,25 @@ membership until it's forgotten. push rule changes never touch room info, so
 `NotificationSettingsChanged` brings a `RoomsChanged` along for the modes in
 the summaries
 
+## image packs
+
+msc2545 packs come from the user's account data, from room state, and from the
+rooms the user enabled everywhere. the room list asks the server for every
+room's `im.ponies.room_emotes`, through the fork's extra required state, so
+packs sit in the store like any other state and a change reaches us even across
+a gap in the timeline. that's every joined room's packs on the first sync,
+tens of kilobytes for a big one, and only changes after. asking only for open
+rooms would lose the enabled packs of rooms nobody has open
+
+`image_packs` reads all of it from the store and hands back the lot, global
+packs and each room's own, and `ImagePacks::for_room` puts a picker's list in
+order. an enabled room the user hasn't joined is skipped, there's no state for
+it. one bad image costs that image and a pack with nothing usable is dropped,
+since pack contents are whatever someone's client wrote. `ImagePacksChanged`
+says a pack moved somewhere, from a state event, an account data change or a
+membership change taking a room's packs with it. it's an invalidation, and the
+mirror refetches on it like it does rooms, keeping the packs in its state
+
 ## verification
 
 `Accept`, `Confirm`, `Reject` and `Cancel`. a request another device sends
@@ -362,6 +381,8 @@ matrix-sdk-ui for our own layer over matrix-sdk-base and matrix-sdk-crypto,
 which keep their public sans-io apis
 
 - `Room::prepare_encryption`, the send path's encryption prep made public
+- `SyncServiceBuilder::with_room_list_extra_required_state`, for state the
+  sdk's hard-coded list leaves out. image packs use it
 
 ## public model
 

Modified file: harness/web/Cargo.lock+1

@@ -1724,6 +1724,7 @@ dependencies = [

17241724 "imbl",

17251725 "js-sys",

17261726 "matrix-sdk",

1727+ "matrix-sdk-base",

17271728 "matrix-sdk-crypto",

17281729 "matrix-sdk-store-encryption",

17291730 "matrix-sdk-ui",

Modified file: src/engine/mod.rs+7 additions−1 deletions@@ -41,6 +41,7 @@ use crate::{
     AccountCommand, ClientHandle, Command, ConnectionState, EncryptionCommand, EngineErrorKind,
     EngineFailure, Error, ErrorKind, Event, EventStream, LifecycleCommand, MediaCommand,
     NotificationCommand, RoomCommand, Termination, TimelineCommand,
+    normalise::ROOM_PACKS,
     transport::{DriverPort, Request, Responder, transport_pair},
 };
 
@@ -313,6 +314,7 @@ impl Engine {
         let sync = Arc::new(
             SyncService::builder(client.clone())
                 .with_offline_mode()
+                .with_room_list_extra_required_state(vec![(ROOM_PACKS.into(), "*".to_owned())])
                 .build()
                 .await
                 .map_err(EngineError::SyncService)?,
@@ -462,7 +464,7 @@ impl Engine {
         let client = &self.client;
         let events = || self.port.event_sender();
         let (notices, notice_queue) = mpsc::unbounded_channel();
-        let watchers: [(&'static str, EngineBoxFuture<'static, ()>); 12] = [
+        let watchers: [(&'static str, EngineBoxFuture<'static, ()>); 13] = [
             (
                 "upload tracker",
                 Box::pin(watchers::uploads(
@@ -528,6 +530,10 @@ impl Engine {
                 Box::pin(watchers::rooms(client.clone(), events())),
             ),
             (
+                "image pack watcher",
+                Box::pin(watchers::image_packs(client.clone(), events())),
+            ),
+            (
                 "notification settings watcher",
                 Box::pin(watchers::notification_settings(client.clone(), events())),
             ),

Modified file: src/engine/watchers.rs+67 additions−3 deletions@@ -6,13 +6,22 @@ use matrix_sdk::{
     ruma::events::{AnyMessageLikeEventContent, room::message::MessageType},
     send_queue::{LocalEchoContent, RoomSendQueueUpdate, SendQueueUpdate},
 };
+use matrix_sdk_base::RoomInfoNotableUpdateReasons;
 use matrix_sdk_ui::sync_service::{State as SyncState, SyncService};
-use ruma::{OwnedRoomId, OwnedTransactionId, time::Instant};
-use tokio::sync::{broadcast::error::RecvError, mpsc, watch};
+use ruma::{
+    OwnedRoomId, OwnedTransactionId,
+    events::{AnyGlobalAccountDataEvent, AnySyncStateEvent},
+    serde::Raw,
+    time::Instant,
+};
+use tokio::sync::{Notify, broadcast::error::RecvError, mpsc, watch};
 
 use super::executor::TaskSet;
 use crate::{
-    ConnectionState, Event, UploadOutcome, encryption::encryption_state, error::classify_sdk_error,
+    ConnectionState, Event, UploadOutcome,
+    encryption::encryption_state,
+    error::classify_sdk_error,
+    normalise::{ENABLED_ROOMS, ROOM_PACKS, USER_PACK},
     transport::EventSender,
 };
 
@@ -425,6 +434,61 @@ pub(super) async fn rooms(client: Client, events: EventSender) {
     }
 }
 
+/// packs live in room state and account data, and leaving a room takes its packs with it
+pub(super) async fn image_packs(client: Client, events: EventSender) {
+    let changed = Arc::new(Notify::new());
+    let handlers = [
+        client.add_event_handler({
+            let changed = changed.clone();
+            move |event: Raw<AnySyncStateEvent>| {
+                let changed = changed.clone();
+                async move {
+                    if event_type(&event).as_deref() == Some(ROOM_PACKS) {
+                        changed.notify_one();
+                    }
+                }
+            }
+        }),
+        client.add_event_handler({
+            let changed = changed.clone();
+            move |event: Raw<AnyGlobalAccountDataEvent>| {
+                let changed = changed.clone();
+                async move {
+                    if matches!(
+                        event_type(&event).as_deref(),
+                        Some(USER_PACK | ENABLED_ROOMS)
+                    ) {
+                        changed.notify_one();
+                    }
+                }
+            }
+        }),
+    ];
+    let mut info = client.room_info_notable_update_receiver();
+    loop {
+        tokio::select! {
+            () = changed.notified() => {}
+            update = info.recv() => match update {
+                Ok(update) if !update.reasons.contains(RoomInfoNotableUpdateReasons::MEMBERSHIP) => {
+                    continue;
+                }
+                Ok(_) | Err(RecvError::Lagged(_)) => {}
+                Err(RecvError::Closed) => break,
+            },
+        }
+        if events.send(Event::ImagePacksChanged).await.is_err() {
+            break;
+        }
+    }
+    for handler in handlers {
+        client.remove_event_handler(handler);
+    }
+}
+
+fn event_type<T>(event: &Raw<T>) -> Option<String> {
+    event.get_field("type").ok().flatten()
+}
+
 pub(super) async fn notification_settings(client: Client, events: EventSender) {
     let settings = client.notification_settings().await;
     let mut changes = settings.subscribe_to_changes();

Modified file: src/lib.rs+13 additions−13 deletions@@ -48,19 +48,19 @@ pub use mirror::{
 pub use model::{
     AccountCommand, AttachmentEncryption, AttachmentUpload, Change, Command, CommandOutput,
     ConnectionState, CreateRoomOptions, EncryptionCommand, EncryptionState, EngineErrorKind,
-    EngineFailure, Event, EventRef, HistoryState, KeyBackupState, LatestEvent, LifecycleCommand,
-    Media, MediaCommand, MediaDescriptor, MediaFile, MediaId, MediaKind, MediaLocation, MediaSize,
-    MemberFilter, MemberMembership, MemberSummary, MembershipChange, Message, MessageContent,
-    MessageEditContent, MessageKind, MessageMentions, NotificationCommand, OwnDevice,
-    PowerLevelChanges, Profile, PushNotification, PushNotificationEvent, PushNotificationRoom,
-    PushNotificationSender, PushNotificationStatus, Reaction, RecoveryState, ReplyDetails,
-    ReplyReference, Retain, RichText, RoomCategory, RoomCommand, RoomHero, RoomListFilter,
-    RoomMatcher, RoomMembership, RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels,
-    RoomPreset, RoomSummary, RoomVisibility, SendState, Sender, SpaceChild, Termination,
-    ThreadSummary, TimelineCommand, TimelineContent, TimelineDiff, TimelineDiffError,
-    TimelineEvent, TimelineId, TimelineItem, TimelineItemId, TypingState, UploadDetails,
-    UploadOutcome, UploadThumbnail, UtdCause, VerificationAction, VerificationFailure,
-    VerificationState, VerificationTarget,
+    EngineFailure, Event, EventRef, HistoryState, ImagePack, ImagePackSource, ImagePacks,
+    KeyBackupState, LatestEvent, LifecycleCommand, Media, MediaCommand, MediaDescriptor, MediaFile,
+    MediaId, MediaKind, MediaLocation, MediaSize, MemberFilter, MemberMembership, MemberSummary,
+    MembershipChange, Message, MessageContent, MessageEditContent, MessageKind, MessageMentions,
+    NotificationCommand, OwnDevice, PackImage, PackUsage, PowerLevelChanges, Profile,
+    PushNotification, PushNotificationEvent, PushNotificationRoom, PushNotificationSender,
+    PushNotificationStatus, Reaction, RecoveryState, ReplyDetails, ReplyReference, Retain,
+    RichText, RoomCategory, RoomCommand, RoomHero, RoomListFilter, RoomMatcher, RoomMembership,
+    RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels, RoomPreset, RoomSummary,
+    RoomVisibility, SendState, Sender, SpaceChild, Termination, ThreadSummary, TimelineCommand,
+    TimelineContent, TimelineDiff, TimelineDiffError, TimelineEvent, TimelineId, TimelineItem,
+    TimelineItemId, TypingState, UploadDetails, UploadOutcome, UploadThumbnail, UtdCause,
+    VerificationAction, VerificationFailure, VerificationState, VerificationTarget,
 };
 pub use ruma;
 pub use transport::{ClientHandle, EventStream, Pending, SpaceHierarchy};

Modified file: src/mirror.rs+87 additions−19 deletions@@ -20,7 +20,7 @@ use tokio::sync::broadcast;
 
 use crate::{
     ClientHandle, Command, ConnectionState, EncryptionState, EngineFailure, ErrorKind, Event,
-    HistoryState, RoomSummary, TimelineCommand, TimelineDiff, TimelineId, TimelineItem,
+    HistoryState, ImagePacks, RoomSummary, TimelineCommand, TimelineDiff, TimelineId, TimelineItem,
     TypingState, UploadOutcome, VerificationState, VerificationTarget,
     engine::executor::EngineBoxFuture,
 };
@@ -46,6 +46,7 @@ pub struct State {
     pub verification: Option<VerificationState>,
     pub verification_target: Option<VerificationTarget>,
     pub rooms: Vector<RoomSummary>,
+    pub image_packs: ImagePacks,
     pub timelines: imbl::HashMap<TimelineId, TimelineState>,
     pub typing: imbl::HashMap<OwnedRoomId, Vec<OwnedUserId>>,
     pub uploads: imbl::HashMap<OwnedTransactionId, UploadState>,
@@ -125,6 +126,7 @@ pub enum StateChange {
     /// room modes arrive with the `Rooms` that follows
     NotificationSettings,
     Rooms(Vector<RoomSummary>),
+    ImagePacks(ImagePacks),
     Timeline {
         timeline_id: TimelineId,
         change: TimelineChange,
@@ -169,6 +171,7 @@ pub struct Mirror {
     shared: Arc<Shared>,
     resyncs: HashMap<TimelineId, AbortHandle>,
     rooms_fetch: Option<AbortHandle>,
+    packs_fetch: Option<AbortHandle>,
 }
 
 impl fmt::Debug for Mirror {
@@ -201,6 +204,7 @@ impl Mirror {
             verification: None,
             verification_target: None,
             rooms: Vector::new(),
+            image_packs: ImagePacks::default(),
             timelines: imbl::HashMap::new(),
             typing: imbl::HashMap::new(),
             uploads: imbl::HashMap::new(),
@@ -212,7 +216,8 @@ impl Mirror {
                 state: Arc::new(state),
                 sequence: 0,
                 updates: Some(broadcast::channel(capacity.max(1)).0),
-                rooms_fetch: RoomsFetch::Idle,
+                rooms_fetch: Fetch::Idle,
+                packs_fetch: Fetch::Idle,
             }),
         });
         #[cfg(not(target_family = "wasm"))]
@@ -225,6 +230,7 @@ impl Mirror {
             shared: shared.clone(),
             resyncs: HashMap::new(),
             rooms_fetch: None,
+            packs_fetch: None,
         };
         (mirror, MirrorHandle { shared })
     }
@@ -233,6 +239,7 @@ impl Mirror {
     pub fn run(mut self) -> EngineBoxFuture<'static, ()> {
         Box::pin(async move {
             self.refetch_rooms();
+            self.refetch_image_packs();
             while let Some(event) = self.events.next().await {
                 self.apply(event);
             }
@@ -286,6 +293,7 @@ impl Mirror {
                 self.publish(|_| Some(StateChange::NotificationSettings));
             }
             Event::RoomsChanged => self.refetch_rooms(),
+            Event::ImagePacksChanged => self.refetch_image_packs(),
             Event::EncryptionChanged(encryption) => self.publish(|state| {
                 state.encryption = Some(encryption.clone());
                 Some(StateChange::Encryption(encryption))
@@ -451,16 +459,22 @@ impl Mirror {
     }
 
     fn refetch_rooms(&mut self) {
-        let mut published = self.shared.lock();
-        match published.rooms_fetch {
-            RoomsFetch::Idle => {
-                published.rooms_fetch = RoomsFetch::Running;
-                drop(published);
-                let task = spawn(fetch_rooms(self.handle.clone(), self.shared.clone()));
-                self.rooms_fetch = Some(task.abort_handle());
-            }
-            RoomsFetch::Running => published.rooms_fetch = RoomsFetch::Again,
-            RoomsFetch::Again => {}
+        let fetch = fetch_rooms(self.handle.clone(), self.shared.clone());
+        if let Some(task) = self
+            .shared
+            .start(|published| &mut published.rooms_fetch, fetch)
+        {
+            self.rooms_fetch = Some(task);
+        }
+    }
+
+    fn refetch_image_packs(&mut self) {
+        let fetch = fetch_image_packs(self.handle.clone(), self.shared.clone());
+        if let Some(task) = self
+            .shared
+            .start(|published| &mut published.packs_fetch, fetch)
+        {
+            self.packs_fetch = Some(task);
         }
     }
 
@@ -484,7 +498,10 @@ impl Mirror {
 impl Drop for Mirror {
     fn drop(&mut self) {
         self.cancel_resyncs();
-        if let Some(task) = self.rooms_fetch.take() {
+        for task in [self.rooms_fetch.take(), self.packs_fetch.take()]
+            .into_iter()
+            .flatten()
+        {
             task.abort();
         }
         self.shared.lock().updates = None;
@@ -518,12 +535,28 @@ async fn fetch_rooms(handle: ClientHandle, shared: Arc<Shared>) {
             state.rooms = rooms.clone();
             Some(StateChange::Rooms(rooms))
         });
-        let mut published = shared.lock();
-        if published.rooms_fetch != RoomsFetch::Again {
-            published.rooms_fetch = RoomsFetch::Idle;
+        if !shared.fetched(|published| &mut published.rooms_fetch) {
+            return;
+        }
+    }
+}
+
+/// a failed read keeps the packs from before, the next change asks again
+async fn fetch_image_packs(handle: ClientHandle, shared: Arc<Shared>) {
+    loop {
+        match handle.image_packs().await {
+            Ok(packs) => shared.publish(|state| {
+                if state.image_packs == packs {
+                    return None;
+                }
+                state.image_packs = packs.clone();
+                Some(StateChange::ImagePacks(packs))
+            }),
+            Err(error) => tracing::warn!(%error, "reading image packs failed"),
+        }
+        if !shared.fetched(|published| &mut published.packs_fetch) {
             return;
         }
-        published.rooms_fetch = RoomsFetch::Running;
     }
 }
 
@@ -592,11 +625,12 @@ struct Published {
     sequence: u64,
     /// gone once the mirror stops, which ends every subscription
     updates: Option<broadcast::Sender<Update>>,
-    rooms_fetch: RoomsFetch,
+    rooms_fetch: Fetch,
+    packs_fetch: Fetch,
 }
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum RoomsFetch {
+enum Fetch {
     Idle,
     Running,
     Again,
@@ -609,6 +643,40 @@ impl Shared {
             .unwrap_or_else(PoisonError::into_inner)
     }
 
+    /// one fetch at a time, with at most one more owed
+    fn start(
+        &self,
+        fetch: impl FnOnce(&mut Published) -> &mut Fetch,
+        task: impl Future<Output = ()> + SendOutsideWasm + 'static,
+    ) -> Option<AbortHandle> {
+        let mut published = self.lock();
+        let fetch = fetch(&mut published);
+        match *fetch {
+            Fetch::Idle => {
+                *fetch = Fetch::Running;
+                drop(published);
+                Some(spawn(task).abort_handle())
+            }
+            Fetch::Running => {
+                *fetch = Fetch::Again;
+                None
+            }
+            Fetch::Again => None,
+        }
+    }
+
+    /// whether another fetch was asked for while this one ran
+    fn fetched(&self, fetch: impl FnOnce(&mut Published) -> &mut Fetch) -> bool {
+        let mut published = self.lock();
+        let fetch = fetch(&mut published);
+        if *fetch == Fetch::Again {
+            *fetch = Fetch::Running;
+            return true;
+        }
+        *fetch = Fetch::Idle;
+        false
+    }
+
     /// state, sequence and send move together under the one lock, which is what lets
     /// `subscribe` take a snapshot and a receiver with nothing between them
     fn publish(&self, change: impl FnOnce(&mut State) -> Option<StateChange>) {

Modified file: src/model.rs+81 additions−1 deletions@@ -3,7 +3,7 @@ use std::{fmt, sync::Arc};
 use imbl::Vector;
 use ruma::{
     MxcUri, OwnedDeviceId, OwnedEventId, OwnedMxcUri, OwnedRoomAliasId, OwnedRoomId,
-    OwnedRoomOrAliasId, OwnedServerName, OwnedTransactionId, OwnedUserId,
+    OwnedRoomOrAliasId, OwnedServerName, OwnedTransactionId, OwnedUserId, RoomId,
     events::{
         AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent, AnySyncTimelineEvent,
         GlobalAccountDataEventType, RoomAccountDataEventType,
@@ -654,6 +654,8 @@ pub enum Event {
     NotificationSettingsChanged,
     VerificationTargetChanged(VerificationTarget),
     RoomsChanged,
+    /// an image pack came, went or changed somewhere. ask `image_packs` again
+    ImagePacksChanged,
     /// replaces whatever you had for this timeline. every diff after it applies to it
     TimelineReset {
         timeline_id: TimelineId,
@@ -1608,6 +1610,84 @@ pub struct Reaction {
     pub senders: Vec<OwnedUserId>,
 }
 
+/// every custom emote and sticker pack the account can reach, from the user's own
+/// pack, each joined room's packs, and the rooms the user enabled everywhere
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub struct ImagePacks {
+    /// the user's own pack, then the packs of rooms enabled everywhere
+    pub global: Vec<ImagePack>,
+    /// each joined room's own packs, rooms without any left out
+    pub rooms: imbl::HashMap<OwnedRoomId, Vec<ImagePack>>,
+}
+
+impl ImagePacks {
+    /// what a picker in this room offers. the user's pack, the room's own, then the
+    /// rest of the enabled ones, each pack once
+    #[must_use]
+    pub fn for_room(&self, room_id: Option<&RoomId>) -> Vec<&ImagePack> {
+        let is_user = |pack: &&ImagePack| pack.source == ImagePackSource::User;
+        let own = room_id.and_then(|room_id| self.rooms.get(room_id));
+        let mut packs: Vec<&ImagePack> = self.global.iter().filter(is_user).collect();
+        packs.extend(own.into_iter().flatten());
+        for pack in self.global.iter().filter(|pack| !is_user(pack)) {
+            if !packs.iter().any(|seen| seen.id == pack.id) {
+                packs.push(pack);
+            }
+        }
+        packs
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub struct ImagePack {
+    /// `user`, or `room:<room id>:<state key>`, the same across calls
+    pub id: String,
+    pub display_name: Option<String>,
+    pub avatar: Option<MediaId>,
+    pub attribution: Option<String>,
+    pub source: ImagePackSource,
+    pub images: Vec<PackImage>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub enum ImagePackSource {
+    User,
+    Room {
+        room_id: OwnedRoomId,
+        state_key: String,
+    },
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub struct PackImage {
+    pub shortcode: String,
+    pub media_id: MediaId,
+    /// what `toggle_reaction` takes to react with this image
+    pub reaction_key: String,
+    pub body: Option<String>,
+    pub mimetype: Option<String>,
+    pub width: Option<u64>,
+    pub height: Option<u64>,
+    /// the image's own, else its pack's, else both
+    pub usage: Vec<PackUsage>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub enum PackUsage {
+    Emoticon,
+    Sticker,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[non_exhaustive]

Added file: src/normalise/image_pack.rs+152 additions@@ -0,0 +1,152 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use matrix_sdk::{Client, Room, deserialized_responses::RawAnySyncOrStrippedState};
+use ruma::{
+    OwnedRoomId,
+    events::{
+        GlobalAccountDataEventType, StateEventType,
+        image_pack::{PackImage as SourceImage, PackInfo, PackUsage as SourceUsage},
+    },
+    serde::{JsonObject, Raw},
+};
+
+use crate::{Error, ImagePack, ImagePackSource, ImagePacks, MediaId, PackImage, PackUsage};
+
+pub(crate) const ROOM_PACKS: &str = "im.ponies.room_emotes";
+pub(crate) const USER_PACK: &str = "im.ponies.user_emotes";
+pub(crate) const ENABLED_ROOMS: &str = "im.ponies.emote_rooms";
+
+/// rooms enabled everywhere that the user hasn't joined are skipped, we hold no state for them
+pub(crate) async fn image_packs(client: &Client) -> Result<ImagePacks, Error> {
+    let mut rooms = imbl::HashMap::new();
+    for room in client.joined_rooms() {
+        let packs = room_packs(&room).await?;
+        if !packs.is_empty() {
+            rooms.insert(room.room_id().to_owned(), packs);
+        }
+    }
+    let mut global = Vec::new();
+    if let Some(content) = account_data(client, USER_PACK).await? {
+        global.extend(pack("user".to_owned(), ImagePackSource::User, &content));
+    }
+    if let Some(content) = account_data(client, ENABLED_ROOMS).await? {
+        let enabled = content
+            .get_field::<BTreeMap<OwnedRoomId, BTreeMap<String, Raw<JsonObject>>>>("rooms")
+            .ok()
+            .flatten()
+            .unwrap_or_default();
+        for (room_id, state_keys) in enabled {
+            let packs = rooms.get(&room_id).into_iter().flatten();
+            global.extend(
+                packs
+                    .filter(|pack| match &pack.source {
+                        ImagePackSource::Room { state_key, .. } => {
+                            state_keys.contains_key(state_key)
+                        }
+                        ImagePackSource::User => false,
+                    })
+                    .cloned(),
+            );
+        }
+    }
+    Ok(ImagePacks { global, rooms })
+}
+
+async fn room_packs(room: &Room) -> Result<Vec<ImagePack>, Error> {
+    let events = room
+        .get_state_events(StateEventType::from(ROOM_PACKS))
+        .await
+        .map_err(|source| Error::from_sdk("reading image packs failed", source))?;
+    let by_state_key: BTreeMap<String, Raw<JsonObject>> = events
+        .into_iter()
+        .filter_map(|event| match event {
+            RawAnySyncOrStrippedState::Sync(event) => Some((
+                event.get_field::<String>("state_key").ok()??,
+                event.get_field::<Raw<JsonObject>>("content").ok()??,
+            )),
+            RawAnySyncOrStrippedState::Stripped(_) => None,
+        })
+        .collect();
+    Ok(by_state_key
+        .into_iter()
+        .filter_map(|(state_key, content)| {
+            let source = ImagePackSource::Room {
+                room_id: room.room_id().to_owned(),
+                state_key: state_key.clone(),
+            };
+            pack(
+                format!("room:{}:{state_key}", room.room_id()),
+                source,
+                &content,
+            )
+        })
+        .collect())
+}
+
+async fn account_data(client: &Client, event_type: &str) -> Result<Option<Raw<JsonObject>>, Error> {
+    Ok(client
+        .account()
+        .account_data_raw(GlobalAccountDataEventType::from(event_type))
+        .await
+        .map_err(|source| Error::from_sdk("reading image packs failed", source))?
+        .map(|content| content.cast_unchecked()))
+}
+
+/// one bad image costs that image, not its pack. a pack with nothing usable left is dropped
+fn pack(id: String, source: ImagePackSource, content: &Raw<JsonObject>) -> Option<ImagePack> {
+    let info = content
+        .get_field::<PackInfo>("pack")
+        .ok()
+        .flatten()
+        .unwrap_or_default();
+    let pack_usage = usage(&info.usage);
+    let images: Vec<PackImage> = content
+        .get_field::<BTreeMap<String, Raw<SourceImage>>>("images")
+        .ok()
+        .flatten()
+        .unwrap_or_default()
+        .into_iter()
+        .filter_map(|(shortcode, image)| {
+            let image = image.deserialize().ok()?;
+            if !image.url.is_valid() {
+                return None;
+            }
+            let info = image.info.as_ref();
+            Some(PackImage {
+                shortcode,
+                media_id: MediaId::plain(&image.url),
+                reaction_key: image.url.to_string(),
+                body: image.body,
+                mimetype: info.and_then(|info| info.mimetype.clone()),
+                width: info.and_then(|info| info.width).map(u64::from),
+                height: info.and_then(|info| info.height).map(u64::from),
+                usage: usage(&image.usage)
+                    .or_else(|| pack_usage.clone())
+                    .unwrap_or_else(|| vec![PackUsage::Emoticon, PackUsage::Sticker]),
+            })
+        })
+        .collect();
+    (!images.is_empty()).then(|| ImagePack {
+        id,
+        display_name: info.display_name,
+        avatar: info
+            .avatar_url
+            .filter(|uri| uri.is_valid())
+            .map(|uri| MediaId::plain(&uri)),
+        attribution: info.attribution,
+        source,
+        images,
+    })
+}
+
+fn usage(usage: &BTreeSet<SourceUsage>) -> Option<Vec<PackUsage>> {
+    let usage: Vec<PackUsage> = usage
+        .iter()
+        .filter_map(|usage| match usage {
+            SourceUsage::Emoticon => Some(PackUsage::Emoticon),
+            SourceUsage::Sticker => Some(PackUsage::Sticker),
+            _ => None,
+        })
+        .collect();
+    (!usage.is_empty()).then_some(usage)
+}

Modified file: src/normalise/mod.rs+2 additions@@ -1,8 +1,10 @@
+mod image_pack;
 mod media;
 mod member;
 mod room;
 mod timeline;
 
+pub(crate) use image_pack::{ENABLED_ROOMS, ROOM_PACKS, USER_PACK, image_packs};
 pub(crate) use media::{MediaRegistration, MediaScope, MediaSourceInfo};
 pub(crate) use member::member_summary;
 pub(crate) use room::notification_mode;

Modified file: src/transport.rs+8 additions−4 deletions@@ -27,9 +27,9 @@ use tokio::sync::{OnceCell, mpsc, oneshot, watch};
 
 use crate::{
     AccountCommand, AttachmentUpload, Command, CommandOutput, ConnectionState, CreateRoomOptions,
-    EncryptionCommand, EncryptionState, Error, ErrorKind, Event, EventRef, InteractiveAuth,
-    InteractiveAuthOutcome, LifecycleCommand, Media, MediaCommand, MediaId, MediaLocation,
-    MediaSize, MemberFilter, MemberSummary, MessageContent, MessageEditContent,
+    EncryptionCommand, EncryptionState, Error, ErrorKind, Event, EventRef, ImagePacks,
+    InteractiveAuth, InteractiveAuthOutcome, LifecycleCommand, Media, MediaCommand, MediaId,
+    MediaLocation, MediaSize, MemberFilter, MemberSummary, MessageContent, MessageEditContent,
     NotificationCommand, OwnDevice, PowerLevelChanges, Profile, PushNotification,
     PushNotificationEvent, PushNotificationStatus, RoomCommand, RoomListFilter,
     RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels, RoomSummary, SpaceChild,
@@ -37,7 +37,7 @@ use crate::{
     encryption::encryption_state,
     engine::{MediaService, StoreLock},
     model::{PushNotificationRoom, PushNotificationSender},
-    normalise::{member_summary, room_summaries, space_child},
+    normalise::{image_packs, member_summary, room_summaries, space_child},
 };
 
 #[derive(Clone)]
@@ -582,6 +582,10 @@ impl ClientHandle {
         room_summaries(&self.client, self.spaces().await).await
     }
 
+    pub async fn image_packs(&self) -> Result<ImagePacks, Error> {
+        image_packs(&self.client).await
+    }
+
     /// browse a space the way the server sees it, rooms you haven't joined included
     pub async fn space_hierarchy(&self, space_id: OwnedRoomId) -> SpaceHierarchy {
         SpaceHierarchy(SpaceRoomList::new(self.client.clone(), space_id).await)

Modified file: tests/state.rs+84 additions−2 deletions@@ -3,12 +3,15 @@ mod support;
 use std::time::Duration;
 
 use macltok::{
-    ConnectionState, Engine, EngineConfig, EngineError, Event, Retain, RoomMembership, Termination,
+    ConnectionState, Engine, EngineConfig, EngineError, Event, PackUsage, Retain, RoomMembership,
+    Termination,
 };
 use matrix_sdk::{
-    ruma::{api::client::sync::sync_events::v5, room_id},
+    ruma::{api::client::sync::sync_events::v5, events::AnySyncStateEvent, room_id, serde::Raw},
     test_utils::mocks::MatrixMockServer,
 };
+use matrix_sdk_test::JoinedRoomBuilder;
+use serde_json::json;
 use tokio::sync::watch;
 
 #[tokio::test]
@@ -151,3 +154,82 @@ async fn wait_for_connection(
     .await
     .expect("expected connection state did not arrive");
 }
+
+#[tokio::test]
+async fn image_packs_follow_room_state_and_account_data() {
+    let server = MatrixMockServer::new().await;
+    server
+        .mock_sliding_sync()
+        .ok(v5::Response::new("1".to_owned()))
+        .mount()
+        .await;
+    let client = server.client_builder().build().await;
+    let room_id = room_id!("!packs:localhost");
+    server.mock_room_state_encryption().plain().mount().await;
+    server.sync_joined_room(&client, room_id).await;
+    let config = EngineConfig::default().with_initial_history_limit(0);
+    let (engine, handle, mut events) = Engine::from_client(client.clone(), config).await.unwrap();
+    let running = tokio::spawn(engine.run());
+
+    let pack: Raw<AnySyncStateEvent> = serde_json::from_value(json!({
+        "type": "im.ponies.room_emotes",
+        "state_key": "cats",
+        "event_id": "$pack",
+        "sender": "@example:localhost",
+        "origin_server_ts": 1,
+        "content": {
+            "pack": { "display_name": "cats", "usage": ["emoticon"] },
+            "images": {
+                "blep": { "url": "mxc://localhost/blep" },
+                "broken": { "url": "not an mxc" },
+                "sticker": { "url": "mxc://localhost/sticker", "usage": ["sticker"] },
+            },
+        },
+    }))
+    .unwrap();
+    server
+        .mock_sync()
+        .ok_and_run(&client, |builder| {
+            builder.add_joined_room(JoinedRoomBuilder::new(room_id).add_state_event(pack));
+            builder.add_custom_global_account_data(json!({
+                "type": "im.ponies.emote_rooms",
+                "content": { "rooms": { "!packs:localhost": { "cats": {} } } },
+            }));
+        })
+        .await;
+    support::receive_until(&mut events, |event| {
+        matches!(event, Event::ImagePacksChanged)
+    })
+    .await;
+
+    let packs = handle.image_packs().await.unwrap();
+    let offered = packs.for_room(Some(room_id));
+    assert_eq!(offered.len(), 1, "an enabled room's pack was offered twice");
+    let pack = offered[0];
+    assert_eq!(pack.id, "room:!packs:localhost:cats");
+    assert_eq!(packs.global, vec![pack.clone()]);
+    let images: Vec<_> = pack
+        .images
+        .iter()
+        .map(|image| {
+            (
+                image.shortcode.as_str(),
+                image.reaction_key.as_str(),
+                image.usage.clone(),
+            )
+        })
+        .collect();
+    assert_eq!(
+        images,
+        [
+            ("blep", "mxc://localhost/blep", vec![PackUsage::Emoticon]),
+            (
+                "sticker",
+                "mxc://localhost/sticker",
+                vec![PackUsage::Sticker]
+            ),
+        ]
+    );
+
+    support::shut_down(&handle, &mut events, running).await;
+}

macltok

Commit 17e042fbc331

Commit `17e042fbc331`