macltok

public

Star0

Fork 0

Watch0

29 commits 3 branches 0 tags 0 releases3.0 MiB

Commit `5b21b919d1e6`

tarball

atagen <boss@atagen.co>2026-09-27 12:08 UTC

7 files changed+350-65

media: let a host locate media and stream it itself

ClientHandle::media buffers the whole file and checks its hash before answering, which a long video can't afford, least of all in wasm memory that never shrinks. media_source hands over the url and attachment keys media would have used, picked by the same choose_source, and access_token the bearer to fetch them with, so a host can serve Range requests and decrypt with AES-CTR on its own. Streaming decryption inside macltok was declined since a browser service worker is a separate context and every chunk would still cross wasm memory.

Modified file: README.md+6 additions@@ -47,6 +47,12 @@ with `TimelineDiff::try_apply_all`. snapshots are `imbl::Vector`s, re-exported
 as `macltok::imbl`, so a copy costs nothing. when a batch doesn't fit none of it
 lands. stop applying, call `resync`, and pick up from the next `TimelineReset`
 
+`media` answers with the whole file, decrypted and checked, which is wrong for a
+long video. `media_source` says where the same bytes live instead, the url and,
+for an encrypted file, its AES-CTR key, iv and hash. `access_token` is what to
+send with it. a host can then fetch and decrypt ranges itself, but the hash
+covers the whole ciphertext, so a range read plays bytes nobody checked
+
 with `sqlite` or `indexeddb`, a `SessionStore` holds one login's record and its
 store. `EncryptedSessionStore` seals the record with the store's passphrase, in a
 file you pick natively or an IndexedDB database beside the store in a browser.

Modified file: docs/design-decisions.md+7 additions@@ -175,6 +175,13 @@ checks the cached bytes against the file's hash and decrypts them if they match
 media ids are registered as timelines render and forgotten when their timeline
 closes, least recently seen first once the cap is hit
 
+streaming decryption inside macltok was declined. in a browser the service
+worker answering range requests is a separate context from the worker running
+macltok, and streaming through macltok would carry every chunk through wasm
+memory, the very thing the host streams to avoid. `media_source` hands over the
+url and keys `media` would have used, picked the same way, and `access_token`
+the rest. the byte cap doesn't apply, nothing is buffered
+
 uploads are commands that answer once the send queue has them. progress only
 goes out on a room's own send queue channel, so each room that uploads gets a
 forwarder, subscribed before the upload is queued and restarted for uploads the

Modified file: src/engine/media.rs+142 additions−44 deletions@@ -22,7 +22,7 @@ use matrix_sdk::{
             client::{authenticated_media, media},
         },
         events::room::{
-            MediaSource,
+            EncryptedFileHash, EncryptedFileHashAlgorithm, EncryptedFileInfo, MediaSource,
             message::{AddMentions, ReplyWithinThread},
         },
     },
@@ -33,7 +33,8 @@ use tokio::sync::Mutex;
 
 use super::executor::EngineBoxFuture;
 use crate::{
-    AttachmentUpload, Error, ErrorKind, Media, MediaId, MediaKind, MediaSize, TimelineId,
+    AttachmentEncryption, AttachmentUpload, Error, ErrorKind, Media, MediaFile, MediaId, MediaKind,
+    MediaLocation, MediaSize, TimelineId,
     normalise::{MediaRegistration, MediaScope},
     upload::prepare_upload,
 };
@@ -84,8 +85,59 @@ impl MediaService {
         media_id: &MediaId,
         size: MediaSize,
     ) -> Result<Media, Error> {
-        let media = self
-            .registry
+        let chosen = choose_source(self.registered(media_id).await?, size)?;
+        if chosen
+            .declared_size
+            .is_some_and(|size| size > self.max_bytes as u64)
+        {
+            return Err(too_large("media exceeds the configured byte limit"));
+        }
+        self.fetch_or_file(chosen.request, chosen.mimetype).await
+    }
+
+    pub(crate) async fn media_location(
+        &self,
+        media_id: &MediaId,
+        size: MediaSize,
+    ) -> Result<MediaLocation, Error> {
+        let media = self.registered(media_id).await?;
+        let chosen = choose_source(media.clone(), size)?;
+        if is_local(&chosen.request.source) {
+            return Ok(MediaLocation::Local);
+        }
+        if !matches!(chosen.request.format, MediaFormat::Thumbnail(_)) {
+            return Ok(MediaLocation::File(
+                media_file(
+                    &self.client,
+                    chosen.request,
+                    chosen.mimetype,
+                    chosen.declared_size,
+                )
+                .await?,
+            ));
+        }
+        let url = media_http_request(&self.client, &chosen.request)
+            .await?
+            .uri()
+            .to_string();
+        let fallback = MediaRequestParameters {
+            source: media.full.source,
+            format: MediaFormat::File,
+        };
+        Ok(MediaLocation::ServerThumbnail {
+            url,
+            fallback: media_file(
+                &self.client,
+                fallback,
+                media.full.mimetype,
+                media.full.declared_size,
+            )
+            .await?,
+        })
+    }
+
+    async fn registered(&self, media_id: &MediaId) -> Result<MediaRegistration, Error> {
+        self.registry
             .lock()
             .await
             .get(media_id)
@@ -95,46 +147,7 @@ impl MediaService {
                     ErrorKind::MediaUnknown,
                     "media is not registered in a retained timeline",
                 )
-            })?;
-        let (source, format, mimetype, declared_size) = match size {
-            MediaSize::Full => (
-                media.full.source,
-                MediaFormat::File,
-                media.full.mimetype,
-                media.full.declared_size,
-            ),
-            MediaSize::Thumbnail { width, height } => match media.thumbnail {
-                Some(thumbnail) => (
-                    thumbnail.source,
-                    MediaFormat::File,
-                    thumbnail.mimetype,
-                    thumbnail.declared_size,
-                ),
-                // the server can only resize what it can read
-                None if matches!(media.kind, MediaKind::Image | MediaKind::Video)
-                    && matches!(media.full.source, MediaSource::Plain(_)) =>
-                {
-                    // the server picks the thumbnail's type, so the original's would be a lie
-                    let format = if is_local(&media.full.source) {
-                        MediaFormat::File
-                    } else {
-                        thumbnail_format(width, height)
-                    };
-                    (media.full.source, format, None, None)
-                }
-                None => {
-                    return Err(Error::new(
-                        ErrorKind::MediaUnavailable,
-                        "the requested media size is unavailable",
-                    ));
-                }
-            },
-        };
-        if declared_size.is_some_and(|size| size > self.max_bytes as u64) {
-            return Err(too_large("media exceeds the configured byte limit"));
-        }
-        self.fetch_or_file(MediaRequestParameters { source, format }, mimetype)
-            .await
+            })
     }
 
     pub(crate) async fn mxc_media(&self, uri: &MxcUri, size: MediaSize) -> Result<Media, Error> {
@@ -214,6 +227,91 @@ impl MediaService {
     }
 }
 
+struct ChosenSource {
+    request: MediaRequestParameters,
+    mimetype: Option<String>,
+    declared_size: Option<u64>,
+}
+
+fn choose_source(media: MediaRegistration, size: MediaSize) -> Result<ChosenSource, Error> {
+    let (source, format, mimetype, declared_size) = match size {
+        MediaSize::Full => (
+            media.full.source,
+            MediaFormat::File,
+            media.full.mimetype,
+            media.full.declared_size,
+        ),
+        MediaSize::Thumbnail { width, height } => match media.thumbnail {
+            Some(thumbnail) => (
+                thumbnail.source,
+                MediaFormat::File,
+                thumbnail.mimetype,
+                thumbnail.declared_size,
+            ),
+            // the server can only resize what it can read
+            None if matches!(media.kind, MediaKind::Image | MediaKind::Video)
+                && matches!(media.full.source, MediaSource::Plain(_)) =>
+            {
+                // the server picks the thumbnail's type, so the original's would be a lie
+                let format = if is_local(&media.full.source) {
+                    MediaFormat::File
+                } else {
+                    thumbnail_format(width, height)
+                };
+                (media.full.source, format, None, None)
+            }
+            None => {
+                return Err(Error::new(
+                    ErrorKind::MediaUnavailable,
+                    "the requested media size is unavailable",
+                ));
+            }
+        },
+    };
+    Ok(ChosenSource {
+        request: MediaRequestParameters { source, format },
+        mimetype,
+        declared_size,
+    })
+}
+
+async fn media_file(
+    client: &Client,
+    request: MediaRequestParameters,
+    mimetype: Option<String>,
+    declared_size: Option<u64>,
+) -> Result<MediaFile, Error> {
+    let url = media_http_request(client, &request)
+        .await?
+        .uri()
+        .to_string();
+    let encryption = match &request.source {
+        MediaSource::Plain(_) => None,
+        MediaSource::Encrypted(file) => {
+            let (EncryptedFileInfo::V2(info), Some(EncryptedFileHash::Sha256(sha256))) = (
+                &file.info,
+                file.hashes.get(&EncryptedFileHashAlgorithm::Sha256),
+            ) else {
+                return Err(Error::new(
+                    ErrorKind::MediaUnavailable,
+                    "encrypted media is not v2 with a sha256 hash",
+                ));
+            };
+            Some(AttachmentEncryption {
+                key: info.k.encode(),
+                iv: info.iv.encode(),
+                sha256: sha256.encode(),
+            })
+        }
+    };
+    Ok(MediaFile {
+        url,
+        mimetype,
+        declared_size,
+        encryption,
+    })
+}
+
 fn thumbnail_format(width: u32, height: u32) -> MediaFormat {
     MediaFormat::Thumbnail(MediaThumbnailSettings::new(width.into(), height.into()))
 }

Modified file: src/lib.rs+15 additions−14 deletions@@ -42,20 +42,21 @@ pub use mirror::{
     TimelineState, Update, UploadFailure, UploadState,
 };
 pub use model::{
-    AccountCommand, AttachmentUpload, Change, Command, CommandOutput, ConnectionState,
-    CreateRoomOptions, EncryptionCommand, EncryptionState, EngineErrorKind, EngineFailure, Event,
-    EventRef, HistoryState, KeyBackupState, LatestEvent, LifecycleCommand, Media, MediaCommand,
-    MediaDescriptor, MediaId, MediaKind, MediaSize, MemberFilter, MemberMembership, MemberSummary,
-    MembershipChange, Message, MessageContent, MessageEditContent, MessageKind, MessageMentions,
-    NotificationCommand, OwnDevice, PowerLevelChanges, Profile, PushNotification,
-    PushNotificationEvent, PushNotificationRoom, PushNotificationSender, PushNotificationStatus,
-    Reaction, RecoveryState, ReplyDetails, ReplyReference, Retain, RichText, RoomCategory,
-    RoomCommand, RoomHero, RoomListFilter, RoomMatcher, RoomMembership, RoomNotificationMode,
-    RoomNotificationSettings, RoomPowerLevels, RoomPreset, RoomSummary, RoomVisibility, SendState,
-    Sender, SpaceChild, Termination, ThreadSummary, TimelineCommand, TimelineContent, TimelineDiff,
-    TimelineDiffError, TimelineEvent, TimelineId, TimelineItem, TimelineItemId, TypingState,
-    UploadDetails, UploadOutcome, UploadThumbnail, UtdCause, VerificationAction,
-    VerificationFailure, VerificationState, VerificationTarget,
+    AccountCommand, AttachmentEncryption, AttachmentUpload, Change, Command, CommandOutput,
+    ConnectionState, CreateRoomOptions, EncryptionCommand, EncryptionState, EngineErrorKind,
+    EngineFailure, Event, EventRef, HistoryState, KeyBackupState, LatestEvent, LifecycleCommand,
+    Media, MediaCommand, MediaDescriptor, MediaFile, MediaId, MediaKind, MediaLocation, MediaSize,
+    MemberFilter, MemberMembership, MemberSummary, MembershipChange, Message, MessageContent,
+    MessageEditContent, MessageKind, MessageMentions, NotificationCommand, OwnDevice,
+    PowerLevelChanges, Profile, PushNotification, PushNotificationEvent, PushNotificationRoom,
+    PushNotificationSender, PushNotificationStatus, Reaction, RecoveryState, ReplyDetails,
+    ReplyReference, Retain, RichText, RoomCategory, RoomCommand, RoomHero, RoomListFilter,
+    RoomMatcher, RoomMembership, RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels,
+    RoomPreset, RoomSummary, RoomVisibility, SendState, Sender, SpaceChild, Termination,
+    ThreadSummary, TimelineCommand, TimelineContent, TimelineDiff, TimelineDiffError,
+    TimelineEvent, TimelineId, TimelineItem, TimelineItemId, TypingState, UploadDetails,
+    UploadOutcome, UploadThumbnail, UtdCause, VerificationAction, VerificationFailure,
+    VerificationState, VerificationTarget,
 };
 pub use transport::{ClientHandle, EventStream, Pending, SpaceHierarchy};
 pub use uiaa::{

Modified file: src/model.rs+39 additions@@ -1681,6 +1681,45 @@ pub struct Media {
     pub mimetype: String,
 }
 
+/// where media lives, for a host that fetches and decrypts it itself
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub enum MediaLocation {
+    File(MediaFile),
+    /// a resize of plaintext, so never encrypted. plenty of servers refuse some, and `media`
+    /// then fetches `fallback` instead, so should you
+    ServerThumbnail {
+        url: String,
+        fallback: MediaFile,
+    },
+    /// still in the send queue with no url yet. `media` has the bytes
+    Local,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub struct MediaFile {
+    pub url: String,
+    pub mimetype: Option<String>,
+    pub declared_size: Option<u64>,
+    pub encryption: Option<AttachmentEncryption>,
+}
+
+/// AES-256-CTR, the only attachment cipher matrix has
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub struct AttachmentEncryption {
+    /// the JWK `k`, unpadded url-safe base64
+    pub key: String,
+    /// unpadded base64, the initial counter block
+    pub iv: String,
+    /// unpadded base64 SHA-256 of the whole ciphertext, so a partial read goes unchecked
+    pub sha256: String,
+}
+
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[non_exhaustive]

Modified file: src/transport.rs+22 additions−6 deletions@@ -28,12 +28,12 @@ use tokio::sync::{OnceCell, mpsc, oneshot, watch};
 use crate::{
     AccountCommand, AttachmentUpload, Command, CommandOutput, ConnectionState, CreateRoomOptions,
     EncryptionCommand, EncryptionState, Error, ErrorKind, Event, EventRef, InteractiveAuth,
-    InteractiveAuthOutcome, LifecycleCommand, Media, MediaCommand, MediaId, MediaSize,
-    MemberFilter, MemberSummary, MessageContent, MessageEditContent, NotificationCommand,
-    OwnDevice, PowerLevelChanges, Profile, PushNotification, PushNotificationEvent,
-    PushNotificationStatus, RoomCommand, RoomListFilter, RoomNotificationMode,
-    RoomNotificationSettings, RoomPowerLevels, RoomSummary, SpaceChild, TimelineCommand,
-    TimelineId, VerificationAction, VerificationTarget,
+    InteractiveAuthOutcome, LifecycleCommand, Media, MediaCommand, MediaId, MediaLocation,
+    MediaSize, MemberFilter, MemberSummary, MessageContent, MessageEditContent,
+    NotificationCommand, OwnDevice, PowerLevelChanges, Profile, PushNotification,
+    PushNotificationEvent, PushNotificationStatus, RoomCommand, RoomListFilter,
+    RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels, RoomSummary, SpaceChild,
+    TimelineCommand, TimelineId, VerificationAction, VerificationTarget,
     encryption::encryption_state,
     engine::{MediaService, StoreLock},
     model::{PushNotificationRoom, PushNotificationSender},
@@ -83,6 +83,12 @@ impl ClientHandle {
         crate::account::session_record(&self.client)
     }
 
+    /// the bearer token. it rotates on every refresh so ask again after a 401 rather than
+    /// keeping it, and keep it in memory and away from all but the homeserver's media endpoints
+    pub fn access_token(&self) -> Option<String> {
+        self.client.access_token()
+    }
+
     /// queue a command, waiting for room in the queue if there isn't any
     pub async fn send(&self, command: Command) -> Result<Pending, Error> {
         let (request, pending) = Request::new(command);
@@ -554,6 +560,16 @@ impl ClientHandle {
         self.media.timeline_media(media_id, size).await
     }
 
+    /// where `media` would fetch from, for a host streaming big files itself. the byte cap
+    /// doesn't apply since nothing is buffered
+    pub async fn media_source(
+        &self,
+        media_id: &MediaId,
+        size: MediaSize,
+    ) -> Result<MediaLocation, Error> {
+        self.media.media_location(media_id, size).await
+    }
+
     /// bytes for any plain mxc uri, avatars mostly. same byte cap as everything else
     pub async fn mxc_media(&self, uri: &MxcUri, size: MediaSize) -> Result<Media, Error> {
         self.media.mxc_media(uri, size).await

Modified file: tests/media.rs+119 additions−1 deletions@@ -4,7 +4,8 @@ use std::{sync::Arc, time::Duration};
 
 use macltok::{
     AttachmentUpload, ClientHandle, Engine, EngineConfig, EngineError, ErrorKind, Event,
-    EventStream, MediaId, MediaSize, UploadDetails, UploadOutcome, UploadThumbnail,
+    EventStream, MediaFile, MediaId, MediaLocation, MediaSize, UploadDetails, UploadOutcome,
+    UploadThumbnail,
 };
 use matrix_sdk::{
     media::{MediaFormat, MediaRequestParameters},
@@ -16,6 +17,7 @@ use matrix_sdk::{
             message::{
                 AudioMessageEventContent, FileInfo, FileMessageEventContent,
                 ImageMessageEventContent, MessageType, RoomMessageEventContent,
+                VideoMessageEventContent,
             },
         },
         media::Method,
@@ -282,6 +284,89 @@ async fn audio_thumbnail_fetch_fails_with_media_unavailable() {
 }
 
 #[tokio::test]
+async fn media_source_locates_what_media_would_fetch() {
+    let keys =
+        matrix_sdk_crypto::AttachmentEncryptor::new(&mut std::io::Cursor::new(b"secret")).finish();
+    let file = EncryptedFile::new(
+        mxc_uri!("mxc://localhost/secret").to_owned(),
+        keys.encryption_info,
+        keys.hashes,
+    );
+    let wire = serde_json::to_value(&file).unwrap();
+    let mut harness = start(RoomMessageEventContent::new(MessageType::File(
+        FileMessageEventContent::encrypted("secret.bin".to_owned(), file),
+    )))
+    .await;
+
+    let MediaLocation::File(MediaFile {
+        url,
+        encryption: Some(encryption),
+        ..
+    }) = harness
+        .handle
+        .media_source(&harness.media_id, MediaSize::Full)
+        .await
+        .unwrap()
+    else {
+        panic!("an encrypted file should locate as one");
+    };
+    assert_eq!(
+        url,
+        format!(
+            "{}/_matrix/client/v1/media/download/localhost/secret",
+            harness.server.uri()
+        )
+    );
+    assert_eq!(encryption.key, wire["key"]["k"]);
+    assert_eq!(encryption.iv, wire["iv"]);
+    assert_eq!(encryption.sha256, wire["hashes"]["sha256"]);
+
+    let video = sync_media(
+        &mut harness,
+        "video.mp4",
+        RoomMessageEventContent::new(MessageType::Video(VideoMessageEventContent::plain(
+            "video.mp4".to_owned(),
+            mxc_uri!("mxc://localhost/video").to_owned(),
+        ))),
+    )
+    .await;
+    let MediaLocation::ServerThumbnail { url, fallback } = harness
+        .handle
+        .media_source(&video, thumbnail_size())
+        .await
+        .unwrap()
+    else {
+        panic!("a video with no thumbnail of its own should ask the server for one");
+    };
+    let url = matrix_sdk::reqwest::Url::parse(&url).unwrap();
+    assert_eq!(
+        url.path(),
+        "/_matrix/client/v1/media/thumbnail/localhost/video"
+    );
+    let query: Vec<_> = url.query_pairs().into_owned().collect();
+    for pair in [("width", "800"), ("height", "800"), ("method", "scale")] {
+        assert!(query.contains(&(pair.0.to_owned(), pair.1.to_owned())));
+    }
+    assert_eq!(
+        MediaLocation::File(fallback),
+        harness
+            .handle
+            .media_source(&video, MediaSize::Full)
+            .await
+            .unwrap()
+    );
+
+    let error = harness
+        .handle
+        .media_source(&MediaId::from("made up".to_owned()), MediaSize::Full)
+        .await
+        .unwrap_err();
+    assert_eq!(error.kind, ErrorKind::MediaUnknown);
+
+    support::shut_down(&harness.handle, &mut harness.events, harness.running).await;
+}
+
+#[tokio::test]
 async fn media_upload_is_queued_and_sent() {
     let mut harness = start(plain_image()).await;
     harness
@@ -373,6 +458,14 @@ async fn queued_offline_attachment_exposes_fetchable_local_media() {
         .await
         .unwrap();
     assert_eq!(&*fetched.bytes, &original[..]);
+    assert_eq!(
+        harness
+            .handle
+            .media_source(&media_id, MediaSize::Full)
+            .await
+            .unwrap(),
+        MediaLocation::Local
+    );
 
     support::shut_down(&harness.handle, &mut harness.events, harness.running).await;
 }
@@ -796,6 +889,31 @@ async fn start_with_config(content: RoomMessageEventContent, config: EngineConfi
     }
 }
 
+async fn sync_media(
+    harness: &mut MediaHarness,
+    filename: &str,
+    content: RoomMessageEventContent,
+) -> MediaId {
+    harness
+        .server
+        .sync_room(
+            &harness.client,
+            JoinedRoomBuilder::new(room_id!("!media:localhost"))
+                .add_timeline_event(EventFactory::new().event(content).sender(&ALICE)),
+        )
+        .await;
+    let media_id = |event: &Event| {
+        support::event_messages(event)
+            .into_iter()
+            .filter_map(support::media_of)
+            .find(|media| media.filename == filename)
+            .map(|media| media.id.clone())
+    };
+    let event =
+        support::receive_until(&mut harness.events, |event| media_id(event).is_some()).await;
+    media_id(&event).unwrap()
+}
+
 async fn mount_downloads(server: &MatrixMockServer) {
     server.mock_media_download().ok_image().mount().await;
     server.mock_authed_media_download().ok_image().mount().await;

macltok

Commit 5b21b919d1e6

Commit `5b21b919d1e6`