macltok

public

Star0

Watch0

4 commits 3 branches 0 tags 0 releases3.0 MiB

use std::{
    borrow::Cow,
    collections::{HashMap, VecDeque},
    io::Read as _,
    panic::AssertUnwindSafe,
    sync::Arc,
};

use futures_util::{
    FutureExt, StreamExt,
    future::{BoxFuture, Shared, WeakShared},
};
use matrix_sdk::{
    Client, Room,
    media::{MediaFormat, MediaRequestParameters, MediaThumbnailSettings},
    room::reply::{EnforceThread, Reply as SdkReply},
    ruma::{
        api::{
            Metadata, OutgoingRequest,
            auth_scheme::SendAccessToken,
            client::{authenticated_media, media},
        },
        events::room::{
            MediaSource,
            message::{AddMentions, ReplyWithinThread},
        },
    },
};
use matrix_sdk_crypto::{AttachmentDecryptor, MediaEncryptionInfo};
use ruma::MxcUri;
use tokio::sync::Mutex;

use crate::{
    AttachmentUpload, Error, ErrorKind, Media, MediaId, MediaKind, MediaSize, TimelineId,
    normalise::{MediaRegistration, MediaScope},
    upload::prepare_upload,
};

/// the uploader's thumbnail is used when there is one, this is only what we ask the server for
const TIMELINE_THUMBNAIL: u32 = 800;

type FlightFuture = BoxFuture<'static, Result<Media, Error>>;
type Flight = Shared<FlightFuture>;

/// owns everything needed to turn a media id or an mxc uri into bytes, so the
/// handle can answer media queries without a trip through the engine
pub(crate) struct MediaService {
    client: Client,
    registry: Mutex<MediaRegistry>,
    flights: Mutex<HashMap<String, WeakShared<FlightFuture>>>,
    max_bytes: usize,
}

impl MediaService {
    pub(crate) fn new(client: Client, max_bytes: usize, max_sources: usize) -> Self {
        Self {
            client,
            registry: Mutex::new(MediaRegistry::new(max_sources)),
            flights: Mutex::new(HashMap::new()),
            max_bytes,
        }
    }

    pub(crate) async fn register(&self, media: Vec<MediaRegistration>) {
        if !media.is_empty() {
            self.registry.lock().await.insert_all(media);
        }
    }

    pub(crate) async fn release(&self, scope: &MediaScope) {
        self.registry.lock().await.release(scope);
    }

    pub(crate) async fn clear(&self) {
        self.registry.lock().await.clear();
    }

    pub(crate) async fn timeline_media(
        &self,
        media_id: &MediaId,
        size: MediaSize,
    ) -> Result<Media, Error> {
        let media = self
            .registry
            .lock()
            .await
            .get(media_id)
            .cloned()
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::MediaUnknown,
                    "media is not registered in a retained timeline",
                )
            })?;
        let (source, format, mimetype, declared_size) = match size {
            MediaSize::Full => (
                media.full.source,
                MediaFormat::File,
                media.full.mimetype,
                media.full.declared_size,
            ),
            MediaSize::Thumbnail { .. } => match media.thumbnail {
                Some(thumbnail) => (
                    thumbnail.source,
                    MediaFormat::File,
                    thumbnail.mimetype,
                    thumbnail.declared_size,
                ),
                // the server can only resize what it can read
                None if matches!(media.kind, MediaKind::Image | MediaKind::Video)
                    && matches!(media.full.source, MediaSource::Plain(_)) =>
                {
                    (
                        media.full.source,
                        thumbnail_format(TIMELINE_THUMBNAIL, TIMELINE_THUMBNAIL),
                        media.full.mimetype,
                        None,
                    )
                }
                None => {
                    return Err(Error::new(
                        ErrorKind::MediaUnavailable,
                        "the requested media size is unavailable",
                    ));
                }
            },
        };
        if declared_size.is_some_and(|size| size > self.max_bytes as u64) {
            return Err(too_large("media exceeds the configured byte limit"));
        }
        self.fetch(MediaRequestParameters { source, format }, mimetype)
            .await
    }

    pub(crate) async fn mxc_media(&self, uri: &MxcUri, size: MediaSize) -> Result<Media, Error> {
        if !uri.is_valid() {
            return Err(Error::new(ErrorKind::Invalid, "mxc uri is malformed"));
        }
        let format = match size {
            MediaSize::Full => MediaFormat::File,
            MediaSize::Thumbnail { width, height } => thumbnail_format(width, height),
        };
        self.fetch(
            MediaRequestParameters {
                source: MediaSource::Plain(uri.to_owned()),
                format,
            },
            None,
        )
        .await
    }

    async fn fetch(
        &self,
        request: MediaRequestParameters,
        mimetype: Option<String>,
    ) -> Result<Media, Error> {
        let key = flight_key(&request);
        let flight = {
            let mut flights = self.flights.lock().await;
            flights.retain(|_, flight| flight.upgrade().is_some());
            match flights.get(&key).and_then(WeakShared::upgrade) {
                Some(flight) => flight,
                None => {
                    let flight: Flight = AssertUnwindSafe(fetch(
                        self.client.clone(),
                        request,
                        mimetype,
                        self.max_bytes,
                    ))
                    .catch_unwind()
                    .map(|result| {
                        result.unwrap_or_else(|_| {
                            Err(Error::new(ErrorKind::Other, "media fetch panicked"))
                        })
                    })
                    .boxed()
                    .shared();
                    if let Some(weak) = flight.downgrade() {
                        flights.insert(key, weak);
                    }
                    flight
                }
            }
        };
        flight.await
    }
}

fn thumbnail_format(width: u32, height: u32) -> MediaFormat {
    MediaFormat::Thumbnail(MediaThumbnailSettings::new(width.into(), height.into()))
}

fn flight_key(request: &MediaRequestParameters) -> String {
    let uri = match &request.source {
        MediaSource::Plain(uri) => uri.as_str(),
        MediaSource::Encrypted(file) => file.url.as_str(),
    };
    match &request.format {
        MediaFormat::File => uri.to_owned(),
        MediaFormat::Thumbnail(settings) => format!(
            "{uri}|{}|{}x{}|{}",
            settings.method, settings.width, settings.height, settings.animated
        ),
    }
}

fn too_large(message: &'static str) -> Error {
    Error::new(ErrorKind::MediaTooLarge, message)
}

async fn fetch(
    client: Client,
    request: MediaRequestParameters,
    mimetype: Option<String>,
    max_bytes: usize,
) -> Result<Media, Error> {
    if let Some(cached) = cached_media(&client, &request).await? {
        if cached.len() > max_bytes {
            if let Ok(store) = client.media_store().lock().await {
                let _ = store.remove_media_content(&request).await;
            }
            return Err(too_large("cached media exceeds the configured byte limit"));
        }
        return Ok(media(cached, mimetype));
    }
    let (bytes, served_mimetype) = download_bounded(&client, &request, max_bytes).await?;
    // the sdk caches plaintext under the encrypted key, so we do too or nobody can read anything
    let bytes = maybe_decrypt(bytes, &request, max_bytes)?;
    if let Ok(store) = client.media_store().lock().await {
        let _ = store
            .add_media_content(
                &request,
                bytes.clone(),
                matrix_sdk::media::store::IgnoreMediaRetentionPolicy::No,
            )
            .await;
    }
    Ok(media(bytes, mimetype.or(served_mimetype)))
}

fn media(bytes: Vec<u8>, mimetype: Option<String>) -> Media {
    let mimetype = mimetype
        .or_else(|| sniff_image(&bytes).map(str::to_owned))
        .unwrap_or_else(|| "application/octet-stream".to_owned());
    Media {
        bytes: Arc::from(bytes),
        mimetype,
    }
}

/// the cache forgets content types, and avatars never declare one. images are the common case
fn sniff_image(bytes: &[u8]) -> Option<&'static str> {
    match bytes {
        [0x89, b'P', b'N', b'G', ..] => Some("image/png"),
        [0xff, 0xd8, 0xff, ..] => Some("image/jpeg"),
        [b'G', b'I', b'F', b'8', ..] => Some("image/gif"),
        [
            b'R',
            b'I',
            b'F',
            b'F',
            _,
            _,
            _,
            _,
            b'W',
            b'E',
            b'B',
            b'P',
            ..,
        ] => Some("image/webp"),
        _ => None,
    }
}

async fn cached_media(
    client: &Client,
    request: &MediaRequestParameters,
) -> Result<Option<Vec<u8>>, Error> {
    fn cache_failed(source: impl std::error::Error + Send + Sync + 'static) -> Error {
        Error::with_source(ErrorKind::Store, "reading media cache failed", source)
    }
    client
        .media_store()
        .lock()
        .await
        .map_err(cache_failed)?
        .get_media_content(request)
        .await
        .map_err(cache_failed)
}

fn maybe_decrypt(
    bytes: Vec<u8>,
    request: &MediaRequestParameters,
    max_bytes: usize,
) -> Result<Vec<u8>, Error> {
    let MediaSource::Encrypted(file) = &request.source else {
        return Ok(bytes);
    };
    fn decrypt_failed(source: impl std::error::Error + Send + Sync + 'static) -> Error {
        Error::with_source(
            ErrorKind::MediaUnavailable,
            "media decryption failed",
            source,
        )
    }
    let info: MediaEncryptionInfo = (**file).clone().into();
    let mut cursor = std::io::Cursor::new(bytes);
    let mut decryptor = AttachmentDecryptor::new(&mut cursor, info).map_err(decrypt_failed)?;
    let mut decrypted = Vec::new();
    let mut chunk = [0_u8; 8192];
    loop {
        let read = decryptor.read(&mut chunk).map_err(decrypt_failed)?;
        if read == 0 {
            return Ok(decrypted);
        }
        if read > max_bytes.saturating_sub(decrypted.len()) {
            return Err(too_large(
                "decrypted media exceeds the configured byte limit",
            ));
        }
        decrypted.extend_from_slice(&chunk[..read]);
    }
}

/// the sdk's own download has no size cap and no timeout, so we do it by hand
async fn download_bounded(
    client: &Client,
    request: &MediaRequestParameters,
    max_bytes: usize,
) -> Result<(Vec<u8>, Option<String>), Error> {
    let uri = match &request.source {
        MediaSource::Plain(uri) => uri,
        MediaSource::Encrypted(file) => &file.url,
    };
    if uri
        .server_name()
        .is_ok_and(|server| server.as_str() == "send-queue.localhost")
    {
        return Err(Error::new(
            ErrorKind::MediaUnavailable,
            "local media is not in the media cache",
        ));
    }
    let network_failed =
        |source| Error::with_source(ErrorKind::Network, "media download failed", source);
    let mut refreshed = false;
    let response = loop {
        let http_request = media_http_request(client, request).await?;
        let (parts, _) = http_request.into_parts();
        let mut builder = client.http_client().get(parts.uri.to_string());
        for (name, value) in &parts.headers {
            builder = builder.header(name.as_str(), value.as_bytes());
        }
        let response = builder.send().await.map_err(network_failed)?;
        if response.status() != matrix_sdk::reqwest::StatusCode::UNAUTHORIZED {
            break response;
        }
        if refreshed
            || client
                .session_tokens()
                .is_none_or(|tokens| tokens.refresh_token.is_none())
        {
            return Err(Error::new(
                ErrorKind::Unauthenticated,
                "media download was not authorized",
            ));
        }
        client.refresh_access_token().await.map_err(|source| {
            Error::from_refresh_token("refreshing media access token failed", source)
        })?;
        refreshed = true;
    };
    let status = response.status();
    if !status.is_success() {
        return Err(Error::new(
            ErrorKind::MediaUnavailable,
            format!("media download failed with status {status}"),
        ));
    }
    if response
        .content_length()
        .is_some_and(|length| length > max_bytes as u64)
    {
        return Err(too_large("media exceeds the configured byte limit"));
    }
    let mimetype = response
        .headers()
        .get(http::header::CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .map(str::to_owned);
    let mut stream = response.bytes_stream();
    let mut bytes = Vec::new();
    while let Some(chunk) = stream.next().await {
        let chunk = chunk.map_err(network_failed)?;
        if chunk.len() > max_bytes.saturating_sub(bytes.len()) {
            return Err(too_large(
                "downloaded media exceeds the configured byte limit",
            ));
        }
        bytes.extend_from_slice(&chunk);
    }
    Ok((bytes, mimetype))
}

async fn media_http_request(
    client: &Client,
    request: &MediaRequestParameters,
) -> Result<http::Request<Vec<u8>>, Error> {
    let versions = client.supported_versions().await.map_err(|source| {
        Error::with_source(
            ErrorKind::Network,
            "reading supported server versions failed",
            source,
        )
    })?;
    let authenticated =
        authenticated_media::get_content::v1::Request::PATH_BUILDER.is_supported(&versions);
    let base = client.homeserver().to_string();
    let token = client.access_token();
    let invalid_uri =
        |source| Error::with_source(ErrorKind::Invalid, "media URI is invalid", source);
    let (uri, thumbnail) = match (&request.source, &request.format) {
        (MediaSource::Encrypted(file), _) => (&file.url, None),
        (MediaSource::Plain(uri), MediaFormat::File) => (uri, None),
        (MediaSource::Plain(uri), MediaFormat::Thumbnail(settings)) => (uri, Some(settings)),
    };
    // each endpoint has its own auth input type, so no generic helper can take them all
    macro_rules! into_http {
        ($endpoint:expr, $token:expr) => {
            $endpoint
                .try_into_http_request::<Vec<u8>>(&base, $token, Cow::Borrowed(&versions))
                .map_err(|source| {
                    Error::with_source(ErrorKind::Other, "media request is invalid", source)
                })?
        };
    }
    let authed = SendAccessToken::IfRequired(token.as_deref().unwrap_or(""));
    #[allow(deprecated)]
    let mut http_request = match (thumbnail, authenticated) {
        (None, true) => into_http!(
            authenticated_media::get_content::v1::Request::from_uri(uri).map_err(invalid_uri)?,
            authed
        ),
        (None, false) => into_http!(
            media::get_content::v3::Request::from_url(uri).map_err(invalid_uri)?,
            SendAccessToken::None
        ),
        (Some(settings), true) => {
            let mut endpoint = authenticated_media::get_content_thumbnail::v1::Request::from_uri(
                uri,
                settings.width,
                settings.height,
            )
            .map_err(invalid_uri)?;
            endpoint.method = Some(settings.method.clone());
            endpoint.animated = Some(settings.animated);
            into_http!(endpoint, authed)
        }
        (Some(settings), false) => {
            let mut endpoint = media::get_content_thumbnail::v3::Request::from_url(
                uri,
                settings.width,
                settings.height,
            )
            .map_err(invalid_uri)?;
            endpoint.method = Some(settings.method.clone());
            endpoint.animated = Some(settings.animated);
            into_http!(endpoint, SendAccessToken::None)
        }
    };
    // legacy endpoints don't ask for a token but plenty of servers want one anyway
    if let Some(token) = token.as_deref() {
        let value = format!("Bearer {token}").parse().map_err(|_| {
            Error::new(ErrorKind::Unauthenticated, "stored access token is invalid")
        })?;
        http_request
            .headers_mut()
            .insert(http::header::AUTHORIZATION, value);
    }
    Ok(http_request)
}

pub(crate) async fn queue_upload(
    room: Room,
    upload: AttachmentUpload,
    max_bytes: usize,
) -> Result<(), Error> {
    let (upload, mime, mut config) = prepare_upload(upload, max_bytes)?;
    if let TimelineId::Thread { root_event_id, .. } = &upload.timeline_id {
        config.reply = Some(SdkReply {
            event_id: root_event_id.clone(),
            enforce_thread: EnforceThread::Threaded(ReplyWithinThread::No),
            add_mentions: AddMentions::No,
        });
    }
    room.send_queue()
        .send_attachment(upload.filename, mime, upload.bytes, config)
        .await
        .map(drop)
        .map_err(|source| {
            Error::with_source(
                ErrorKind::MediaUnavailable,
                "media upload could not be queued",
                source,
            )
        })
}

/// least recently seen goes first. stale order slots are skipped by stamp rather than hunted down
struct MediaRegistry {
    entries: HashMap<MediaId, (MediaRegistration, u64)>,
    order: VecDeque<(MediaId, u64)>,
    next_stamp: u64,
    limit: usize,
}

impl MediaRegistry {
    fn new(limit: usize) -> Self {
        Self {
            entries: HashMap::new(),
            order: VecDeque::new(),
            next_stamp: 0,
            limit,
        }
    }

    fn get(&self, id: &MediaId) -> Option<&MediaRegistration> {
        self.entries.get(id).map(|(media, _)| media)
    }

    fn insert_all(&mut self, media: Vec<MediaRegistration>) {
        for media in media {
            let stamp = self.next_stamp;
            self.next_stamp += 1;
            self.order.push_back((media.id.clone(), stamp));
            self.entries.insert(media.id.clone(), (media, stamp));
        }
        while self.entries.len() > self.limit {
            let Some((id, stamp)) = self.order.pop_front() else {
                break;
            };
            if self.is_current(&id, stamp) {
                self.entries.remove(&id);
            }
        }
        if self.order.len() > self.limit.saturating_mul(4) {
            let entries = &self.entries;
            self.order
                .retain(|(id, stamp)| entries.get(id).is_some_and(|(_, current)| current == stamp));
        }
    }

    fn is_current(&self, id: &MediaId, stamp: u64) -> bool {
        self.entries
            .get(id)
            .is_some_and(|(_, current)| *current == stamp)
    }

    fn release(&mut self, scope: &MediaScope) {
        self.entries.retain(|id, _| !scope.owns(id));
        let entries = &self.entries;
        self.order.retain(|(id, _)| entries.contains_key(id));
    }

    fn clear(&mut self) {
        self.entries.clear();
        self.order.clear();
    }
}

1	use std::{
2	borrow::Cow,
3	collections::{HashMap, VecDeque},
4	io::Read as _,
5	panic::AssertUnwindSafe,
6	sync::Arc,
7	};
8
9	use futures_util::{
10	FutureExt, StreamExt,
11	future::{BoxFuture, Shared, WeakShared},
12	};
13	use matrix_sdk::{
14	Client, Room,
15	media::{MediaFormat, MediaRequestParameters, MediaThumbnailSettings},
16	room::reply::{EnforceThread, Reply as SdkReply},
17	ruma::{
18	api::{
19	Metadata, OutgoingRequest,
20	auth_scheme::SendAccessToken,
21	client::{authenticated_media, media},
22	},
23	events::room::{
24	MediaSource,
25	message::{AddMentions, ReplyWithinThread},
26	},
27	},
28	};
29	use matrix_sdk_crypto::{AttachmentDecryptor, MediaEncryptionInfo};
30	use ruma::MxcUri;
31	use tokio::sync::Mutex;
32
33	use crate::{
34	AttachmentUpload, Error, ErrorKind, Media, MediaId, MediaKind, MediaSize, TimelineId,
35	normalise::{MediaRegistration, MediaScope},
36	upload::prepare_upload,
37	};
38
39	/// the uploader's thumbnail is used when there is one, this is only what we ask the server for
40	const TIMELINE_THUMBNAIL: u32 = 800;
41
42	type FlightFuture = BoxFuture<'static, Result<Media, Error>>;
43	type Flight = Shared<FlightFuture>;
44
45	/// owns everything needed to turn a media id or an mxc uri into bytes, so the
46	/// handle can answer media queries without a trip through the engine
47	pub(crate) struct MediaService {
48	client: Client,
49	registry: Mutex<MediaRegistry>,
50	flights: Mutex<HashMap<String, WeakShared<FlightFuture>>>,
51	max_bytes: usize,
52	}
53
54	impl MediaService {
55	pub(crate) fn new(client: Client, max_bytes: usize, max_sources: usize) -> Self {
56	Self {
57	client,
58	registry: Mutex::new(MediaRegistry::new(max_sources)),
59	flights: Mutex::new(HashMap::new()),
60	max_bytes,
61	}
62	}
63
64	pub(crate) async fn register(&self, media: Vec<MediaRegistration>) {
65	if !media.is_empty() {
66	self.registry.lock().await.insert_all(media);
67	}
68	}
69
70	pub(crate) async fn release(&self, scope: &MediaScope) {
71	self.registry.lock().await.release(scope);
72	}
73
74	pub(crate) async fn clear(&self) {
75	self.registry.lock().await.clear();
76	}
77
78	pub(crate) async fn timeline_media(
79	&self,
80	media_id: &MediaId,
81	size: MediaSize,
82	) -> Result<Media, Error> {
83	let media = self
84	.registry
85	.lock()
86	.await
87	.get(media_id)
88	.cloned()
89	.ok_or_else(\|\| {
90	Error::new(
91	ErrorKind::MediaUnknown,
92	"media is not registered in a retained timeline",
93	)
94	})?;
95	let (source, format, mimetype, declared_size) = match size {
96	MediaSize::Full => (
97	media.full.source,
98	MediaFormat::File,
99	media.full.mimetype,
100	media.full.declared_size,
101	),
102	MediaSize::Thumbnail { .. } => match media.thumbnail {
103	Some(thumbnail) => (
104	thumbnail.source,
105	MediaFormat::File,
106	thumbnail.mimetype,
107	thumbnail.declared_size,
108	),
109	// the server can only resize what it can read
110	None if matches!(media.kind, MediaKind::Image \| MediaKind::Video)
111	&& matches!(media.full.source, MediaSource::Plain(_)) =>
112	{
113	(
114	media.full.source,
115	thumbnail_format(TIMELINE_THUMBNAIL, TIMELINE_THUMBNAIL),
116	media.full.mimetype,
117	None,
118	)
119	}
120	None => {
121	return Err(Error::new(
122	ErrorKind::MediaUnavailable,
123	"the requested media size is unavailable",
124	));
125	}
126	},
127	};
128	if declared_size.is_some_and(\|size\| size > self.max_bytes as u64) {
129	return Err(too_large("media exceeds the configured byte limit"));
130	}
131	self.fetch(MediaRequestParameters { source, format }, mimetype)
132	.await
133	}
134
135	pub(crate) async fn mxc_media(&self, uri: &MxcUri, size: MediaSize) -> Result<Media, Error> {
136	if !uri.is_valid() {
137	return Err(Error::new(ErrorKind::Invalid, "mxc uri is malformed"));
138	}
139	let format = match size {
140	MediaSize::Full => MediaFormat::File,
141	MediaSize::Thumbnail { width, height } => thumbnail_format(width, height),
142	};
143	self.fetch(
144	MediaRequestParameters {
145	source: MediaSource::Plain(uri.to_owned()),
146	format,
147	},
148	None,
149	)
150	.await
151	}
152
153	async fn fetch(
154	&self,
155	request: MediaRequestParameters,
156	mimetype: Option<String>,
157	) -> Result<Media, Error> {
158	let key = flight_key(&request);
159	let flight = {
160	let mut flights = self.flights.lock().await;
161	flights.retain(\|_, flight\| flight.upgrade().is_some());
162	match flights.get(&key).and_then(WeakShared::upgrade) {
163	Some(flight) => flight,
164	None => {
165	let flight: Flight = AssertUnwindSafe(fetch(
166	self.client.clone(),
167	request,
168	mimetype,
169	self.max_bytes,
170	))
171	.catch_unwind()
172	.map(\|result\| {
173	result.unwrap_or_else(\|_\| {
174	Err(Error::new(ErrorKind::Other, "media fetch panicked"))
175	})
176	})
177	.boxed()
178	.shared();
179	if let Some(weak) = flight.downgrade() {
180	flights.insert(key, weak);
181	}
182	flight
183	}
184	}
185	};
186	flight.await
187	}
188	}
189
190	fn thumbnail_format(width: u32, height: u32) -> MediaFormat {
191	MediaFormat::Thumbnail(MediaThumbnailSettings::new(width.into(), height.into()))
192	}
193
194	fn flight_key(request: &MediaRequestParameters) -> String {
195	let uri = match &request.source {
196	MediaSource::Plain(uri) => uri.as_str(),
197	MediaSource::Encrypted(file) => file.url.as_str(),
198	};
199	match &request.format {
200	MediaFormat::File => uri.to_owned(),
201	MediaFormat::Thumbnail(settings) => format!(
202	"{uri}\|{}\|{}x{}\|{}",
203	settings.method, settings.width, settings.height, settings.animated
204	),
205	}
206	}
207
208	fn too_large(message: &'static str) -> Error {
209	Error::new(ErrorKind::MediaTooLarge, message)
210	}
211
212	async fn fetch(
213	client: Client,
214	request: MediaRequestParameters,
215	mimetype: Option<String>,
216	max_bytes: usize,
217	) -> Result<Media, Error> {
218	if let Some(cached) = cached_media(&client, &request).await? {
219	if cached.len() > max_bytes {
220	if let Ok(store) = client.media_store().lock().await {
221	let _ = store.remove_media_content(&request).await;
222	}
223	return Err(too_large("cached media exceeds the configured byte limit"));
224	}
225	return Ok(media(cached, mimetype));
226	}
227	let (bytes, served_mimetype) = download_bounded(&client, &request, max_bytes).await?;
228	// the sdk caches plaintext under the encrypted key, so we do too or nobody can read anything
229	let bytes = maybe_decrypt(bytes, &request, max_bytes)?;
230	if let Ok(store) = client.media_store().lock().await {
231	let _ = store
232	.add_media_content(
233	&request,
234	bytes.clone(),
235	matrix_sdk::media::store::IgnoreMediaRetentionPolicy::No,
236	)
237	.await;
238	}
239	Ok(media(bytes, mimetype.or(served_mimetype)))
240	}
241
242	fn media(bytes: Vec<u8>, mimetype: Option<String>) -> Media {
243	let mimetype = mimetype
244	.or_else(\|\| sniff_image(&bytes).map(str::to_owned))
245	.unwrap_or_else(\|\| "application/octet-stream".to_owned());
246	Media {
247	bytes: Arc::from(bytes),
248	mimetype,
249	}
250	}
251
252	/// the cache forgets content types, and avatars never declare one. images are the common case
253	fn sniff_image(bytes: &[u8]) -> Option<&'static str> {
254	match bytes {
255	[0x89, b'P', b'N', b'G', ..] => Some("image/png"),
256	[0xff, 0xd8, 0xff, ..] => Some("image/jpeg"),
257	[b'G', b'I', b'F', b'8', ..] => Some("image/gif"),
258	[
259	b'R',
260	b'I',
261	b'F',
262	b'F',
263	_,
264	_,
265	_,
266	_,
267	b'W',
268	b'E',
269	b'B',
270	b'P',
271	..,
272	] => Some("image/webp"),
273	_ => None,
274	}
275	}
276
277	async fn cached_media(
278	client: &Client,
279	request: &MediaRequestParameters,
280	) -> Result<Option<Vec<u8>>, Error> {
281	fn cache_failed(source: impl std::error::Error + Send + Sync + 'static) -> Error {
282	Error::with_source(ErrorKind::Store, "reading media cache failed", source)
283	}
284	client
285	.media_store()
286	.lock()
287	.await
288	.map_err(cache_failed)?
289	.get_media_content(request)
290	.await
291	.map_err(cache_failed)
292	}
293
294	fn maybe_decrypt(
295	bytes: Vec<u8>,
296	request: &MediaRequestParameters,
297	max_bytes: usize,
298	) -> Result<Vec<u8>, Error> {
299	let MediaSource::Encrypted(file) = &request.source else {
300	return Ok(bytes);
301	};
302	fn decrypt_failed(source: impl std::error::Error + Send + Sync + 'static) -> Error {
303	Error::with_source(
304	ErrorKind::MediaUnavailable,
305	"media decryption failed",
306	source,
307	)
308	}
309	let info: MediaEncryptionInfo = (**file).clone().into();
310	let mut cursor = std::io::Cursor::new(bytes);
311	let mut decryptor = AttachmentDecryptor::new(&mut cursor, info).map_err(decrypt_failed)?;
312	let mut decrypted = Vec::new();
313	let mut chunk = [0_u8; 8192];
314	loop {
315	let read = decryptor.read(&mut chunk).map_err(decrypt_failed)?;
316	if read == 0 {
317	return Ok(decrypted);
318	}
319	if read > max_bytes.saturating_sub(decrypted.len()) {
320	return Err(too_large(
321	"decrypted media exceeds the configured byte limit",
322	));
323	}
324	decrypted.extend_from_slice(&chunk[..read]);
325	}
326	}
327
328	/// the sdk's own download has no size cap and no timeout, so we do it by hand
329	async fn download_bounded(
330	client: &Client,
331	request: &MediaRequestParameters,
332	max_bytes: usize,
333	) -> Result<(Vec<u8>, Option<String>), Error> {
334	let uri = match &request.source {
335	MediaSource::Plain(uri) => uri,
336	MediaSource::Encrypted(file) => &file.url,
337	};
338	if uri
339	.server_name()
340	.is_ok_and(\|server\| server.as_str() == "send-queue.localhost")
341	{
342	return Err(Error::new(
343	ErrorKind::MediaUnavailable,
344	"local media is not in the media cache",
345	));
346	}
347	let network_failed =
348	\|source\| Error::with_source(ErrorKind::Network, "media download failed", source);
349	let mut refreshed = false;
350	let response = loop {
351	let http_request = media_http_request(client, request).await?;
352	let (parts, _) = http_request.into_parts();
353	let mut builder = client.http_client().get(parts.uri.to_string());
354	for (name, value) in &parts.headers {
355	builder = builder.header(name.as_str(), value.as_bytes());
356	}
357	let response = builder.send().await.map_err(network_failed)?;
358	if response.status() != matrix_sdk::reqwest::StatusCode::UNAUTHORIZED {
359	break response;
360	}
361	if refreshed
362	\|\| client
363	.session_tokens()
364	.is_none_or(\|tokens\| tokens.refresh_token.is_none())
365	{
366	return Err(Error::new(
367	ErrorKind::Unauthenticated,
368	"media download was not authorized",
369	));
370	}
371	client.refresh_access_token().await.map_err(\|source\| {
372	Error::from_refresh_token("refreshing media access token failed", source)
373	})?;
374	refreshed = true;
375	};
376	let status = response.status();
377	if !status.is_success() {
378	return Err(Error::new(
379	ErrorKind::MediaUnavailable,
380	format!("media download failed with status {status}"),
381	));
382	}
383	if response
384	.content_length()
385	.is_some_and(\|length\| length > max_bytes as u64)
386	{
387	return Err(too_large("media exceeds the configured byte limit"));
388	}
389	let mimetype = response
390	.headers()
391	.get(http::header::CONTENT_TYPE)
392	.and_then(\|value\| value.to_str().ok())
393	.map(str::to_owned);
394	let mut stream = response.bytes_stream();
395	let mut bytes = Vec::new();
396	while let Some(chunk) = stream.next().await {
397	let chunk = chunk.map_err(network_failed)?;
398	if chunk.len() > max_bytes.saturating_sub(bytes.len()) {
399	return Err(too_large(
400	"downloaded media exceeds the configured byte limit",
401	));
402	}
403	bytes.extend_from_slice(&chunk);
404	}
405	Ok((bytes, mimetype))
406	}
407
408	async fn media_http_request(
409	client: &Client,
410	request: &MediaRequestParameters,
411	) -> Result<http::Request<Vec<u8>>, Error> {
412	let versions = client.supported_versions().await.map_err(\|source\| {
413	Error::with_source(
414	ErrorKind::Network,
415	"reading supported server versions failed",
416	source,
417	)
418	})?;
419	let authenticated =
420	authenticated_media::get_content::v1::Request::PATH_BUILDER.is_supported(&versions);
421	let base = client.homeserver().to_string();
422	let token = client.access_token();
423	let invalid_uri =
424	\|source\| Error::with_source(ErrorKind::Invalid, "media URI is invalid", source);
425	let (uri, thumbnail) = match (&request.source, &request.format) {
426	(MediaSource::Encrypted(file), _) => (&file.url, None),
427	(MediaSource::Plain(uri), MediaFormat::File) => (uri, None),
428	(MediaSource::Plain(uri), MediaFormat::Thumbnail(settings)) => (uri, Some(settings)),
429	};
430	// each endpoint has its own auth input type, so no generic helper can take them all
431	macro_rules! into_http {
432	($endpoint:expr, $token:expr) => {
433	$endpoint
434	.try_into_http_request::<Vec<u8>>(&base, $token, Cow::Borrowed(&versions))
435	.map_err(\|source\| {
436	Error::with_source(ErrorKind::Other, "media request is invalid", source)
437	})?
438	};
439	}
440	let authed = SendAccessToken::IfRequired(token.as_deref().unwrap_or(""));
441	#[allow(deprecated)]
442	let mut http_request = match (thumbnail, authenticated) {
443	(None, true) => into_http!(
444	authenticated_media::get_content::v1::Request::from_uri(uri).map_err(invalid_uri)?,
445	authed
446	),
447	(None, false) => into_http!(
448	media::get_content::v3::Request::from_url(uri).map_err(invalid_uri)?,
449	SendAccessToken::None
450	),
451	(Some(settings), true) => {
452	let mut endpoint = authenticated_media::get_content_thumbnail::v1::Request::from_uri(
453	uri,
454	settings.width,
455	settings.height,
456	)
457	.map_err(invalid_uri)?;
458	endpoint.method = Some(settings.method.clone());
459	endpoint.animated = Some(settings.animated);
460	into_http!(endpoint, authed)
461	}
462	(Some(settings), false) => {
463	let mut endpoint = media::get_content_thumbnail::v3::Request::from_url(
464	uri,
465	settings.width,
466	settings.height,
467	)
468	.map_err(invalid_uri)?;
469	endpoint.method = Some(settings.method.clone());
470	endpoint.animated = Some(settings.animated);
471	into_http!(endpoint, SendAccessToken::None)
472	}
473	};
474	// legacy endpoints don't ask for a token but plenty of servers want one anyway
475	if let Some(token) = token.as_deref() {
476	let value = format!("Bearer {token}").parse().map_err(\|_\| {
477	Error::new(ErrorKind::Unauthenticated, "stored access token is invalid")
478	})?;
479	http_request
480	.headers_mut()
481	.insert(http::header::AUTHORIZATION, value);
482	}
483	Ok(http_request)
484	}
485
486	pub(crate) async fn queue_upload(
487	room: Room,
488	upload: AttachmentUpload,
489	max_bytes: usize,
490	) -> Result<(), Error> {
491	let (upload, mime, mut config) = prepare_upload(upload, max_bytes)?;
492	if let TimelineId::Thread { root_event_id, .. } = &upload.timeline_id {
493	config.reply = Some(SdkReply {
494	event_id: root_event_id.clone(),
495	enforce_thread: EnforceThread::Threaded(ReplyWithinThread::No),
496	add_mentions: AddMentions::No,
497	});
498	}
499	room.send_queue()
500	.send_attachment(upload.filename, mime, upload.bytes, config)
501	.await
502	.map(drop)
503	.map_err(\|source\| {
504	Error::with_source(
505	ErrorKind::MediaUnavailable,
506	"media upload could not be queued",
507	source,
508	)
509	})
510	}
511
512	/// least recently seen goes first. stale order slots are skipped by stamp rather than hunted down
513	struct MediaRegistry {
514	entries: HashMap<MediaId, (MediaRegistration, u64)>,
515	order: VecDeque<(MediaId, u64)>,
516	next_stamp: u64,
517	limit: usize,
518	}
519
520	impl MediaRegistry {
521	fn new(limit: usize) -> Self {
522	Self {
523	entries: HashMap::new(),
524	order: VecDeque::new(),
525	next_stamp: 0,
526	limit,
527	}
528	}
529
530	fn get(&self, id: &MediaId) -> Option<&MediaRegistration> {
531	self.entries.get(id).map(\|(media, _)\| media)
532	}
533
534	fn insert_all(&mut self, media: Vec<MediaRegistration>) {
535	for media in media {
536	let stamp = self.next_stamp;
537	self.next_stamp += 1;
538	self.order.push_back((media.id.clone(), stamp));
539	self.entries.insert(media.id.clone(), (media, stamp));
540	}
541	while self.entries.len() > self.limit {
542	let Some((id, stamp)) = self.order.pop_front() else {
543	break;
544	};
545	if self.is_current(&id, stamp) {
546	self.entries.remove(&id);
547	}
548	}
549	if self.order.len() > self.limit.saturating_mul(4) {
550	let entries = &self.entries;
551	self.order
552	.retain(\|(id, stamp)\| entries.get(id).is_some_and(\|(_, current)\| current == stamp));
553	}
554	}
555
556	fn is_current(&self, id: &MediaId, stamp: u64) -> bool {
557	self.entries
558	.get(id)
559	.is_some_and(\|(_, current)\| *current == stamp)
560	}
561
562	fn release(&mut self, scope: &MediaScope) {
563	self.entries.retain(\|id, _\| !scope.owns(id));
564	let entries = &self.entries;
565	self.order.retain(\|(id, _)\| entries.contains_key(id));
566	}
567
568	fn clear(&mut self) {
569	self.entries.clear();
570	self.order.clear();
571	}
572	}