From d0ec5dacaf2b18a04ca5d1ad11b2bc98e9bdb979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 14 Aug 2024 10:10:58 +0200 Subject: [PATCH] message-toolbar: Be smarter about generated thumbnail Always use WebP to generate thumbnails, as it is known to be widely supported and have a good compression ratio. Only generate thumbnails when the bandwith savings make sense. --- Cargo.lock | 22 +- Cargo.toml | 2 +- meson.build | 1 + .../room_details/edit_details_subpage.rs | 4 +- .../room_history/message_toolbar/mod.rs | 67 ++-- src/utils/media.rs | 299 +++++++++++++++++- 6 files changed, 338 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 305e468e9..c72c7de1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1566,6 +1566,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "webp", ] [[package]] @@ -3101,6 +3102,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebp-sys" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829b6b604f31ed6d2bccbac841fe0788de93dbd87e4eb1ba2c4adfe8c012a838" +dependencies = [ + "cc", + "glob", +] + [[package]] name = "linkify" version = "0.10.0" @@ -3305,7 +3316,6 @@ dependencies = [ "futures-util", "gloo-timers", "http", - "image", "imbl", "indexmap", "js_int", @@ -5997,6 +6007,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f53152f51fb5af0c08484c33d16cca96175881d1f3dec068c23b31a158c2d99" +dependencies = [ + "image", + "libwebp-sys", +] + [[package]] name = "weezl" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 99b4f086e..300d5b3ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ tokio-stream = { version = "0.1", features = ["sync"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = "2" +webp = "0.3" # gtk-rs project and dependents. These usually need to be updated together. adw = { package = "libadwaita", version = "0.7", features = ["v1_5"] } @@ -71,7 +72,6 @@ features = [ "sso-login", "markdown", "qrcode", - "image-rayon", ] [dependencies.matrix-sdk-ui] diff --git a/meson.build b/meson.build index 864a41d45..995af8936 100644 --- a/meson.build +++ b/meson.build @@ -39,6 +39,7 @@ dependency( fallback: ['gtksourceview', 'gtksource_dep'], default_options: ['gtk_doc=false', 'sysprof=false', 'gir=false', 'vapi=false', 'install_tests=false'] ) +dependency('libwebp', version: '>= 1.0.0') dependency('openssl', version: '>= 1.0.1') dependency('shumate-1.0', version: '>= 1.0.0') dependency('sqlite3', version: '>= 3.24.0') diff --git a/src/session/view/content/room_details/edit_details_subpage.rs b/src/session/view/content/room_details/edit_details_subpage.rs index 03c33bc76..f2b319818 100644 --- a/src/session/view/content/room_details/edit_details_subpage.rs +++ b/src/session/view/content/room_details/edit_details_subpage.rs @@ -17,7 +17,7 @@ use crate::{ session::model::Room, spawn_tokio, toast, utils::{ - media::{get_image_info, load_file}, + media::{load_file, ImageInfoLoader}, template_callbacks::TemplateCallbacks, BoundObjectWeakRef, OngoingAsyncAction, }, @@ -183,7 +183,7 @@ mod imp { } }; - let base_image_info = get_image_info(file).await; + let base_image_info = ImageInfoLoader::from(file).load_info().await; let image_info = assign!(ImageInfo::new(), { width: base_image_info.width, height: base_image_info.height, diff --git a/src/session/view/content/room_history/message_toolbar/mod.rs b/src/session/view/content/room_history/message_toolbar/mod.rs index 3ee82a49c..db478ce70 100644 --- a/src/session/view/content/room_history/message_toolbar/mod.rs +++ b/src/session/view/content/room_history/message_toolbar/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fmt::Write, io::Cursor}; +use std::{collections::HashMap, fmt::Write}; use adw::{prelude::*, subclass::prelude::*}; use futures_util::{future, pin_mut, StreamExt}; @@ -8,12 +8,8 @@ use gtk::{ glib::{self, clone}, CompositeTemplate, }; -use image::ImageFormat; use matrix_sdk::{ - attachment::{ - generate_image_thumbnail, AttachmentConfig, AttachmentInfo, BaseFileInfo, BaseImageInfo, - ThumbnailFormat, - }, + attachment::{AttachmentConfig, AttachmentInfo, BaseFileInfo, Thumbnail}, room::edit::EditedContent, }; use matrix_sdk_ui::timeline::{RepliedToInfo, TimelineItemContent}; @@ -44,7 +40,7 @@ use crate::{ spawn, spawn_tokio, toast, utils::{ matrix::AT_ROOM, - media::{filename_for_mime, get_audio_info, get_image_info, get_video_info, load_file}, + media::{filename_for_mime, get_audio_info, get_video_info, load_file, ImageInfoLoader}, template_callbacks::TemplateCallbacks, Location, LocationError, TokioDrop, }, @@ -791,31 +787,22 @@ impl MessageToolbar { mime: mime::Mime, body: String, info: AttachmentInfo, + thumbnail: Option, ) { let Some(room) = self.room() else { return; }; + let config = if let Some(thumbnail) = thumbnail { + AttachmentConfig::with_thumbnail(thumbnail) + } else { + AttachmentConfig::new() + } + .info(info); + let matrix_room = room.matrix_room().clone(); let handle = spawn_tokio!(async move { - // The method will filter compatible mime types so we don't need to, - // since we ignore errors. - let thumbnail = generate_image_thumbnail( - &mime, - Cursor::new(&bytes), - None, - ThumbnailFormat::Fallback(ImageFormat::Jpeg), - ) - .ok(); - - let config = if let Some(thumbnail) = thumbnail { - AttachmentConfig::with_thumbnail(thumbnail) - } else { - AttachmentConfig::new() - } - .info(info); - matrix_room .send_attachment(&body, &mime, bytes, config) .await @@ -845,14 +832,15 @@ impl MessageToolbar { } let bytes = image.save_to_png_bytes(); - let info = AttachmentInfo::Image(BaseImageInfo { - width: Some((image.width() as u32).into()), - height: Some((image.height() as u32).into()), - size: Some((bytes.len() as u32).into()), - blurhash: None, - }); + let filesize = bytes.len().try_into().ok(); - self.send_attachment(bytes.to_vec(), mime::IMAGE_PNG, filename, info) + let (mut base_info, thumbnail) = ImageInfoLoader::from(image) + .load_info_and_thumbnail(filesize) + .await; + base_info.size = filesize.map(Into::into); + + let info = AttachmentInfo::Image(base_info); + self.send_attachment(bytes.to_vec(), mime::IMAGE_PNG, filename, info, thumbnail) .await; } @@ -912,26 +900,29 @@ impl MessageToolbar { } let size = file_info.size.map(Into::into); - let info = match file_info.mime.type_() { + let (info, thumbnail) = match file_info.mime.type_() { mime::IMAGE => { - let mut info = get_image_info(file).await; + let (mut info, thumbnail) = ImageInfoLoader::from(file) + .load_info_and_thumbnail(file_info.size) + .await; info.size = size; - AttachmentInfo::Image(info) + + (AttachmentInfo::Image(info), thumbnail) } mime::VIDEO => { let mut info = get_video_info(&file).await; info.size = size; - AttachmentInfo::Video(info) + (AttachmentInfo::Video(info), None) } mime::AUDIO => { let mut info = get_audio_info(&file).await; info.size = size; - AttachmentInfo::Audio(info) + (AttachmentInfo::Audio(info), None) } - _ => AttachmentInfo::File(BaseFileInfo { size }), + _ => (AttachmentInfo::File(BaseFileInfo { size }), None), }; - self.send_attachment(bytes, file_info.mime, file_info.filename, info) + self.send_attachment(bytes, file_info.mime, file_info.filename, info, thumbnail) .await; } diff --git a/src/utils/media.rs b/src/utils/media.rs index 5e944d4ef..ec182b60e 100644 --- a/src/utils/media.rs +++ b/src/utils/media.rs @@ -3,13 +3,38 @@ use std::{cell::Cell, str::FromStr, sync::Mutex}; use gettextrs::gettext; -use glycin::Image; use gtk::{gdk, gio, glib, prelude::*}; -use matrix_sdk::attachment::{BaseAudioInfo, BaseImageInfo, BaseVideoInfo}; +use image::{ColorType, DynamicImage, ImageDecoder, ImageResult}; +use matrix_sdk::attachment::{ + BaseAudioInfo, BaseImageInfo, BaseThumbnailInfo, BaseVideoInfo, Thumbnail, +}; use mime::Mime; use crate::{components::AnimatedImagePaintable, spawn_tokio, DISABLE_GLYCIN_SANDBOX}; +/// The default width of a generated thumbnail. +const THUMBNAIL_DEFAULT_WIDTH: u32 = 800; +/// The default height of a generated thumbnail. +const THUMBNAIL_DEFAULT_HEIGHT: u32 = 600; +/// The content type of WebP. +const WEBP_CONTENT_TYPE: &str = "image/webp"; +/// The default WebP quality used for a generated thumbnail. +const WEBP_DEFAULT_QUALITY: f32 = 60.0; +/// The maximum file size threshold in bytes for generating a thumbnail. +/// +/// If the file size of the original image is larger than this, we assume it is +/// worth it to generate a thumbnail, even if its dimensions are smaller than +/// wanted. This is particularly helpful for some image formats that can take up +/// a lot of space. +/// +/// This is 1MB. +const THUMBNAIL_MAX_FILESIZE_THRESHOLD: u32 = 1024 * 1024; +/// The dimension threshold in pixels before we start to generate a thumbnail. +/// +/// If the original image is larger than thumbnail_dimensions + threshold, we +/// assume it's worth it to generate a thumbnail. +const THUMBNAIL_DIMENSIONS_THRESHOLD: u32 = 200; + /// Get a default filename for a mime type. /// /// Tries to guess the file extension, but it might not find it. @@ -98,7 +123,7 @@ pub async fn load_file(file: &gio::File) -> Result<(Vec, FileInfo), glib::Er } /// Get an image reader for the given file. -pub async fn image_reader(file: gio::File) -> Result, glycin::ErrorCtx> { +async fn image_reader(file: gio::File) -> Result, glycin::ErrorCtx> { let mut loader = glycin::Loader::new(file); if DISABLE_GLYCIN_SANDBOX { @@ -130,21 +155,265 @@ pub async fn load_image(file: gio::File) -> Result BaseImageInfo { - let mut info = BaseImageInfo { - width: None, - height: None, - size: None, - blurhash: None, - }; +/// An API to load image information. +pub enum ImageInfoLoader { + /// An image file. + File(gio::File), + /// A texture in memory. + Texture(gdk::Texture), +} - if let Ok(image) = image_reader(file).await { - let image_info = image.info(); - info.width = Some(image_info.width.into()); - info.height = Some(image_info.height.into()); +impl ImageInfoLoader { + /// Load the first frame for this source. + /// + /// We need to load the first frame of an image so that EXIF rotation is + /// applied and we get the proper dimensions. + async fn into_first_frame(self) -> Option { + match self { + Self::File(file) => { + let image_reader = image_reader(file).await.ok()?; + let handle = spawn_tokio!(async move { image_reader.next_frame().await }); + Some(Frame::Glycin(handle.await.unwrap().ok()?)) + } + Self::Texture(texture) => Some(Frame::Texture(gdk::TextureDownloader::new(&texture))), + } } - info + /// Load the information for this image. + pub async fn load_info(self) -> BaseImageInfo { + self.into_first_frame() + .await + .map(|f| f.dimensions()) + .unwrap_or_default() + .into() + } + + /// Load the information for this image and try to generate a thumbnail + /// given the filesize of the original image. + pub async fn load_info_and_thumbnail( + self, + filesize: Option, + ) -> (BaseImageInfo, Option) { + let Some(frame) = self.into_first_frame().await else { + return (ImageDimensions::default().into(), None); + }; + + let dimensions = frame.dimensions(); + let info = dimensions.into(); + + if !filesize.is_some_and(|s| s >= THUMBNAIL_MAX_FILESIZE_THRESHOLD) + && !dimensions + .width + .is_some_and(|w| w > (THUMBNAIL_DEFAULT_WIDTH + THUMBNAIL_DIMENSIONS_THRESHOLD)) + && !dimensions + .height + .is_some_and(|h| h > (THUMBNAIL_DEFAULT_HEIGHT + THUMBNAIL_DIMENSIONS_THRESHOLD)) + { + // It is not worth it to generate a thumbnail. + return (info, None); + } + + let thumbnail = frame.generate_thumbnail(); + + (info, thumbnail) + } +} + +impl From for ImageInfoLoader { + fn from(value: gio::File) -> Self { + Self::File(value) + } +} + +impl From for ImageInfoLoader { + fn from(value: gdk::Texture) -> Self { + Self::Texture(value) + } +} + +/// A frame of an image. +enum Frame { + /// A frame loaded via glycin. + Glycin(glycin::Frame), + /// A downloader for a texture in memory, + Texture(gdk::TextureDownloader), +} + +impl Frame { + /// The dimensions of the frame. + fn dimensions(&self) -> ImageDimensions { + match self { + Self::Glycin(frame) => ImageDimensions { + width: Some(frame.width()), + height: Some(frame.height()), + }, + Self::Texture(downloader) => { + let texture = downloader.texture(); + ImageDimensions { + width: texture.width().try_into().ok(), + height: texture.height().try_into().ok(), + } + } + } + } + + /// Whether the memory format of the frame is supported by the image crate. + fn is_supported(&self) -> bool { + match self { + Self::Glycin(frame) => { + matches!( + frame.memory_format(), + glycin::MemoryFormat::G8 + | glycin::MemoryFormat::G8a8 + | glycin::MemoryFormat::R8g8b8 + | glycin::MemoryFormat::R8g8b8a8 + | glycin::MemoryFormat::G16 + | glycin::MemoryFormat::G16a16 + | glycin::MemoryFormat::R16g16b16 + | glycin::MemoryFormat::R16g16b16a16 + | glycin::MemoryFormat::R32g32b32Float + | glycin::MemoryFormat::R32g32b32a32Float + ) + } + Self::Texture(downloader) => { + matches!( + downloader.format(), + gdk::MemoryFormat::G8 + | gdk::MemoryFormat::G8a8 + | gdk::MemoryFormat::R8g8b8 + | gdk::MemoryFormat::R8g8b8a8 + | gdk::MemoryFormat::G16 + | gdk::MemoryFormat::G16a16 + | gdk::MemoryFormat::R16g16b16 + | gdk::MemoryFormat::R16g16b16a16 + | gdk::MemoryFormat::R32g32b32Float + | gdk::MemoryFormat::R32g32b32a32Float + ) + } + } + } + + /// Generate a thumbnail of this frame. + fn generate_thumbnail(self) -> Option { + if !self.is_supported() { + return None; + } + + let image = DynamicImage::from_decoder(self).ok()?; + let thumbnail = image.thumbnail(THUMBNAIL_DEFAULT_WIDTH, THUMBNAIL_DEFAULT_HEIGHT); + + // Convert to RGB8/RGBA8 since it's the only format supported by webp. + let thumbnail: DynamicImage = match &thumbnail { + DynamicImage::ImageLuma8(_) + | DynamicImage::ImageRgb8(_) + | DynamicImage::ImageLuma16(_) + | DynamicImage::ImageRgb16(_) + | DynamicImage::ImageRgb32F(_) => thumbnail.into_rgb8().into(), + DynamicImage::ImageLumaA8(_) + | DynamicImage::ImageRgba8(_) + | DynamicImage::ImageLumaA16(_) + | DynamicImage::ImageRgba16(_) + | DynamicImage::ImageRgba32F(_) => thumbnail.into_rgba8().into(), + _ => return None, + }; + + let encoder = webp::Encoder::from_image(&thumbnail).ok()?; + let thumbnail_bytes = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec(); + + let thumbnail_content_type = mime::Mime::from_str(WEBP_CONTENT_TYPE) + .expect("image should provide a valid content type"); + + let thumbnail_info = BaseThumbnailInfo { + width: Some(thumbnail.width().into()), + height: Some(thumbnail.height().into()), + size: thumbnail_bytes.len().try_into().ok(), + }; + + Some(Thumbnail { + data: thumbnail_bytes, + content_type: thumbnail_content_type, + info: Some(thumbnail_info), + }) + } +} + +impl ImageDecoder for Frame { + fn dimensions(&self) -> (u32, u32) { + let dimensions = self.dimensions(); + ( + dimensions.width.unwrap_or(0), + dimensions.height.unwrap_or(0), + ) + } + + fn color_type(&self) -> ColorType { + match self { + Self::Glycin(frame) => match frame.memory_format() { + glycin::MemoryFormat::G8 => ColorType::L8, + glycin::MemoryFormat::G8a8 => ColorType::La8, + glycin::MemoryFormat::R8g8b8 => ColorType::Rgb8, + glycin::MemoryFormat::R8g8b8a8 => ColorType::Rgba8, + glycin::MemoryFormat::G16 => ColorType::L16, + glycin::MemoryFormat::G16a16 => ColorType::La16, + glycin::MemoryFormat::R16g16b16 => ColorType::Rgb16, + glycin::MemoryFormat::R16g16b16a16 => ColorType::Rgba16, + glycin::MemoryFormat::R32g32b32Float => ColorType::Rgb32F, + glycin::MemoryFormat::R32g32b32a32Float => ColorType::Rgba32F, + _ => unimplemented!(), + }, + Self::Texture(downloader) => match downloader.format() { + gdk::MemoryFormat::G8 => ColorType::L8, + gdk::MemoryFormat::G8a8 => ColorType::La8, + gdk::MemoryFormat::R8g8b8 => ColorType::Rgb8, + gdk::MemoryFormat::R8g8b8a8 => ColorType::Rgba8, + gdk::MemoryFormat::G16 => ColorType::L16, + gdk::MemoryFormat::G16a16 => ColorType::La16, + gdk::MemoryFormat::R16g16b16 => ColorType::Rgb16, + gdk::MemoryFormat::R16g16b16a16 => ColorType::Rgba16, + gdk::MemoryFormat::R32g32b32Float => ColorType::Rgb32F, + gdk::MemoryFormat::R32g32b32a32Float => ColorType::Rgba32F, + _ => unimplemented!(), + }, + } + } + + fn read_image(self, buf: &mut [u8]) -> ImageResult<()> + where + Self: Sized, + { + let bytes = match &self { + Self::Glycin(frame) => frame.buf_bytes(), + Self::Texture(texture) => texture.download_bytes().0, + }; + buf.copy_from_slice(&bytes); + + Ok(()) + } + + fn read_image_boxed(self: Box, _buf: &mut [u8]) -> ImageResult<()> { + unimplemented!() + } +} + +/// Dimensions of an image. +#[derive(Debug, Clone, Copy, Default)] +struct ImageDimensions { + /// The width of the image. + width: Option, + /// The height of the image. + height: Option, +} + +impl From for BaseImageInfo { + fn from(value: ImageDimensions) -> Self { + let ImageDimensions { width, height } = value; + BaseImageInfo { + height: height.map(Into::into), + width: width.map(Into::into), + size: None, + blurhash: None, + } + } } async fn get_gstreamer_media_info(file: &gio::File) -> Option { -- GitLab