From ae96d863a44ae6630e626f10d31deb35a4e1d19f Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 24 Oct 2023 16:53:53 +0200 Subject: [PATCH] rewrite link fields for avatar, banner etc --- Cargo.lock | 1 + crates/api/src/local_user/save_settings.rs | 13 +++++--- crates/api_common/Cargo.toml | 11 ++++--- crates/api_common/src/utils.rs | 37 ++++++++++++++++++++-- crates/api_crud/src/community/create.rs | 8 ++--- crates/api_crud/src/community/update.rs | 13 +++++--- crates/api_crud/src/post/create.rs | 24 +++++++++++--- crates/api_crud/src/post/update.rs | 19 +++++++---- crates/api_crud/src/site/create.rs | 9 ++++-- crates/api_crud/src/site/update.rs | 9 ++++-- crates/db_schema/src/impls/images.rs | 7 ++-- crates/routes/src/image_proxy.rs | 3 +- 12 files changed, 112 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a8a71bd9..e38e08609 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2681,6 +2681,7 @@ dependencies = [ "tracing", "ts-rs", "url", + "urlencoding", "uuid", "webpage", ] diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index 2948d97b6..beea4e8dc 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -2,7 +2,12 @@ use actix_web::web::{Data, Json}; use lemmy_api_common::{ context::LemmyContext, person::SaveUserSettings, - utils::{local_site_to_slur_regex, process_markdown_opt, send_verification_email}, + utils::{ + local_site_to_slur_regex, + process_markdown_opt, + proxy_image_link_opt, + send_verification_email, + }, SuccessResponse, }; use lemmy_db_schema::{ @@ -12,7 +17,7 @@ use lemmy_db_schema::{ person::{Person, PersonUpdateForm}, }, traits::Crud, - utils::{diesel_option_overwrite, diesel_option_overwrite_to_url}, + utils::diesel_option_overwrite, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ @@ -31,8 +36,8 @@ pub async fn save_user_settings( let slur_regex = local_site_to_slur_regex(&site_view.local_site); let bio = process_markdown_opt(&data.bio, &slur_regex, &context).await?; - let avatar = diesel_option_overwrite_to_url(&data.avatar)?; - let banner = diesel_option_overwrite_to_url(&data.banner)?; + let avatar = proxy_image_link_opt(&data.avatar, &context).await?; + let banner = proxy_image_link_opt(&data.banner, &context).await?; let bio = diesel_option_overwrite(bio); let display_name = diesel_option_overwrite(data.display_name.clone()); let matrix_user_id = diesel_option_overwrite(data.matrix_user_id.clone()); diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index a01e6008c..c3a7fa8d9 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -53,10 +53,6 @@ reqwest-middleware = { workspace = true, optional = true } regex = { workspace = true } rosetta-i18n = { workspace = true, optional = true } percent-encoding = { workspace = true, optional = true } -webpage = { version = "1.6", default-features = false, features = [ - "serde", -], optional = true } -encoding = { version = "0.2.33", optional = true } anyhow = { workspace = true } futures = { workspace = true, optional = true } uuid = { workspace = true, optional = true } @@ -65,10 +61,15 @@ reqwest = { workspace = true, optional = true } ts-rs = { workspace = true, optional = true } once_cell = { workspace = true, optional = true } actix-web = { workspace = true, optional = true } +enum-map = { workspace = true } +urlencoding = { workspace = true } +webpage = { version = "1.6", default-features = false, features = [ + "serde", +], optional = true } +encoding = { version = "0.2.33", optional = true } jsonwebtoken = { version = "8.3.0", optional = true } # necessary for wasmt compilation getrandom = { version = "0.2.10", features = ["js"] } -enum-map = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index f2dbbb296..2f3b9406b 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -24,7 +24,7 @@ use lemmy_db_schema::{ post::{Post, PostRead}, }, traits::Crud, - utils::DbPool, + utils::{diesel_option_overwrite_to_url, DbPool}, }; use lemmy_db_views::{comment_view::CommentQuery, structs::LocalUserView}; use lemmy_db_views_actor::structs::{ @@ -37,7 +37,7 @@ use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult}, location_info, rate_limit::{ActionType, BucketConfig}, - settings::structs::Settings, + settings::{structs::Settings, SETTINGS}, utils::{ markdown::markdown_rewrite_image_links, slurs::{build_slur_regex, remove_slurs}, @@ -48,6 +48,7 @@ use rosetta_i18n::{Language, LanguageId}; use std::collections::HashSet; use tracing::warn; use url::{ParseError, Url}; +use urlencoding::encode; pub static AUTH_COOKIE_NAME: &str = "auth"; @@ -797,7 +798,7 @@ pub async fn process_markdown( ) -> LemmyResult { let text = remove_slurs(text, slur_regex); let (text, links) = markdown_rewrite_image_links(text); - RemoteImage::create_many(&mut context.pool(), links).await?; + RemoteImage::create(&mut context.pool(), links).await?; Ok(text) } @@ -812,6 +813,36 @@ pub async fn process_markdown_opt( } } +pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult { + // Dont rewrite links pointing to local domain. + if link.domain() == Some(&SETTINGS.hostname) { + return Ok(link.into()); + } + + let proxied = format!( + "{}/api/v3/image_proxy?url={}", + SETTINGS.get_protocol_and_hostname(), + encode(link.as_str()) + ); + RemoteImage::create(&mut context.pool(), vec![link]).await?; + Ok(Url::parse(&proxied)?.into()) +} + +pub async fn proxy_image_link_opt( + link: &Option, + context: &LemmyContext, +) -> LemmyResult>> { + let link = diesel_option_overwrite_to_url(link)?; + if let Some(Some(l)) = link { + proxy_image_link(l.into(), context) + .await + .map(Some) + .map(Some) + } else { + Ok(link) + } +} + #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] diff --git a/crates/api_crud/src/community/create.rs b/crates/api_crud/src/community/create.rs index 1110995aa..4f0c735a1 100644 --- a/crates/api_crud/src/community/create.rs +++ b/crates/api_crud/src/community/create.rs @@ -12,6 +12,7 @@ use lemmy_api_common::{ is_admin, local_site_to_slur_regex, process_markdown_opt, + proxy_image_link_opt, EndpointType, }, }; @@ -28,7 +29,6 @@ use lemmy_db_schema::{ }, }, traits::{ApubActor, Crud, Followable, Joinable}, - utils::diesel_option_overwrite_to_url_create, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ @@ -52,14 +52,12 @@ pub async fn create_community( Err(LemmyErrorType::OnlyAdminsCanCreateCommunities)? } - // Check to make sure the icon and banners are urls - let icon = diesel_option_overwrite_to_url_create(&data.icon)?; - let banner = diesel_option_overwrite_to_url_create(&data.banner)?; - let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.name, &slur_regex)?; check_slurs(&data.title, &slur_regex)?; let description = process_markdown_opt(&data.description, &slur_regex, &context).await?; + let icon = proxy_image_link_opt(&data.icon, &context).await?.unwrap(); + let banner = proxy_image_link_opt(&data.banner, &context).await?.unwrap(); is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?; is_valid_body_field(&data.description, false)?; diff --git a/crates/api_crud/src/community/update.rs b/crates/api_crud/src/community/update.rs index df6d6570f..83020cacb 100644 --- a/crates/api_crud/src/community/update.rs +++ b/crates/api_crud/src/community/update.rs @@ -5,7 +5,12 @@ use lemmy_api_common::{ community::{CommunityResponse, EditCommunity}, context::LemmyContext, send_activity::{ActivityChannel, SendActivityData}, - utils::{check_community_mod_action, local_site_to_slur_regex, process_markdown_opt}, + utils::{ + check_community_mod_action, + local_site_to_slur_regex, + process_markdown_opt, + proxy_image_link_opt, + }, }; use lemmy_db_schema::{ source::{ @@ -14,7 +19,7 @@ use lemmy_db_schema::{ local_site::LocalSite, }, traits::Crud, - utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now}, + utils::{diesel_option_overwrite, naive_now}, }; use lemmy_db_views::structs::LocalUserView; use lemmy_utils::{ @@ -35,9 +40,9 @@ pub async fn update_community( let description = process_markdown_opt(&data.description, &slur_regex, &context).await?; is_valid_body_field(&data.description, false)?; - let icon = diesel_option_overwrite_to_url(&data.icon)?; - let banner = diesel_option_overwrite_to_url(&data.banner)?; let description = diesel_option_overwrite(description); + let icon = proxy_image_link_opt(&data.icon, &context).await?; + let banner = proxy_image_link_opt(&data.banner, &context).await?; // Verify its a mod (only mods can edit it) check_community_mod_action( diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 3ffa54685..6ee2bab44 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -53,9 +53,6 @@ pub async fn create_post( let body = process_markdown_opt(&data.body, &slur_regex, &context).await?; honeypot_check(&data.honeypot)?; - let data_url = data.url.as_ref(); - let url = data_url.map(clean_url_params).map(Into::into); // TODO no good way to handle a "clear" - is_valid_post_title(&data.name)?; is_valid_body_field(&body, true)?; check_url_scheme(&data.url)?; @@ -83,12 +80,29 @@ pub async fn create_post( } // Fetch post links and pictrs cached image - let (metadata_res, thumbnail_url) = - fetch_site_data(context.client(), context.settings(), data_url, true).await; + let (metadata_res, thumbnail_url) = fetch_site_data( + context.client(), + context.settings(), + data.url.as_ref(), + true, + ) + .await; let (embed_title, embed_description, embed_video_url) = metadata_res .map(|u| (u.title, u.description, u.embed_video_url)) .unwrap_or_default(); + // TODO No good way to handle a clear. + // Issue link: https://github.com/LemmyNet/lemmy/issues/2287 + let url = data.url.as_ref().map(clean_url_params).map(Into::into); + + // TODO: not sure how to get this working + /* + let url_is_image = todo!(); + if url_is_image { + proxy_image_link_opt(url, &context).await?; + } + */ + // Only need to check if language is allowed in case user set it explicitly. When using default // language, it already only returns allowed languages. CommunityLanguage::is_allowed_community_language( diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index 73a4962f9..e8e127b1d 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -35,12 +35,6 @@ pub async fn update_post( ) -> Result, LemmyError> { let local_site = LocalSite::read(&mut context.pool()).await?; - let data_url = data.url.as_ref(); - - // TODO No good way to handle a clear. - // Issue link: https://github.com/LemmyNet/lemmy/issues/2287 - let url = Some(data_url.map(clean_url_params).map(Into::into)); - let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.name, &slur_regex)?; let body = process_markdown_opt(&data.body, &slur_regex, &context).await?; @@ -75,6 +69,19 @@ pub async fn update_post( .map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url))) .unwrap_or_default(); + // TODO No good way to handle a clear. + // Issue link: https://github.com/LemmyNet/lemmy/issues/2287 + let url = Some(data.url.as_ref().map(clean_url_params).map(Into::into)); + + // TODO: not sure how to get this working + /* + let url_is_image = todo!(); + if url_is_image { + data_url = proxy_image_link_opt(url, &context).await?; + } + + */ + let language_id = data.language_id; CommunityLanguage::is_allowed_community_language( &mut context.pool(), diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 2f3322512..aff2d40dc 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -10,6 +10,7 @@ use lemmy_api_common::{ local_site_rate_limit_to_rate_limit_config, local_site_to_slur_regex, process_markdown_opt, + proxy_image_link_opt, }, }; use lemmy_db_schema::{ @@ -21,7 +22,7 @@ use lemmy_db_schema::{ tagline::Tagline, }, traits::Crud, - utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now}, + utils::{diesel_option_overwrite, naive_now}, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ @@ -58,13 +59,15 @@ pub async fn create_site( let slur_regex = local_site_to_slur_regex(&local_site); let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?; + let icon = proxy_image_link_opt(&data.icon, &context).await?; + let banner = proxy_image_link_opt(&data.banner, &context).await?; let site_form = SiteUpdateForm { name: Some(data.name.clone()), sidebar: diesel_option_overwrite(sidebar), description: diesel_option_overwrite(data.description.clone()), - icon: diesel_option_overwrite_to_url(&data.icon)?, - banner: diesel_option_overwrite_to_url(&data.banner)?, + icon, + banner, actor_id: Some(actor_id), last_refreshed_at: Some(naive_now()), inbox_url, diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index 97bdb108a..440bc90ec 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -8,6 +8,7 @@ use lemmy_api_common::{ local_site_rate_limit_to_rate_limit_config, local_site_to_slur_regex, process_markdown_opt, + proxy_image_link_opt, }, }; use lemmy_db_schema::{ @@ -22,7 +23,7 @@ use lemmy_db_schema::{ tagline::Tagline, }, traits::Crud, - utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now}, + utils::{diesel_option_overwrite, naive_now}, RegistrationMode, }; use lemmy_db_views::structs::{LocalUserView, SiteView}; @@ -61,13 +62,15 @@ pub async fn update_site( let slur_regex = local_site_to_slur_regex(&local_site); let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?; + let icon = proxy_image_link_opt(&data.icon, &context).await?; + let banner = proxy_image_link_opt(&data.banner, &context).await?; let site_form = SiteUpdateForm { name: data.name.clone(), sidebar: diesel_option_overwrite(sidebar), description: diesel_option_overwrite(data.description.clone()), - icon: diesel_option_overwrite_to_url(&data.icon)?, - banner: diesel_option_overwrite_to_url(&data.banner)?, + icon, + banner, updated: Some(Some(naive_now())), ..Default::default() }; diff --git a/crates/db_schema/src/impls/images.rs b/crates/db_schema/src/impls/images.rs index 7533ca603..3fbb8ddbb 100644 --- a/crates/db_schema/src/impls/images.rs +++ b/crates/db_schema/src/impls/images.rs @@ -2,7 +2,7 @@ use crate::{ newtypes::{DbUrl, LocalImageId, LocalUserId}, schema::{ local_image::dsl::{local_image, local_user_id, pictrs_alias}, - remote_image::dsl::{remote_image, link}, + remote_image::dsl::{link, remote_image}, }, source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm}, utils::{get_conn, DbPool}, @@ -60,7 +60,7 @@ impl LocalImage { } impl RemoteImage { - pub async fn create_many(pool: &mut DbPool<'_>, links: Vec) -> Result { + pub async fn create(pool: &mut DbPool<'_>, links: Vec) -> Result { let conn = &mut get_conn(pool).await?; let forms = links .into_iter() @@ -68,7 +68,8 @@ impl RemoteImage { .collect::>(); insert_into(remote_image) .values(forms) - .get_result::(conn) + .on_conflict_do_nothing() + .execute(conn) .await } diff --git a/crates/routes/src/image_proxy.rs b/crates/routes/src/image_proxy.rs index 50d4f4a30..9d95404fb 100644 --- a/crates/routes/src/image_proxy.rs +++ b/crates/routes/src/image_proxy.rs @@ -33,7 +33,8 @@ async fn image_proxy( // for arbitrary purposes. RemoteImage::validate(&mut context.pool(), url.clone().into()).await?; - // TODO: Once pictrs 0.5 is out, use it for proxying like GET /image/original?proxy={url} + // TODO: Once pictrs 0.5 is out, use it for proxying like `GET /image/original?proxy={url}`. In + // case pictrs is unavailable, fallback to this impl. // https://git.asonix.dog/asonix/pict-rs/#api let image_response = context.client().get(url).send().await?;