rewrite link fields for avatar, banner etc

Cargo.lock

@@ -2681,6 +2681,7 @@ dependencies = [
  "tracing",
  "ts-rs",
  "url",
+ "urlencoding",
  "uuid",
  "webpage",
 ]

crates/api/src/local_user/save_settings.rs

@@ -2,7 +2,12 @@ use actix_web::web::{Data, Json};
 use lemmy_api_common::{
   context::LemmyContext,
   person::SaveUserSettings,
-  utils::{local_site_to_slur_regex, process_markdown_opt, send_verification_email},
+  utils::{
+    local_site_to_slur_regex,
+    process_markdown_opt,
+    proxy_image_link_opt,
+    send_verification_email,
+  },
   SuccessResponse,
 };
 use lemmy_db_schema::{
@@ -12,7 +17,7 @@ use lemmy_db_schema::{
     person::{Person, PersonUpdateForm},
   },
   traits::Crud,
-  utils::{diesel_option_overwrite, diesel_option_overwrite_to_url},
+  utils::diesel_option_overwrite,
 };
 use lemmy_db_views::structs::{LocalUserView, SiteView};
 use lemmy_utils::{
@@ -31,8 +36,8 @@ pub async fn save_user_settings(
   let slur_regex = local_site_to_slur_regex(&site_view.local_site);
   let bio = process_markdown_opt(&data.bio, &slur_regex, &context).await?;
 
-  let avatar = diesel_option_overwrite_to_url(&data.avatar)?;
+  let avatar = proxy_image_link_opt(&data.avatar, &context).await?;
-  let banner = diesel_option_overwrite_to_url(&data.banner)?;
+  let banner = proxy_image_link_opt(&data.banner, &context).await?;
   let bio = diesel_option_overwrite(bio);
   let display_name = diesel_option_overwrite(data.display_name.clone());
   let matrix_user_id = diesel_option_overwrite(data.matrix_user_id.clone());

crates/api_common/Cargo.toml

@@ -53,10 +53,6 @@ reqwest-middleware = { workspace = true, optional = true }
 regex = { workspace = true }
 rosetta-i18n = { workspace = true, optional = true }
 percent-encoding = { workspace = true, optional = true }
-webpage = { version = "1.6", default-features = false, features = [
-  "serde",
-], optional = true }
-encoding = { version = "0.2.33", optional = true }
 anyhow = { workspace = true }
 futures = { workspace = true, optional = true }
 uuid = { workspace = true, optional = true }
@@ -65,10 +61,15 @@ reqwest = { workspace = true, optional = true }
 ts-rs = { workspace = true, optional = true }
 once_cell = { workspace = true, optional = true }
 actix-web = { workspace = true, optional = true }
+enum-map = { workspace = true }
+urlencoding = { workspace = true }
+webpage = { version = "1.6", default-features = false, features = [
+  "serde",
+], optional = true }
+encoding = { version = "0.2.33", optional = true }
 jsonwebtoken = { version = "8.3.0", optional = true }
 # necessary for wasmt compilation
 getrandom = { version = "0.2.10", features = ["js"] }
-enum-map = { workspace = true }
 
 [dev-dependencies]
 serial_test = { workspace = true }

crates/api_common/src/utils.rs

@@ -24,7 +24,7 @@ use lemmy_db_schema::{
     post::{Post, PostRead},
   },
   traits::Crud,
-  utils::DbPool,
+  utils::{diesel_option_overwrite_to_url, DbPool},
 };
 use lemmy_db_views::{comment_view::CommentQuery, structs::LocalUserView};
 use lemmy_db_views_actor::structs::{
@@ -37,7 +37,7 @@ use lemmy_utils::{
   error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult},
   location_info,
   rate_limit::{ActionType, BucketConfig},
-  settings::structs::Settings,
+  settings::{structs::Settings, SETTINGS},
   utils::{
     markdown::markdown_rewrite_image_links,
     slurs::{build_slur_regex, remove_slurs},
@@ -48,6 +48,7 @@ use rosetta_i18n::{Language, LanguageId};
 use std::collections::HashSet;
 use tracing::warn;
 use url::{ParseError, Url};
+use urlencoding::encode;
 
 pub static AUTH_COOKIE_NAME: &str = "auth";
 
@@ -797,7 +798,7 @@ pub async fn process_markdown(
 ) -> LemmyResult<String> {
   let text = remove_slurs(text, slur_regex);
   let (text, links) = markdown_rewrite_image_links(text);
-  RemoteImage::create_many(&mut context.pool(), links).await?;
+  RemoteImage::create(&mut context.pool(), links).await?;
   Ok(text)
 }
 
@@ -812,6 +813,36 @@ pub async fn process_markdown_opt(
   }
 }
 
+pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> {
+  // Dont rewrite links pointing to local domain.
+  if link.domain() == Some(&SETTINGS.hostname) {
+    return Ok(link.into());
+  }
+
+  let proxied = format!(
+    "{}/api/v3/image_proxy?url={}",
+    SETTINGS.get_protocol_and_hostname(),
+    encode(link.as_str())
+  );
+  RemoteImage::create(&mut context.pool(), vec![link]).await?;
+  Ok(Url::parse(&proxied)?.into())
+}
+
+pub async fn proxy_image_link_opt(
+  link: &Option<String>,
+  context: &LemmyContext,
+) -> LemmyResult<Option<Option<DbUrl>>> {
+  let link = diesel_option_overwrite_to_url(link)?;
+  if let Some(Some(l)) = link {
+    proxy_image_link(l.into(), context)
+      .await
+      .map(Some)
+      .map(Some)
+  } else {
+    Ok(link)
+  }
+}
+
 #[cfg(test)]
 mod tests {
   #![allow(clippy::unwrap_used)]

crates/api_crud/src/community/create.rs

@@ -12,6 +12,7 @@ use lemmy_api_common::{
     is_admin,
     local_site_to_slur_regex,
     process_markdown_opt,
+    proxy_image_link_opt,
     EndpointType,
   },
 };
@@ -28,7 +29,6 @@ use lemmy_db_schema::{
     },
   },
   traits::{ApubActor, Crud, Followable, Joinable},
-  utils::diesel_option_overwrite_to_url_create,
 };
 use lemmy_db_views::structs::{LocalUserView, SiteView};
 use lemmy_utils::{
@@ -52,14 +52,12 @@ pub async fn create_community(
     Err(LemmyErrorType::OnlyAdminsCanCreateCommunities)?
   }
 
-  // Check to make sure the icon and banners are urls
-  let icon = diesel_option_overwrite_to_url_create(&data.icon)?;
-  let banner = diesel_option_overwrite_to_url_create(&data.banner)?;
-
   let slur_regex = local_site_to_slur_regex(&local_site);
   check_slurs(&data.name, &slur_regex)?;
   check_slurs(&data.title, &slur_regex)?;
   let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
+  let icon = proxy_image_link_opt(&data.icon, &context).await?.unwrap();
+  let banner = proxy_image_link_opt(&data.banner, &context).await?.unwrap();
 
   is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?;
   is_valid_body_field(&data.description, false)?;

crates/api_crud/src/community/update.rs

@@ -5,7 +5,12 @@ use lemmy_api_common::{
   community::{CommunityResponse, EditCommunity},
   context::LemmyContext,
   send_activity::{ActivityChannel, SendActivityData},
-  utils::{check_community_mod_action, local_site_to_slur_regex, process_markdown_opt},
+  utils::{
+    check_community_mod_action,
+    local_site_to_slur_regex,
+    process_markdown_opt,
+    proxy_image_link_opt,
+  },
 };
 use lemmy_db_schema::{
   source::{
@@ -14,7 +19,7 @@ use lemmy_db_schema::{
     local_site::LocalSite,
   },
   traits::Crud,
-  utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
+  utils::{diesel_option_overwrite, naive_now},
 };
 use lemmy_db_views::structs::LocalUserView;
 use lemmy_utils::{
@@ -35,9 +40,9 @@ pub async fn update_community(
   let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
   is_valid_body_field(&data.description, false)?;
 
-  let icon = diesel_option_overwrite_to_url(&data.icon)?;
-  let banner = diesel_option_overwrite_to_url(&data.banner)?;
   let description = diesel_option_overwrite(description);
+  let icon = proxy_image_link_opt(&data.icon, &context).await?;
+  let banner = proxy_image_link_opt(&data.banner, &context).await?;
 
   // Verify its a mod (only mods can edit it)
   check_community_mod_action(

crates/api_crud/src/post/create.rs

@@ -53,9 +53,6 @@ pub async fn create_post(
   let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
   honeypot_check(&data.honeypot)?;
 
-  let data_url = data.url.as_ref();
-  let url = data_url.map(clean_url_params).map(Into::into); // TODO no good way to handle a "clear"
-
   is_valid_post_title(&data.name)?;
   is_valid_body_field(&body, true)?;
   check_url_scheme(&data.url)?;
@@ -83,12 +80,29 @@ pub async fn create_post(
   }
 
   // Fetch post links and pictrs cached image
-  let (metadata_res, thumbnail_url) =
+  let (metadata_res, thumbnail_url) = fetch_site_data(
-    fetch_site_data(context.client(), context.settings(), data_url, true).await;
+    context.client(),
+    context.settings(),
+    data.url.as_ref(),
+    true,
+  )
+  .await;
   let (embed_title, embed_description, embed_video_url) = metadata_res
     .map(|u| (u.title, u.description, u.embed_video_url))
     .unwrap_or_default();
 
+  // TODO No good way to handle a clear.
+  // Issue link: https://github.com/LemmyNet/lemmy/issues/2287
+  let url = data.url.as_ref().map(clean_url_params).map(Into::into);
+
+  // TODO: not sure how to get this working
+  /*
+  let url_is_image = todo!();
+  if url_is_image {
+    proxy_image_link_opt(url, &context).await?;
+  }
+  */
+
   // Only need to check if language is allowed in case user set it explicitly. When using default
   // language, it already only returns allowed languages.
   CommunityLanguage::is_allowed_community_language(

crates/api_crud/src/post/update.rs

@@ -35,12 +35,6 @@ pub async fn update_post(
 ) -> Result<Json<PostResponse>, LemmyError> {
   let local_site = LocalSite::read(&mut context.pool()).await?;
 
-  let data_url = data.url.as_ref();
-
-  // TODO No good way to handle a clear.
-  // Issue link: https://github.com/LemmyNet/lemmy/issues/2287
-  let url = Some(data_url.map(clean_url_params).map(Into::into));
-
   let slur_regex = local_site_to_slur_regex(&local_site);
   check_slurs_opt(&data.name, &slur_regex)?;
   let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
@@ -75,6 +69,19 @@ pub async fn update_post(
     .map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
     .unwrap_or_default();
 
+  // TODO No good way to handle a clear.
+  // Issue link: https://github.com/LemmyNet/lemmy/issues/2287
+  let url = Some(data.url.as_ref().map(clean_url_params).map(Into::into));
+
+  // TODO: not sure how to get this working
+  /*
+  let url_is_image = todo!();
+  if url_is_image {
+    data_url = proxy_image_link_opt(url, &context).await?;
+  }
+
+  */
+
   let language_id = data.language_id;
   CommunityLanguage::is_allowed_community_language(
     &mut context.pool(),

crates/api_crud/src/site/create.rs

@@ -10,6 +10,7 @@ use lemmy_api_common::{
     local_site_rate_limit_to_rate_limit_config,
     local_site_to_slur_regex,
     process_markdown_opt,
+    proxy_image_link_opt,
   },
 };
 use lemmy_db_schema::{
@@ -21,7 +22,7 @@ use lemmy_db_schema::{
     tagline::Tagline,
   },
   traits::Crud,
-  utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
+  utils::{diesel_option_overwrite, naive_now},
 };
 use lemmy_db_views::structs::{LocalUserView, SiteView};
 use lemmy_utils::{
@@ -58,13 +59,15 @@ pub async fn create_site(
 
   let slur_regex = local_site_to_slur_regex(&local_site);
   let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
+  let icon = proxy_image_link_opt(&data.icon, &context).await?;
+  let banner = proxy_image_link_opt(&data.banner, &context).await?;
 
   let site_form = SiteUpdateForm {
     name: Some(data.name.clone()),
     sidebar: diesel_option_overwrite(sidebar),
     description: diesel_option_overwrite(data.description.clone()),
-    icon: diesel_option_overwrite_to_url(&data.icon)?,
+    icon,
-    banner: diesel_option_overwrite_to_url(&data.banner)?,
+    banner,
     actor_id: Some(actor_id),
     last_refreshed_at: Some(naive_now()),
     inbox_url,

crates/api_crud/src/site/update.rs

@@ -8,6 +8,7 @@ use lemmy_api_common::{
     local_site_rate_limit_to_rate_limit_config,
     local_site_to_slur_regex,
     process_markdown_opt,
+    proxy_image_link_opt,
   },
 };
 use lemmy_db_schema::{
@@ -22,7 +23,7 @@ use lemmy_db_schema::{
     tagline::Tagline,
   },
   traits::Crud,
-  utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
+  utils::{diesel_option_overwrite, naive_now},
   RegistrationMode,
 };
 use lemmy_db_views::structs::{LocalUserView, SiteView};
@@ -61,13 +62,15 @@ pub async fn update_site(
 
   let slur_regex = local_site_to_slur_regex(&local_site);
   let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
+  let icon = proxy_image_link_opt(&data.icon, &context).await?;
+  let banner = proxy_image_link_opt(&data.banner, &context).await?;
 
   let site_form = SiteUpdateForm {
     name: data.name.clone(),
     sidebar: diesel_option_overwrite(sidebar),
     description: diesel_option_overwrite(data.description.clone()),
-    icon: diesel_option_overwrite_to_url(&data.icon)?,
+    icon,
-    banner: diesel_option_overwrite_to_url(&data.banner)?,
+    banner,
     updated: Some(Some(naive_now())),
     ..Default::default()
   };

crates/db_schema/src/impls/images.rs

@@ -2,7 +2,7 @@ use crate::{
   newtypes::{DbUrl, LocalImageId, LocalUserId},
   schema::{
     local_image::dsl::{local_image, local_user_id, pictrs_alias},
-    remote_image::dsl::{remote_image, link},
+    remote_image::dsl::{link, remote_image},
   },
   source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm},
   utils::{get_conn, DbPool},
@@ -60,7 +60,7 @@ impl LocalImage {
 }
 
 impl RemoteImage {
-  pub async fn create_many(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<Self, Error> {
+  pub async fn create(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<usize, Error> {
     let conn = &mut get_conn(pool).await?;
     let forms = links
       .into_iter()
@@ -68,7 +68,8 @@ impl RemoteImage {
       .collect::<Vec<_>>();
     insert_into(remote_image)
       .values(forms)
-      .get_result::<Self>(conn)
+      .on_conflict_do_nothing()
+      .execute(conn)
       .await
   }
 

crates/routes/src/image_proxy.rs

@@ -33,7 +33,8 @@ async fn image_proxy(
   // for arbitrary purposes.
   RemoteImage::validate(&mut context.pool(), url.clone().into()).await?;
 
-  // TODO: Once pictrs 0.5 is out, use it for proxying like GET /image/original?proxy={url}
+  // TODO: Once pictrs 0.5 is out, use it for proxying like `GET /image/original?proxy={url}`. In
+  //       case pictrs is unavailable, fallback to this impl.
   //       https://git.asonix.dog/asonix/pict-rs/#api
   let image_response = context.client().get(url).send().await?;