add db table to validate proxied links

crates/api/src/local_user/save_settings.rs

@@ -28,8 +28,8 @@ pub async fn save_user_settings(
 ) -> Result<Json<SuccessResponse>, LemmyError> {
   let site_view = SiteView::read_local(&mut context.pool()).await?;
 
-  let bio =
-    process_markdown_opt(&data.bio, &local_site_to_slur_regex(&site_view.local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&site_view.local_site);
+  let bio = process_markdown_opt(&data.bio, &slur_regex, &context).await?;
 
   let avatar = diesel_option_overwrite_to_url(&data.avatar)?;
   let banner = diesel_option_overwrite_to_url(&data.banner)?;

crates/api/src/site/purge/person.rs

@@ -8,7 +8,7 @@ use lemmy_api_common::{
 };
 use lemmy_db_schema::{
   source::{
-    image_upload::ImageUpload,
+    images::LocalImage,
     moderator::{AdminPurgePerson, AdminPurgePersonForm},
     person::Person,
   },
@@ -31,7 +31,7 @@ pub async fn purge_person(
 
   let local_user = LocalUserView::read_person(&mut context.pool(), person_id).await?;
   let pictrs_uploads =
-    ImageUpload::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
+    LocalImage::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
 
   for upload in pictrs_uploads {
     delete_image_from_pictrs(&upload.pictrs_alias, &upload.pictrs_delete_token, &context)

crates/api_common/src/utils.rs

@@ -14,6 +14,7 @@ use lemmy_db_schema::{
     comment::{Comment, CommentUpdateForm},
     community::{Community, CommunityModerator, CommunityUpdateForm},
     email_verification::{EmailVerification, EmailVerificationForm},
+    images::RemoteImage,
     instance::Instance,
     local_site::LocalSite,
     local_site_rate_limit::LocalSiteRateLimit,
@@ -789,18 +790,24 @@ fn limit_expire_time(expires: DateTime<Utc>) -> LemmyResult<Option<DateTime<Utc>
   }
 }
 
-pub async fn process_markdown(text: &str, slur_regex: &Option<Regex>) -> LemmyResult<String> {
+pub async fn process_markdown(
+  text: &str,
+  slur_regex: &Option<Regex>,
+  context: &LemmyContext,
+) -> LemmyResult<String> {
   let text = remove_slurs(text, slur_regex);
-  let text = markdown_rewrite_image_links(text);
+  let (text, links) = markdown_rewrite_image_links(text);
+  RemoteImage::create_many(&mut context.pool(), links).await?;
   Ok(text)
 }
 
 pub async fn process_markdown_opt(
   text: &Option<String>,
   slur_regex: &Option<Regex>,
+  context: &LemmyContext,
 ) -> LemmyResult<Option<String>> {
   match text {
-    Some(t) => process_markdown(t, slur_regex).await.map(Some),
+    Some(t) => process_markdown(t, slur_regex, context).await.map(Some),
     None => Ok(None),
   }
 }

crates/api_crud/src/comment/create.rs

@@ -42,7 +42,8 @@ pub async fn create_comment(
 ) -> Result<Json<CommentResponse>, LemmyError> {
   let local_site = LocalSite::read(&mut context.pool()).await?;
 
-  let content = process_markdown(&data.content, &local_site_to_slur_regex(&local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&local_site);
+  let content = process_markdown(&data.content, &slur_regex, &context).await?;
   is_valid_body_field(&Some(content.clone()), false)?;
 
   // Check for a community ban

crates/api_crud/src/comment/update.rs

@@ -53,7 +53,8 @@ pub async fn update_comment(
   )
   .await?;
 
-  let content = process_markdown_opt(&data.content, &local_site_to_slur_regex(&local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&local_site);
+  let content = process_markdown_opt(&data.content, &slur_regex, &context).await?;
   is_valid_body_field(&content, false)?;
 
   let comment_id = data.comment_id;

crates/api_crud/src/community/create.rs

@@ -59,7 +59,7 @@ pub async fn create_community(
   let slur_regex = local_site_to_slur_regex(&local_site);
   check_slurs(&data.name, &slur_regex)?;
   check_slurs(&data.title, &slur_regex)?;
-  let description = process_markdown_opt(&data.description, &slur_regex).await?;
+  let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
 
   is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?;
   is_valid_body_field(&data.description, false)?;

crates/api_crud/src/community/update.rs

@@ -32,7 +32,7 @@ pub async fn update_community(
 
   let slur_regex = local_site_to_slur_regex(&local_site);
   check_slurs_opt(&data.title, &slur_regex)?;
-  let description = process_markdown_opt(&data.description, &slur_regex).await?;
+  let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
   is_valid_body_field(&data.description, false)?;
 
   let icon = diesel_option_overwrite_to_url(&data.icon)?;

crates/api_crud/src/post/create.rs

@@ -50,7 +50,7 @@ pub async fn create_post(
 
   let slur_regex = local_site_to_slur_regex(&local_site);
   check_slurs(&data.name, &slur_regex)?;
-  let body = process_markdown_opt(&data.body, &slur_regex).await?;
+  let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
   honeypot_check(&data.honeypot)?;
 
   let data_url = data.url.as_ref();

crates/api_crud/src/post/update.rs

@@ -43,7 +43,7 @@ pub async fn update_post(
 
   let slur_regex = local_site_to_slur_regex(&local_site);
   check_slurs_opt(&data.name, &slur_regex)?;
-  let body = process_markdown_opt(&data.body, &slur_regex).await?;
+  let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
 
   if let Some(name) = &data.name {
     is_valid_post_title(name)?;

crates/api_crud/src/private_message/create.rs

@@ -35,7 +35,8 @@ pub async fn create_private_message(
 ) -> Result<Json<PrivateMessageResponse>, LemmyError> {
   let local_site = LocalSite::read(&mut context.pool()).await?;
 
-  let content = process_markdown(&data.content, &local_site_to_slur_regex(&local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&local_site);
+  let content = process_markdown(&data.content, &slur_regex, &context).await?;
   is_valid_body_field(&Some(content.clone()), false)?;
 
   check_person_block(

crates/api_crud/src/private_message/update.rs

@@ -36,7 +36,8 @@ pub async fn update_private_message(
   }
 
   // Doing the update
-  let content = process_markdown(&data.content, &local_site_to_slur_regex(&local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&local_site);
+  let content = process_markdown(&data.content, &slur_regex, &context).await?;
   is_valid_body_field(&Some(content.clone()), false)?;
 
   let private_message_id = data.private_message_id;

crates/api_crud/src/site/create.rs

@@ -56,7 +56,8 @@ pub async fn create_site(
   let inbox_url = Some(generate_site_inbox_url(&actor_id)?);
   let keypair = generate_actor_keypair()?;
 
-  let sidebar = process_markdown_opt(&data.sidebar, &local_site_to_slur_regex(&local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&local_site);
+  let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
 
   let site_form = SiteUpdateForm {
     name: Some(data.name.clone()),

crates/api_crud/src/site/update.rs

@@ -59,7 +59,8 @@ pub async fn update_site(
     SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?;
   }
 
-  let sidebar = process_markdown_opt(&data.sidebar, &local_site_to_slur_regex(&local_site)).await?;
+  let slur_regex = local_site_to_slur_regex(&local_site);
+  let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
 
   let site_form = SiteUpdateForm {
     name: data.name.clone(),

crates/apub/src/activities/community/update.rs

@@ -8,7 +8,7 @@ use crate::{
   },
   activity_lists::AnnouncableActivities,
   insert_received_activity,
-  objects::{community::ApubCommunity, person::ApubPerson},
+  objects::{community::ApubCommunity, person::ApubPerson, read_from_string_or_source_opt},
   protocol::{activities::community::update::UpdateCommunity, InCommunity},
 };
 use activitypub_federation::{
@@ -18,8 +18,13 @@ use activitypub_federation::{
 };
 use lemmy_api_common::context::LemmyContext;
 use lemmy_db_schema::{
-  source::{activity::ActivitySendTargets, community::Community, person::Person},
+  source::{
+    activity::ActivitySendTargets,
+    community::{Community, CommunityUpdateForm},
+    person::Person,
+  },
   traits::Crud,
+  utils::naive_now,
 };
 use lemmy_utils::error::LemmyError;
 use url::Url;
@@ -85,7 +90,33 @@ impl ActivityHandler for UpdateCommunity {
   async fn receive(self, context: &Data<Self::DataType>) -> Result<(), LemmyError> {
     let community = self.community(context).await?;
 
-    let community_update_form = self.object.into_update_form();
+    let community_update_form = CommunityUpdateForm {
+      title: Some(self.object.name.unwrap_or(self.object.preferred_username)),
+      description: Some(read_from_string_or_source_opt(
+        &self.object.summary,
+        &None,
+        &self.object.source,
+      )),
+      removed: None,
+      published: self.object.published.map(Into::into),
+      updated: Some(self.object.updated.map(Into::into)),
+      deleted: None,
+      nsfw: Some(self.object.sensitive.unwrap_or(false)),
+      actor_id: Some(self.object.id.into()),
+      local: None,
+      private_key: None,
+      hidden: None,
+      public_key: Some(self.object.public_key.public_key_pem),
+      last_refreshed_at: Some(naive_now()),
+      icon: Some(self.object.icon.map(|i| i.url.into())),
+      banner: Some(self.object.image.map(|i| i.url.into())),
+      followers_url: Some(self.object.followers.into()),
+      inbox_url: Some(self.object.inbox.into()),
+      shared_inbox_url: Some(self.object.endpoints.map(|e| e.shared_inbox.into())),
+      moderators_url: self.object.attributed_to.map(Into::into),
+      posting_restricted_to_mods: self.object.posting_restricted_to_mods,
+      featured_url: self.object.featured.map(Into::into),
+    };
 
     Community::update(&mut context.pool(), community.id, &community_update_form).await?;
     Ok(())

crates/apub/src/objects/comment.rs

@@ -161,7 +161,7 @@ impl Object for ApubComment {
 
     let local_site = LocalSite::read(&mut context.pool()).await.ok();
     let slur_regex = &local_site_opt_to_slur_regex(&local_site);
-    let content = process_markdown(&content, slur_regex).await?;
+    let content = process_markdown(&content, slur_regex, &context).await?;
     let language_id =
       LanguageTag::to_language_id_single(note.language, &mut context.pool()).await?;
 

crates/apub/src/objects/community.rs

@@ -142,7 +142,7 @@ impl Object for ApubCommunity {
     let local_site = LocalSite::read(&mut context.pool()).await.ok();
     let slur_regex = &local_site_opt_to_slur_regex(&local_site);
     let description = read_from_string_or_source_opt(&group.summary, &None, &group.source);
-    let description = process_markdown_opt(&description, slur_regex).await?;
+    let description = process_markdown_opt(&description, slur_regex, &context).await?;
 
     let form = CommunityInsertForm {
       name: group.preferred_username.clone(),

crates/apub/src/objects/instance.rs

@@ -130,14 +130,14 @@ impl Object for ApubSite {
   }
 
   #[tracing::instrument(skip_all)]
-  async fn from_json(apub: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, LemmyError> {
+  async fn from_json(apub: Self::Kind, context: &Data<Self::DataType>) -> Result<Self, LemmyError> {
     let domain = apub.id.inner().domain().expect("group id has domain");
-    let instance = DbInstance::read_or_create(&mut data.pool(), domain.to_string()).await?;
+    let instance = DbInstance::read_or_create(&mut context.pool(), domain.to_string()).await?;
 
-    let local_site = LocalSite::read(&mut data.pool()).await.ok();
+    let local_site = LocalSite::read(&mut context.pool()).await.ok();
     let slur_regex = &local_site_opt_to_slur_regex(&local_site);
     let sidebar = read_from_string_or_source_opt(&apub.content, &None, &apub.source);
-    let sidebar = process_markdown_opt(&sidebar, slur_regex).await?;
+    let sidebar = process_markdown_opt(&sidebar, slur_regex, &context).await?;
 
     let site_form = SiteInsertForm {
       name: apub.name.clone(),
@@ -153,10 +153,11 @@ impl Object for ApubSite {
       private_key: None,
       instance_id: instance.id,
     };
-    let languages = LanguageTag::to_language_id_multiple(apub.language, &mut data.pool()).await?;
+    let languages =
+      LanguageTag::to_language_id_multiple(apub.language, &mut context.pool()).await?;
 
-    let site = Site::create(&mut data.pool(), &site_form).await?;
-    SiteLanguage::update(&mut data.pool(), languages, &site).await?;
+    let site = Site::create(&mut context.pool(), &site_form).await?;
+    SiteLanguage::update(&mut context.pool(), languages, &site).await?;
     Ok(site.into())
   }
 }

crates/apub/src/objects/person.rs

@@ -148,7 +148,7 @@ impl Object for ApubPerson {
     let local_site = LocalSite::read(&mut context.pool()).await.ok();
     let slur_regex = &local_site_opt_to_slur_regex(&local_site);
     let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source);
-    let bio = process_markdown_opt(&bio, slur_regex).await?;
+    let bio = process_markdown_opt(&bio, slur_regex, &context).await?;
 
     // Some Mastodon users have `name: ""` (empty string), need to convert that to `None`
     // https://github.com/mastodon/mastodon/issues/25233

crates/apub/src/objects/post.rs

@@ -237,7 +237,7 @@ impl Object for ApubPost {
       let slur_regex = &local_site_opt_to_slur_regex(&local_site);
 
       let body = read_from_string_or_source_opt(&page.content, &page.media_type, &page.source);
-      let body = process_markdown_opt(&body, slur_regex).await?;
+      let body = process_markdown_opt(&body, slur_regex, &context).await?;
       let language_id =
         LanguageTag::to_language_id_single(page.language, &mut context.pool()).await?;
 

crates/apub/src/objects/private_message.rs

@@ -128,7 +128,7 @@ impl Object for ApubPrivateMessage {
     let local_site = LocalSite::read(&mut context.pool()).await.ok();
     let slur_regex = &local_site_opt_to_slur_regex(&local_site);
     let content = read_from_string_or_source(&note.content, &None, &note.source);
-    let content = process_markdown(&content, slur_regex).await?;
+    let content = process_markdown(&content, slur_regex, &context).await?;
 
     let form = PrivateMessageInsertForm {
       creator_id: creator.id,

crates/apub/src/protocol/objects/group.rs

@@ -25,7 +25,6 @@ use activitypub_federation::{
 };
 use chrono::{DateTime, Utc};
 use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
-use lemmy_db_schema::{source::community::CommunityUpdateForm, utils::naive_now};
 use lemmy_utils::{
   error::LemmyError,
   utils::slurs::{check_slurs, check_slurs_opt},
@@ -89,34 +88,4 @@ impl Group {
     check_slurs_opt(&description, slur_regex)?;
     Ok(())
   }
-
-  pub(crate) fn into_update_form(self) -> CommunityUpdateForm {
-    CommunityUpdateForm {
-      title: Some(self.name.unwrap_or(self.preferred_username)),
-      description: Some(read_from_string_or_source_opt(
-        &self.summary,
-        &None,
-        &self.source,
-      )),
-      removed: None,
-      published: self.published.map(Into::into),
-      updated: Some(self.updated.map(Into::into)),
-      deleted: None,
-      nsfw: Some(self.sensitive.unwrap_or(false)),
-      actor_id: Some(self.id.into()),
-      local: None,
-      private_key: None,
-      hidden: None,
-      public_key: Some(self.public_key.public_key_pem),
-      last_refreshed_at: Some(naive_now()),
-      icon: Some(self.icon.map(|i| i.url.into())),
-      banner: Some(self.image.map(|i| i.url.into())),
-      followers_url: Some(self.followers.into()),
-      inbox_url: Some(self.inbox.into()),
-      shared_inbox_url: Some(self.endpoints.map(|e| e.shared_inbox.into())),
-      moderators_url: self.attributed_to.map(Into::into),
-      posting_restricted_to_mods: self.posting_restricted_to_mods,
-      featured_url: self.featured.map(Into::into),
-    }
-  }
 }

crates/db_schema/src/impls/image_upload.rs

@@ -1,47 +0,0 @@
-use crate::{
-  newtypes::{ImageUploadId, LocalUserId},
-  schema::image_upload::dsl::{image_upload, local_user_id, pictrs_alias},
-  source::image_upload::{ImageUpload, ImageUploadForm},
-  utils::{get_conn, DbPool},
-};
-use diesel::{insert_into, result::Error, ExpressionMethods, QueryDsl, Table};
-use diesel_async::RunQueryDsl;
-
-impl ImageUpload {
-  pub async fn create(pool: &mut DbPool<'_>, form: &ImageUploadForm) -> Result<Self, Error> {
-    let conn = &mut get_conn(pool).await?;
-    insert_into(image_upload)
-      .values(form)
-      .get_result::<Self>(conn)
-      .await
-  }
-
-  pub async fn get_all_by_local_user_id(
-    pool: &mut DbPool<'_>,
-    user_id: &LocalUserId,
-  ) -> Result<Vec<Self>, Error> {
-    let conn = &mut get_conn(pool).await?;
-    image_upload
-      .filter(local_user_id.eq(user_id))
-      .select(image_upload::all_columns())
-      .load::<ImageUpload>(conn)
-      .await
-  }
-
-  pub async fn delete(
-    pool: &mut DbPool<'_>,
-    image_upload_id: ImageUploadId,
-  ) -> Result<usize, Error> {
-    let conn = &mut get_conn(pool).await?;
-    diesel::delete(image_upload.find(image_upload_id))
-      .execute(conn)
-      .await
-  }
-
-  pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
-    let conn = &mut get_conn(pool).await?;
-    diesel::delete(image_upload.filter(pictrs_alias.eq(alias)))
-      .execute(conn)
-      .await
-  }
-}

crates/db_schema/src/impls/images.rs

@@ -0,0 +1,87 @@
+use crate::{
+  newtypes::{DbUrl, LocalImageId, LocalUserId},
+  schema::{
+    local_image::dsl::{local_image, local_user_id, pictrs_alias},
+    remote_image::dsl::{remote_image, link},
+  },
+  source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm},
+  utils::{get_conn, DbPool},
+};
+use diesel::{
+  dsl::exists,
+  insert_into,
+  result::Error,
+  select,
+  ExpressionMethods,
+  NotFound,
+  QueryDsl,
+  Table,
+};
+use diesel_async::RunQueryDsl;
+use url::Url;
+
+impl LocalImage {
+  pub async fn create(pool: &mut DbPool<'_>, form: &LocalImageForm) -> Result<Self, Error> {
+    let conn = &mut get_conn(pool).await?;
+    insert_into(local_image)
+      .values(form)
+      .get_result::<Self>(conn)
+      .await
+  }
+
+  pub async fn get_all_by_local_user_id(
+    pool: &mut DbPool<'_>,
+    user_id: &LocalUserId,
+  ) -> Result<Vec<Self>, Error> {
+    let conn = &mut get_conn(pool).await?;
+    local_image
+      .filter(local_user_id.eq(user_id))
+      .select(local_image::all_columns())
+      .load::<LocalImage>(conn)
+      .await
+  }
+
+  pub async fn delete(
+    pool: &mut DbPool<'_>,
+    image_upload_id: LocalImageId,
+  ) -> Result<usize, Error> {
+    let conn = &mut get_conn(pool).await?;
+    diesel::delete(local_image.find(image_upload_id))
+      .execute(conn)
+      .await
+  }
+
+  pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
+    let conn = &mut get_conn(pool).await?;
+    diesel::delete(local_image.filter(pictrs_alias.eq(alias)))
+      .execute(conn)
+      .await
+  }
+}
+
+impl RemoteImage {
+  pub async fn create_many(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<Self, Error> {
+    let conn = &mut get_conn(pool).await?;
+    let forms = links
+      .into_iter()
+      .map(|url| RemoteImageForm { link: url.into() })
+      .collect::<Vec<_>>();
+    insert_into(remote_image)
+      .values(forms)
+      .get_result::<Self>(conn)
+      .await
+  }
+
+  pub async fn validate(pool: &mut DbPool<'_>, link_: DbUrl) -> Result<(), Error> {
+    let conn = &mut get_conn(pool).await?;
+
+    let exists = select(exists(remote_image.filter((link).eq(link_))))
+      .get_result::<bool>(conn)
+      .await?;
+    if exists {
+      Ok(())
+    } else {
+      Err(NotFound)
+    }
+  }
+}

crates/db_schema/src/impls/mod.rs

@@ -10,7 +10,7 @@ pub mod custom_emoji;
 pub mod email_verification;
 pub mod federation_allowlist;
 pub mod federation_blocklist;
-pub mod image_upload;
+pub mod images;
 pub mod instance;
 pub mod instance_block;
 pub mod language;

crates/db_schema/src/newtypes.rs

@@ -141,7 +141,7 @@ pub struct CommentReplyId(i32);
 #[cfg_attr(feature = "full", derive(DieselNewType, TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// The Image Upload id.
-pub struct ImageUploadId(i32);
+pub struct LocalImageId(i32);
 
 #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
 #[cfg_attr(feature = "full", derive(DieselNewType, TS))]

crates/db_schema/src/schema.rs

@@ -314,7 +314,7 @@ diesel::table! {
 }
 
 diesel::table! {
-    image_upload (id) {
+    local_image (id) {
         id -> Int4,
         local_user_id -> Int4,
         pictrs_alias -> Text,
@@ -832,6 +832,14 @@ diesel::table! {
     }
 }
 
+diesel::table! {
+    remote_image (id) {
+        id -> Int4,
+        link -> Text,
+        published -> Timestamptz,
+    }
+}
+
 diesel::table! {
     secret (id) {
         id -> Int4,
@@ -949,7 +957,7 @@ diesel::joinable!(email_verification -> local_user (local_user_id));
 diesel::joinable!(federation_allowlist -> instance (instance_id));
 diesel::joinable!(federation_blocklist -> instance (instance_id));
 diesel::joinable!(federation_queue_state -> instance (instance_id));
-diesel::joinable!(image_upload -> local_user (local_user_id));
+diesel::joinable!(local_image -> local_user (local_user_id));
 diesel::joinable!(instance_block -> instance (instance_id));
 diesel::joinable!(instance_block -> person (person_id));
 diesel::joinable!(local_site -> site (site_id));
@@ -1029,7 +1037,7 @@ diesel::allow_tables_to_appear_in_same_query!(
     federation_allowlist,
     federation_blocklist,
     federation_queue_state,
-    image_upload,
+    local_image,
     instance,
     instance_block,
     language,
@@ -1067,6 +1075,7 @@ diesel::allow_tables_to_appear_in_same_query!(
     private_message_report,
     received_activity,
     registration_application,
+    remote_image,
     secret,
     sent_activity,
     site,

crates/db_schema/src/source/image_upload.rs

@@ -1,36 +0,0 @@
-use crate::newtypes::{ImageUploadId, LocalUserId};
-#[cfg(feature = "full")]
-use crate::schema::image_upload;
-use chrono::{DateTime, Utc};
-use serde::{Deserialize, Serialize};
-use serde_with::skip_serializing_none;
-use std::fmt::Debug;
-#[cfg(feature = "full")]
-use ts_rs::TS;
-use typed_builder::TypedBuilder;
-
-#[skip_serializing_none]
-#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
-#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))]
-#[cfg_attr(feature = "full", diesel(table_name = image_upload))]
-#[cfg_attr(feature = "full", ts(export))]
-#[cfg_attr(
-  feature = "full",
-  diesel(belongs_to(crate::source::local_user::LocalUser))
-)]
-pub struct ImageUpload {
-  pub id: ImageUploadId,
-  pub local_user_id: LocalUserId,
-  pub pictrs_alias: String,
-  pub pictrs_delete_token: String,
-  pub published: DateTime<Utc>,
-}
-
-#[derive(Debug, Clone, TypedBuilder)]
-#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
-#[cfg_attr(feature = "full", diesel(table_name = image_upload))]
-pub struct ImageUploadForm {
-  pub local_user_id: LocalUserId,
-  pub pictrs_alias: String,
-  pub pictrs_delete_token: String,
-}

crates/db_schema/src/source/images.rs

@@ -0,0 +1,49 @@
+use crate::newtypes::{DbUrl, LocalImageId, LocalUserId};
+#[cfg(feature = "full")]
+use crate::schema::{local_image, remote_image};
+use chrono::{DateTime, Utc};
+use serde_with::skip_serializing_none;
+use std::fmt::Debug;
+use typed_builder::TypedBuilder;
+
+#[skip_serializing_none]
+#[derive(PartialEq, Eq, Debug, Clone)]
+#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable))]
+#[cfg_attr(feature = "full", diesel(table_name = local_image))]
+#[cfg_attr(
+  feature = "full",
+  diesel(belongs_to(crate::source::local_user::LocalUser))
+)]
+pub struct LocalImage {
+  pub id: LocalImageId,
+  pub local_user_id: LocalUserId,
+  pub pictrs_alias: String,
+  pub pictrs_delete_token: String,
+  pub published: DateTime<Utc>,
+}
+
+#[derive(Debug, Clone, TypedBuilder)]
+#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
+#[cfg_attr(feature = "full", diesel(table_name = local_image))]
+pub struct LocalImageForm {
+  pub local_user_id: LocalUserId,
+  pub pictrs_alias: String,
+  pub pictrs_delete_token: String,
+}
+
+#[skip_serializing_none]
+#[derive(PartialEq, Eq, Debug, Clone)]
+#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
+#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
+pub struct RemoteImage {
+  pub id: i32,
+  pub link: DbUrl,
+  pub published: DateTime<Utc>,
+}
+
+#[derive(Debug, Clone, TypedBuilder)]
+#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
+#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
+pub struct RemoteImageForm {
+  pub link: DbUrl,
+}

crates/db_schema/src/source/mod.rs

@@ -15,7 +15,7 @@ pub mod custom_emoji_keyword;
 pub mod email_verification;
 pub mod federation_allowlist;
 pub mod federation_blocklist;
-pub mod image_upload;
+pub mod images;
 pub mod instance;
 pub mod instance_block;
 pub mod language;

crates/routes/src/image_proxy.rs

@@ -4,8 +4,10 @@ use actix_web::{
   HttpResponse,
 };
 use lemmy_api_common::context::LemmyContext;
+use lemmy_db_schema::source::images::RemoteImage;
 use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell};
 use serde::Deserialize;
+use url::Url;
 use urlencoding::decode;
 
 pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) {
@@ -25,9 +27,12 @@ async fn image_proxy(
   Query(params): Query<ImageProxyParams>,
   context: web::Data<LemmyContext>,
 ) -> LemmyResult<HttpResponse> {
-  // TODO: Check that url corresponds to a federated image so that this can't be abused as a proxy
+  let url = Url::parse(&decode(&params.url)?)?;
+
+  // Check that url corresponds to a federated image so that this can't be abused as a proxy
   // for arbitrary purposes.
-  let url = decode(&params.url)?.into_owned();
+  RemoteImage::validate(&mut context.pool(), url.clone().into()).await?;
+
   // TODO: Once pictrs 0.5 is out, use it for proxying like GET /image/original?proxy={url}
   //       https://git.asonix.dog/asonix/pict-rs/#api
   let image_response = context.client().get(url).send().await?;

crates/routes/src/images.rs

@@ -13,7 +13,7 @@ use actix_web::{
 use futures::stream::{Stream, StreamExt};
 use lemmy_api_common::context::LemmyContext;
 use lemmy_db_schema::source::{
-  image_upload::{ImageUpload, ImageUploadForm},
+  images::{LocalImage, LocalImageForm},
   local_site::LocalSite,
 };
 use lemmy_db_views::structs::LocalUserView;
@@ -112,12 +112,12 @@ async fn upload(
   let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
   if let Some(images) = &images.files {
     for uploaded_image in images {
-      let form = ImageUploadForm {
+      let form = LocalImageForm {
         local_user_id: local_user_view.local_user.id,
         pictrs_alias: uploaded_image.file.to_string(),
         pictrs_delete_token: uploaded_image.delete_token.to_string(),
       };
-      ImageUpload::create(&mut context.pool(), &form)
+      LocalImage::create(&mut context.pool(), &form)
         .await
         .map_err(error::ErrorBadRequest)?;
     }
@@ -213,7 +213,7 @@ async fn delete(
 
   let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
 
-  ImageUpload::delete_by_alias(&mut context.pool(), &file)
+  LocalImage::delete_by_alias(&mut context.pool(), &file)
     .await
     .map_err(error::ErrorBadRequest)?;
 

crates/utils/src/utils/markdown/mod.rs

@@ -35,9 +35,9 @@ pub fn markdown_to_html(text: &str) -> String {
 }
 
 /// Rewrites all links to remote domains in markdown, so they go through `/api/v3/image_proxy`.
-pub fn markdown_rewrite_image_links(mut src: String) -> String {
+pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec<Url>) {
   let ast = MARKDOWN_PARSER.parse(&src);
-  let mut links = vec![];
+  let mut links_offsets = vec![];
 
   // Walk the syntax tree to find positions of image links
   ast.walk(|node, _depth| {
@@ -46,15 +46,17 @@ pub fn markdown_rewrite_image_links(mut src: String) -> String {
       let start_offset = node_offsets.1 - image.url.len() - 1;
       let end_offset = node_offsets.1 - 1;
 
-      links.push((start_offset, end_offset));
+      links_offsets.push((start_offset, end_offset));
     }
   });
 
-  // Go through the collected links
-  while let Some((start, end)) = links.pop() {
+  let mut links = vec![];
+  // Go through the collected links in reverse order
+  while let Some((start, end)) = links_offsets.pop() {
     let url = &src.get(start..end).unwrap_or_default();
     match Url::parse(url) {
       Ok(parsed) => {
+        links.push(parsed.clone());
         // If link points to remote domain, replace with proxied link
         if parsed.domain() != Some(&SETTINGS.hostname) {
           let proxied = format!(
@@ -72,7 +74,7 @@ pub fn markdown_rewrite_image_links(mut src: String) -> String {
     }
   }
 
-  src
+  (src, links)
 }
 
 #[cfg(test)]
@@ -201,7 +203,7 @@ mod tests {
       let result = markdown_rewrite_image_links(input.to_string());
 
       assert_eq!(
-        result, expected,
+        result.0, expected,
         "Testing {}, with original input '{}'",
         msg, input
       );

migrations/2023-10-24-131607_proxy_links/down.sql

@@ -0,0 +1,3 @@
+drop table remote_image;
+
+alter table local_image rename to image_upload;

migrations/2023-10-24-131607_proxy_links/up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE remote_image (
+    id serial PRIMARY KEY,
+    link text not null unique,
+    published timestamptz DEFAULT now() NOT NULL
+);
+
+alter table image_upload rename to local_image;