From ef7942263250828de7a972ccd9bdf3edafaf9831 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 24 Oct 2023 15:56:04 +0200 Subject: [PATCH] add db table to validate proxied links --- crates/api/src/local_user/save_settings.rs | 4 +- crates/api/src/site/purge/person.rs | 4 +- crates/api_common/src/utils.rs | 13 ++- crates/api_crud/src/comment/create.rs | 3 +- crates/api_crud/src/comment/update.rs | 3 +- crates/api_crud/src/community/create.rs | 2 +- crates/api_crud/src/community/update.rs | 2 +- crates/api_crud/src/post/create.rs | 2 +- crates/api_crud/src/post/update.rs | 2 +- crates/api_crud/src/private_message/create.rs | 3 +- crates/api_crud/src/private_message/update.rs | 3 +- crates/api_crud/src/site/create.rs | 3 +- crates/api_crud/src/site/update.rs | 3 +- .../apub/src/activities/community/update.rs | 37 +++++++- crates/apub/src/objects/comment.rs | 2 +- crates/apub/src/objects/community.rs | 2 +- crates/apub/src/objects/instance.rs | 15 ++-- crates/apub/src/objects/person.rs | 2 +- crates/apub/src/objects/post.rs | 2 +- crates/apub/src/objects/private_message.rs | 2 +- crates/apub/src/protocol/objects/group.rs | 31 ------- crates/db_schema/src/impls/image_upload.rs | 47 ---------- crates/db_schema/src/impls/images.rs | 87 +++++++++++++++++++ crates/db_schema/src/impls/mod.rs | 2 +- crates/db_schema/src/newtypes.rs | 2 +- crates/db_schema/src/schema.rs | 15 +++- crates/db_schema/src/source/image_upload.rs | 36 -------- crates/db_schema/src/source/images.rs | 49 +++++++++++ crates/db_schema/src/source/mod.rs | 2 +- crates/routes/src/image_proxy.rs | 11 ++- crates/routes/src/images.rs | 8 +- crates/utils/src/utils/markdown/mod.rs | 16 ++-- .../2023-10-24-131607_proxy_links/down.sql | 3 + .../2023-10-24-131607_proxy_links/up.sql | 7 ++ 34 files changed, 259 insertions(+), 166 deletions(-) delete mode 100644 crates/db_schema/src/impls/image_upload.rs create mode 100644 crates/db_schema/src/impls/images.rs delete mode 100644 crates/db_schema/src/source/image_upload.rs create mode 100644 crates/db_schema/src/source/images.rs create mode 100644 migrations/2023-10-24-131607_proxy_links/down.sql create mode 100644 migrations/2023-10-24-131607_proxy_links/up.sql diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index 5d4565184..2948d97b6 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -28,8 +28,8 @@ pub async fn save_user_settings( ) -> Result, LemmyError> { let site_view = SiteView::read_local(&mut context.pool()).await?; - let bio = - process_markdown_opt(&data.bio, &local_site_to_slur_regex(&site_view.local_site)).await?; + let slur_regex = local_site_to_slur_regex(&site_view.local_site); + let bio = process_markdown_opt(&data.bio, &slur_regex, &context).await?; let avatar = diesel_option_overwrite_to_url(&data.avatar)?; let banner = diesel_option_overwrite_to_url(&data.banner)?; diff --git a/crates/api/src/site/purge/person.rs b/crates/api/src/site/purge/person.rs index c59e06931..2e581c2a9 100644 --- a/crates/api/src/site/purge/person.rs +++ b/crates/api/src/site/purge/person.rs @@ -8,7 +8,7 @@ use lemmy_api_common::{ }; use lemmy_db_schema::{ source::{ - image_upload::ImageUpload, + images::LocalImage, moderator::{AdminPurgePerson, AdminPurgePersonForm}, person::Person, }, @@ -31,7 +31,7 @@ pub async fn purge_person( let local_user = LocalUserView::read_person(&mut context.pool(), person_id).await?; let pictrs_uploads = - ImageUpload::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?; + LocalImage::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?; for upload in pictrs_uploads { delete_image_from_pictrs(&upload.pictrs_alias, &upload.pictrs_delete_token, &context) diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 7d49668f6..f2dbbb296 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -14,6 +14,7 @@ use lemmy_db_schema::{ comment::{Comment, CommentUpdateForm}, community::{Community, CommunityModerator, CommunityUpdateForm}, email_verification::{EmailVerification, EmailVerificationForm}, + images::RemoteImage, instance::Instance, local_site::LocalSite, local_site_rate_limit::LocalSiteRateLimit, @@ -789,18 +790,24 @@ fn limit_expire_time(expires: DateTime) -> LemmyResult } } -pub async fn process_markdown(text: &str, slur_regex: &Option) -> LemmyResult { +pub async fn process_markdown( + text: &str, + slur_regex: &Option, + context: &LemmyContext, +) -> LemmyResult { let text = remove_slurs(text, slur_regex); - let text = markdown_rewrite_image_links(text); + let (text, links) = markdown_rewrite_image_links(text); + RemoteImage::create_many(&mut context.pool(), links).await?; Ok(text) } pub async fn process_markdown_opt( text: &Option, slur_regex: &Option, + context: &LemmyContext, ) -> LemmyResult> { match text { - Some(t) => process_markdown(t, slur_regex).await.map(Some), + Some(t) => process_markdown(t, slur_regex, context).await.map(Some), None => Ok(None), } } diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index 8b5af72e0..64f8a3cea 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -42,7 +42,8 @@ pub async fn create_comment( ) -> Result, LemmyError> { let local_site = LocalSite::read(&mut context.pool()).await?; - let content = process_markdown(&data.content, &local_site_to_slur_regex(&local_site)).await?; + let slur_regex = local_site_to_slur_regex(&local_site); + let content = process_markdown(&data.content, &slur_regex, &context).await?; is_valid_body_field(&Some(content.clone()), false)?; // Check for a community ban diff --git a/crates/api_crud/src/comment/update.rs b/crates/api_crud/src/comment/update.rs index 538eae257..2d6bf79be 100644 --- a/crates/api_crud/src/comment/update.rs +++ b/crates/api_crud/src/comment/update.rs @@ -53,7 +53,8 @@ pub async fn update_comment( ) .await?; - let content = process_markdown_opt(&data.content, &local_site_to_slur_regex(&local_site)).await?; + let slur_regex = local_site_to_slur_regex(&local_site); + let content = process_markdown_opt(&data.content, &slur_regex, &context).await?; is_valid_body_field(&content, false)?; let comment_id = data.comment_id; diff --git a/crates/api_crud/src/community/create.rs b/crates/api_crud/src/community/create.rs index ed6d419e7..1110995aa 100644 --- a/crates/api_crud/src/community/create.rs +++ b/crates/api_crud/src/community/create.rs @@ -59,7 +59,7 @@ pub async fn create_community( let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.name, &slur_regex)?; check_slurs(&data.title, &slur_regex)?; - let description = process_markdown_opt(&data.description, &slur_regex).await?; + let description = process_markdown_opt(&data.description, &slur_regex, &context).await?; is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?; is_valid_body_field(&data.description, false)?; diff --git a/crates/api_crud/src/community/update.rs b/crates/api_crud/src/community/update.rs index dd7abd18b..df6d6570f 100644 --- a/crates/api_crud/src/community/update.rs +++ b/crates/api_crud/src/community/update.rs @@ -32,7 +32,7 @@ pub async fn update_community( let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.title, &slur_regex)?; - let description = process_markdown_opt(&data.description, &slur_regex).await?; + let description = process_markdown_opt(&data.description, &slur_regex, &context).await?; is_valid_body_field(&data.description, false)?; let icon = diesel_option_overwrite_to_url(&data.icon)?; diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index d45ed2cad..3ffa54685 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -50,7 +50,7 @@ pub async fn create_post( let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.name, &slur_regex)?; - let body = process_markdown_opt(&data.body, &slur_regex).await?; + let body = process_markdown_opt(&data.body, &slur_regex, &context).await?; honeypot_check(&data.honeypot)?; let data_url = data.url.as_ref(); diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index 1f9f3a82c..73a4962f9 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -43,7 +43,7 @@ pub async fn update_post( let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.name, &slur_regex)?; - let body = process_markdown_opt(&data.body, &slur_regex).await?; + let body = process_markdown_opt(&data.body, &slur_regex, &context).await?; if let Some(name) = &data.name { is_valid_post_title(name)?; diff --git a/crates/api_crud/src/private_message/create.rs b/crates/api_crud/src/private_message/create.rs index 87be2b381..c4832ec70 100644 --- a/crates/api_crud/src/private_message/create.rs +++ b/crates/api_crud/src/private_message/create.rs @@ -35,7 +35,8 @@ pub async fn create_private_message( ) -> Result, LemmyError> { let local_site = LocalSite::read(&mut context.pool()).await?; - let content = process_markdown(&data.content, &local_site_to_slur_regex(&local_site)).await?; + let slur_regex = local_site_to_slur_regex(&local_site); + let content = process_markdown(&data.content, &slur_regex, &context).await?; is_valid_body_field(&Some(content.clone()), false)?; check_person_block( diff --git a/crates/api_crud/src/private_message/update.rs b/crates/api_crud/src/private_message/update.rs index b33481c7a..dfcf522a8 100644 --- a/crates/api_crud/src/private_message/update.rs +++ b/crates/api_crud/src/private_message/update.rs @@ -36,7 +36,8 @@ pub async fn update_private_message( } // Doing the update - let content = process_markdown(&data.content, &local_site_to_slur_regex(&local_site)).await?; + let slur_regex = local_site_to_slur_regex(&local_site); + let content = process_markdown(&data.content, &slur_regex, &context).await?; is_valid_body_field(&Some(content.clone()), false)?; let private_message_id = data.private_message_id; diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index f6fd09711..2f3322512 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -56,7 +56,8 @@ pub async fn create_site( let inbox_url = Some(generate_site_inbox_url(&actor_id)?); let keypair = generate_actor_keypair()?; - let sidebar = process_markdown_opt(&data.sidebar, &local_site_to_slur_regex(&local_site)).await?; + let slur_regex = local_site_to_slur_regex(&local_site); + let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?; let site_form = SiteUpdateForm { name: Some(data.name.clone()), diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index e921187da..97bdb108a 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -59,7 +59,8 @@ pub async fn update_site( SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?; } - let sidebar = process_markdown_opt(&data.sidebar, &local_site_to_slur_regex(&local_site)).await?; + let slur_regex = local_site_to_slur_regex(&local_site); + let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?; let site_form = SiteUpdateForm { name: data.name.clone(), diff --git a/crates/apub/src/activities/community/update.rs b/crates/apub/src/activities/community/update.rs index 11040f6b9..29ee83437 100644 --- a/crates/apub/src/activities/community/update.rs +++ b/crates/apub/src/activities/community/update.rs @@ -8,7 +8,7 @@ use crate::{ }, activity_lists::AnnouncableActivities, insert_received_activity, - objects::{community::ApubCommunity, person::ApubPerson}, + objects::{community::ApubCommunity, person::ApubPerson, read_from_string_or_source_opt}, protocol::{activities::community::update::UpdateCommunity, InCommunity}, }; use activitypub_federation::{ @@ -18,8 +18,13 @@ use activitypub_federation::{ }; use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::{ - source::{activity::ActivitySendTargets, community::Community, person::Person}, + source::{ + activity::ActivitySendTargets, + community::{Community, CommunityUpdateForm}, + person::Person, + }, traits::Crud, + utils::naive_now, }; use lemmy_utils::error::LemmyError; use url::Url; @@ -85,7 +90,33 @@ impl ActivityHandler for UpdateCommunity { async fn receive(self, context: &Data) -> Result<(), LemmyError> { let community = self.community(context).await?; - let community_update_form = self.object.into_update_form(); + let community_update_form = CommunityUpdateForm { + title: Some(self.object.name.unwrap_or(self.object.preferred_username)), + description: Some(read_from_string_or_source_opt( + &self.object.summary, + &None, + &self.object.source, + )), + removed: None, + published: self.object.published.map(Into::into), + updated: Some(self.object.updated.map(Into::into)), + deleted: None, + nsfw: Some(self.object.sensitive.unwrap_or(false)), + actor_id: Some(self.object.id.into()), + local: None, + private_key: None, + hidden: None, + public_key: Some(self.object.public_key.public_key_pem), + last_refreshed_at: Some(naive_now()), + icon: Some(self.object.icon.map(|i| i.url.into())), + banner: Some(self.object.image.map(|i| i.url.into())), + followers_url: Some(self.object.followers.into()), + inbox_url: Some(self.object.inbox.into()), + shared_inbox_url: Some(self.object.endpoints.map(|e| e.shared_inbox.into())), + moderators_url: self.object.attributed_to.map(Into::into), + posting_restricted_to_mods: self.object.posting_restricted_to_mods, + featured_url: self.object.featured.map(Into::into), + }; Community::update(&mut context.pool(), community.id, &community_update_form).await?; Ok(()) diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index 1feeaa70f..c90804b59 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -161,7 +161,7 @@ impl Object for ApubComment { let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); - let content = process_markdown(&content, slur_regex).await?; + let content = process_markdown(&content, slur_regex, &context).await?; let language_id = LanguageTag::to_language_id_single(note.language, &mut context.pool()).await?; diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 5f0f63b1f..d5557c6c1 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -142,7 +142,7 @@ impl Object for ApubCommunity { let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); let description = read_from_string_or_source_opt(&group.summary, &None, &group.source); - let description = process_markdown_opt(&description, slur_regex).await?; + let description = process_markdown_opt(&description, slur_regex, &context).await?; let form = CommunityInsertForm { name: group.preferred_username.clone(), diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index b951e970d..5c2becf4f 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -130,14 +130,14 @@ impl Object for ApubSite { } #[tracing::instrument(skip_all)] - async fn from_json(apub: Self::Kind, data: &Data) -> Result { + async fn from_json(apub: Self::Kind, context: &Data) -> Result { let domain = apub.id.inner().domain().expect("group id has domain"); - let instance = DbInstance::read_or_create(&mut data.pool(), domain.to_string()).await?; + let instance = DbInstance::read_or_create(&mut context.pool(), domain.to_string()).await?; - let local_site = LocalSite::read(&mut data.pool()).await.ok(); + let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); let sidebar = read_from_string_or_source_opt(&apub.content, &None, &apub.source); - let sidebar = process_markdown_opt(&sidebar, slur_regex).await?; + let sidebar = process_markdown_opt(&sidebar, slur_regex, &context).await?; let site_form = SiteInsertForm { name: apub.name.clone(), @@ -153,10 +153,11 @@ impl Object for ApubSite { private_key: None, instance_id: instance.id, }; - let languages = LanguageTag::to_language_id_multiple(apub.language, &mut data.pool()).await?; + let languages = + LanguageTag::to_language_id_multiple(apub.language, &mut context.pool()).await?; - let site = Site::create(&mut data.pool(), &site_form).await?; - SiteLanguage::update(&mut data.pool(), languages, &site).await?; + let site = Site::create(&mut context.pool(), &site_form).await?; + SiteLanguage::update(&mut context.pool(), languages, &site).await?; Ok(site.into()) } } diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index b7d53dee8..6a251d4f2 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -148,7 +148,7 @@ impl Object for ApubPerson { let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source); - let bio = process_markdown_opt(&bio, slur_regex).await?; + let bio = process_markdown_opt(&bio, slur_regex, &context).await?; // Some Mastodon users have `name: ""` (empty string), need to convert that to `None` // https://github.com/mastodon/mastodon/issues/25233 diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 46e3542b3..237d3835e 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -237,7 +237,7 @@ impl Object for ApubPost { let slur_regex = &local_site_opt_to_slur_regex(&local_site); let body = read_from_string_or_source_opt(&page.content, &page.media_type, &page.source); - let body = process_markdown_opt(&body, slur_regex).await?; + let body = process_markdown_opt(&body, slur_regex, &context).await?; let language_id = LanguageTag::to_language_id_single(page.language, &mut context.pool()).await?; diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 23af75704..12c5d59d0 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -128,7 +128,7 @@ impl Object for ApubPrivateMessage { let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); let content = read_from_string_or_source(¬e.content, &None, ¬e.source); - let content = process_markdown(&content, slur_regex).await?; + let content = process_markdown(&content, slur_regex, &context).await?; let form = PrivateMessageInsertForm { creator_id: creator.id, diff --git a/crates/apub/src/protocol/objects/group.rs b/crates/apub/src/protocol/objects/group.rs index 9447d7769..2b70d1fa5 100644 --- a/crates/apub/src/protocol/objects/group.rs +++ b/crates/apub/src/protocol/objects/group.rs @@ -25,7 +25,6 @@ use activitypub_federation::{ }; use chrono::{DateTime, Utc}; use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex}; -use lemmy_db_schema::{source::community::CommunityUpdateForm, utils::naive_now}; use lemmy_utils::{ error::LemmyError, utils::slurs::{check_slurs, check_slurs_opt}, @@ -89,34 +88,4 @@ impl Group { check_slurs_opt(&description, slur_regex)?; Ok(()) } - - pub(crate) fn into_update_form(self) -> CommunityUpdateForm { - CommunityUpdateForm { - title: Some(self.name.unwrap_or(self.preferred_username)), - description: Some(read_from_string_or_source_opt( - &self.summary, - &None, - &self.source, - )), - removed: None, - published: self.published.map(Into::into), - updated: Some(self.updated.map(Into::into)), - deleted: None, - nsfw: Some(self.sensitive.unwrap_or(false)), - actor_id: Some(self.id.into()), - local: None, - private_key: None, - hidden: None, - public_key: Some(self.public_key.public_key_pem), - last_refreshed_at: Some(naive_now()), - icon: Some(self.icon.map(|i| i.url.into())), - banner: Some(self.image.map(|i| i.url.into())), - followers_url: Some(self.followers.into()), - inbox_url: Some(self.inbox.into()), - shared_inbox_url: Some(self.endpoints.map(|e| e.shared_inbox.into())), - moderators_url: self.attributed_to.map(Into::into), - posting_restricted_to_mods: self.posting_restricted_to_mods, - featured_url: self.featured.map(Into::into), - } - } } diff --git a/crates/db_schema/src/impls/image_upload.rs b/crates/db_schema/src/impls/image_upload.rs deleted file mode 100644 index 58edbf2e3..000000000 --- a/crates/db_schema/src/impls/image_upload.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::{ - newtypes::{ImageUploadId, LocalUserId}, - schema::image_upload::dsl::{image_upload, local_user_id, pictrs_alias}, - source::image_upload::{ImageUpload, ImageUploadForm}, - utils::{get_conn, DbPool}, -}; -use diesel::{insert_into, result::Error, ExpressionMethods, QueryDsl, Table}; -use diesel_async::RunQueryDsl; - -impl ImageUpload { - pub async fn create(pool: &mut DbPool<'_>, form: &ImageUploadForm) -> Result { - let conn = &mut get_conn(pool).await?; - insert_into(image_upload) - .values(form) - .get_result::(conn) - .await - } - - pub async fn get_all_by_local_user_id( - pool: &mut DbPool<'_>, - user_id: &LocalUserId, - ) -> Result, Error> { - let conn = &mut get_conn(pool).await?; - image_upload - .filter(local_user_id.eq(user_id)) - .select(image_upload::all_columns()) - .load::(conn) - .await - } - - pub async fn delete( - pool: &mut DbPool<'_>, - image_upload_id: ImageUploadId, - ) -> Result { - let conn = &mut get_conn(pool).await?; - diesel::delete(image_upload.find(image_upload_id)) - .execute(conn) - .await - } - - pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result { - let conn = &mut get_conn(pool).await?; - diesel::delete(image_upload.filter(pictrs_alias.eq(alias))) - .execute(conn) - .await - } -} diff --git a/crates/db_schema/src/impls/images.rs b/crates/db_schema/src/impls/images.rs new file mode 100644 index 000000000..7533ca603 --- /dev/null +++ b/crates/db_schema/src/impls/images.rs @@ -0,0 +1,87 @@ +use crate::{ + newtypes::{DbUrl, LocalImageId, LocalUserId}, + schema::{ + local_image::dsl::{local_image, local_user_id, pictrs_alias}, + remote_image::dsl::{remote_image, link}, + }, + source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm}, + utils::{get_conn, DbPool}, +}; +use diesel::{ + dsl::exists, + insert_into, + result::Error, + select, + ExpressionMethods, + NotFound, + QueryDsl, + Table, +}; +use diesel_async::RunQueryDsl; +use url::Url; + +impl LocalImage { + pub async fn create(pool: &mut DbPool<'_>, form: &LocalImageForm) -> Result { + let conn = &mut get_conn(pool).await?; + insert_into(local_image) + .values(form) + .get_result::(conn) + .await + } + + pub async fn get_all_by_local_user_id( + pool: &mut DbPool<'_>, + user_id: &LocalUserId, + ) -> Result, Error> { + let conn = &mut get_conn(pool).await?; + local_image + .filter(local_user_id.eq(user_id)) + .select(local_image::all_columns()) + .load::(conn) + .await + } + + pub async fn delete( + pool: &mut DbPool<'_>, + image_upload_id: LocalImageId, + ) -> Result { + let conn = &mut get_conn(pool).await?; + diesel::delete(local_image.find(image_upload_id)) + .execute(conn) + .await + } + + pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result { + let conn = &mut get_conn(pool).await?; + diesel::delete(local_image.filter(pictrs_alias.eq(alias))) + .execute(conn) + .await + } +} + +impl RemoteImage { + pub async fn create_many(pool: &mut DbPool<'_>, links: Vec) -> Result { + let conn = &mut get_conn(pool).await?; + let forms = links + .into_iter() + .map(|url| RemoteImageForm { link: url.into() }) + .collect::>(); + insert_into(remote_image) + .values(forms) + .get_result::(conn) + .await + } + + pub async fn validate(pool: &mut DbPool<'_>, link_: DbUrl) -> Result<(), Error> { + let conn = &mut get_conn(pool).await?; + + let exists = select(exists(remote_image.filter((link).eq(link_)))) + .get_result::(conn) + .await?; + if exists { + Ok(()) + } else { + Err(NotFound) + } + } +} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index 3cf0f1066..3455abb25 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -10,7 +10,7 @@ pub mod custom_emoji; pub mod email_verification; pub mod federation_allowlist; pub mod federation_blocklist; -pub mod image_upload; +pub mod images; pub mod instance; pub mod instance_block; pub mod language; diff --git a/crates/db_schema/src/newtypes.rs b/crates/db_schema/src/newtypes.rs index 555b98256..6c55e516b 100644 --- a/crates/db_schema/src/newtypes.rs +++ b/crates/db_schema/src/newtypes.rs @@ -141,7 +141,7 @@ pub struct CommentReplyId(i32); #[cfg_attr(feature = "full", derive(DieselNewType, TS))] #[cfg_attr(feature = "full", ts(export))] /// The Image Upload id. -pub struct ImageUploadId(i32); +pub struct LocalImageId(i32); #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)] #[cfg_attr(feature = "full", derive(DieselNewType, TS))] diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 440cb09fa..bf16d6efd 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -314,7 +314,7 @@ diesel::table! { } diesel::table! { - image_upload (id) { + local_image (id) { id -> Int4, local_user_id -> Int4, pictrs_alias -> Text, @@ -832,6 +832,14 @@ diesel::table! { } } +diesel::table! { + remote_image (id) { + id -> Int4, + link -> Text, + published -> Timestamptz, + } +} + diesel::table! { secret (id) { id -> Int4, @@ -949,7 +957,7 @@ diesel::joinable!(email_verification -> local_user (local_user_id)); diesel::joinable!(federation_allowlist -> instance (instance_id)); diesel::joinable!(federation_blocklist -> instance (instance_id)); diesel::joinable!(federation_queue_state -> instance (instance_id)); -diesel::joinable!(image_upload -> local_user (local_user_id)); +diesel::joinable!(local_image -> local_user (local_user_id)); diesel::joinable!(instance_block -> instance (instance_id)); diesel::joinable!(instance_block -> person (person_id)); diesel::joinable!(local_site -> site (site_id)); @@ -1029,7 +1037,7 @@ diesel::allow_tables_to_appear_in_same_query!( federation_allowlist, federation_blocklist, federation_queue_state, - image_upload, + local_image, instance, instance_block, language, @@ -1067,6 +1075,7 @@ diesel::allow_tables_to_appear_in_same_query!( private_message_report, received_activity, registration_application, + remote_image, secret, sent_activity, site, diff --git a/crates/db_schema/src/source/image_upload.rs b/crates/db_schema/src/source/image_upload.rs deleted file mode 100644 index 0a3c4d6c4..000000000 --- a/crates/db_schema/src/source/image_upload.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::newtypes::{ImageUploadId, LocalUserId}; -#[cfg(feature = "full")] -use crate::schema::image_upload; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; -use std::fmt::Debug; -#[cfg(feature = "full")] -use ts_rs::TS; -use typed_builder::TypedBuilder; - -#[skip_serializing_none] -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))] -#[cfg_attr(feature = "full", diesel(table_name = image_upload))] -#[cfg_attr(feature = "full", ts(export))] -#[cfg_attr( - feature = "full", - diesel(belongs_to(crate::source::local_user::LocalUser)) -)] -pub struct ImageUpload { - pub id: ImageUploadId, - pub local_user_id: LocalUserId, - pub pictrs_alias: String, - pub pictrs_delete_token: String, - pub published: DateTime, -} - -#[derive(Debug, Clone, TypedBuilder)] -#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] -#[cfg_attr(feature = "full", diesel(table_name = image_upload))] -pub struct ImageUploadForm { - pub local_user_id: LocalUserId, - pub pictrs_alias: String, - pub pictrs_delete_token: String, -} diff --git a/crates/db_schema/src/source/images.rs b/crates/db_schema/src/source/images.rs new file mode 100644 index 000000000..60386f2be --- /dev/null +++ b/crates/db_schema/src/source/images.rs @@ -0,0 +1,49 @@ +use crate::newtypes::{DbUrl, LocalImageId, LocalUserId}; +#[cfg(feature = "full")] +use crate::schema::{local_image, remote_image}; +use chrono::{DateTime, Utc}; +use serde_with::skip_serializing_none; +use std::fmt::Debug; +use typed_builder::TypedBuilder; + +#[skip_serializing_none] +#[derive(PartialEq, Eq, Debug, Clone)] +#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable))] +#[cfg_attr(feature = "full", diesel(table_name = local_image))] +#[cfg_attr( + feature = "full", + diesel(belongs_to(crate::source::local_user::LocalUser)) +)] +pub struct LocalImage { + pub id: LocalImageId, + pub local_user_id: LocalUserId, + pub pictrs_alias: String, + pub pictrs_delete_token: String, + pub published: DateTime, +} + +#[derive(Debug, Clone, TypedBuilder)] +#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] +#[cfg_attr(feature = "full", diesel(table_name = local_image))] +pub struct LocalImageForm { + pub local_user_id: LocalUserId, + pub pictrs_alias: String, + pub pictrs_delete_token: String, +} + +#[skip_serializing_none] +#[derive(PartialEq, Eq, Debug, Clone)] +#[cfg_attr(feature = "full", derive(Queryable, Identifiable))] +#[cfg_attr(feature = "full", diesel(table_name = remote_image))] +pub struct RemoteImage { + pub id: i32, + pub link: DbUrl, + pub published: DateTime, +} + +#[derive(Debug, Clone, TypedBuilder)] +#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] +#[cfg_attr(feature = "full", diesel(table_name = remote_image))] +pub struct RemoteImageForm { + pub link: DbUrl, +} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index 9879ef35f..76e84cd37 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -15,7 +15,7 @@ pub mod custom_emoji_keyword; pub mod email_verification; pub mod federation_allowlist; pub mod federation_blocklist; -pub mod image_upload; +pub mod images; pub mod instance; pub mod instance_block; pub mod language; diff --git a/crates/routes/src/image_proxy.rs b/crates/routes/src/image_proxy.rs index 92fe70b51..50d4f4a30 100644 --- a/crates/routes/src/image_proxy.rs +++ b/crates/routes/src/image_proxy.rs @@ -4,8 +4,10 @@ use actix_web::{ HttpResponse, }; use lemmy_api_common::context::LemmyContext; +use lemmy_db_schema::source::images::RemoteImage; use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell}; use serde::Deserialize; +use url::Url; use urlencoding::decode; pub fn config(cfg: &mut ServiceConfig, rate_limit: &RateLimitCell) { @@ -25,9 +27,12 @@ async fn image_proxy( Query(params): Query, context: web::Data, ) -> LemmyResult { - // TODO: Check that url corresponds to a federated image so that this can't be abused as a proxy - // for arbitrary purposes. - let url = decode(¶ms.url)?.into_owned(); + let url = Url::parse(&decode(¶ms.url)?)?; + + // Check that url corresponds to a federated image so that this can't be abused as a proxy + // for arbitrary purposes. + RemoteImage::validate(&mut context.pool(), url.clone().into()).await?; + // TODO: Once pictrs 0.5 is out, use it for proxying like GET /image/original?proxy={url} // https://git.asonix.dog/asonix/pict-rs/#api let image_response = context.client().get(url).send().await?; diff --git a/crates/routes/src/images.rs b/crates/routes/src/images.rs index 2b4c9b579..a3f19d6d9 100644 --- a/crates/routes/src/images.rs +++ b/crates/routes/src/images.rs @@ -13,7 +13,7 @@ use actix_web::{ use futures::stream::{Stream, StreamExt}; use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::source::{ - image_upload::{ImageUpload, ImageUploadForm}, + images::{LocalImage, LocalImageForm}, local_site::LocalSite, }; use lemmy_db_views::structs::LocalUserView; @@ -112,12 +112,12 @@ async fn upload( let images = res.json::().await.map_err(error::ErrorBadRequest)?; if let Some(images) = &images.files { for uploaded_image in images { - let form = ImageUploadForm { + let form = LocalImageForm { local_user_id: local_user_view.local_user.id, pictrs_alias: uploaded_image.file.to_string(), pictrs_delete_token: uploaded_image.delete_token.to_string(), }; - ImageUpload::create(&mut context.pool(), &form) + LocalImage::create(&mut context.pool(), &form) .await .map_err(error::ErrorBadRequest)?; } @@ -213,7 +213,7 @@ async fn delete( let res = client_req.send().await.map_err(error::ErrorBadRequest)?; - ImageUpload::delete_by_alias(&mut context.pool(), &file) + LocalImage::delete_by_alias(&mut context.pool(), &file) .await .map_err(error::ErrorBadRequest)?; diff --git a/crates/utils/src/utils/markdown/mod.rs b/crates/utils/src/utils/markdown/mod.rs index f4a6dc6db..6a3eb147f 100644 --- a/crates/utils/src/utils/markdown/mod.rs +++ b/crates/utils/src/utils/markdown/mod.rs @@ -35,9 +35,9 @@ pub fn markdown_to_html(text: &str) -> String { } /// Rewrites all links to remote domains in markdown, so they go through `/api/v3/image_proxy`. -pub fn markdown_rewrite_image_links(mut src: String) -> String { +pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec) { let ast = MARKDOWN_PARSER.parse(&src); - let mut links = vec![]; + let mut links_offsets = vec![]; // Walk the syntax tree to find positions of image links ast.walk(|node, _depth| { @@ -46,15 +46,17 @@ pub fn markdown_rewrite_image_links(mut src: String) -> String { let start_offset = node_offsets.1 - image.url.len() - 1; let end_offset = node_offsets.1 - 1; - links.push((start_offset, end_offset)); + links_offsets.push((start_offset, end_offset)); } }); - // Go through the collected links - while let Some((start, end)) = links.pop() { + let mut links = vec![]; + // Go through the collected links in reverse order + while let Some((start, end)) = links_offsets.pop() { let url = &src.get(start..end).unwrap_or_default(); match Url::parse(url) { Ok(parsed) => { + links.push(parsed.clone()); // If link points to remote domain, replace with proxied link if parsed.domain() != Some(&SETTINGS.hostname) { let proxied = format!( @@ -72,7 +74,7 @@ pub fn markdown_rewrite_image_links(mut src: String) -> String { } } - src + (src, links) } #[cfg(test)] @@ -201,7 +203,7 @@ mod tests { let result = markdown_rewrite_image_links(input.to_string()); assert_eq!( - result, expected, + result.0, expected, "Testing {}, with original input '{}'", msg, input ); diff --git a/migrations/2023-10-24-131607_proxy_links/down.sql b/migrations/2023-10-24-131607_proxy_links/down.sql new file mode 100644 index 000000000..c1bf58bc0 --- /dev/null +++ b/migrations/2023-10-24-131607_proxy_links/down.sql @@ -0,0 +1,3 @@ +drop table remote_image; + +alter table local_image rename to image_upload; \ No newline at end of file diff --git a/migrations/2023-10-24-131607_proxy_links/up.sql b/migrations/2023-10-24-131607_proxy_links/up.sql new file mode 100644 index 000000000..3d909c9d7 --- /dev/null +++ b/migrations/2023-10-24-131607_proxy_links/up.sql @@ -0,0 +1,7 @@ +CREATE TABLE remote_image ( + id serial PRIMARY KEY, + link text not null unique, + published timestamptz DEFAULT now() NOT NULL +); + +alter table image_upload rename to local_image;