diff --git a/.woodpecker.yml b/.woodpecker.yml index a51a1499d..4bac67169 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -2,7 +2,7 @@ # See https://github.com/woodpecker-ci/woodpecker/issues/1677 variables: - - &rust_image "rust:1.75" + - &rust_image "rust:1.76" - &install_pnpm "corepack enable pnpm" - &slow_check_paths - path: diff --git a/crates/api_common/src/post.rs b/crates/api_common/src/post.rs index 36f971ca7..1db07e451 100644 --- a/crates/api_common/src/post.rs +++ b/crates/api_common/src/post.rs @@ -28,6 +28,9 @@ pub struct CreatePost { pub honeypot: Option, pub nsfw: Option, pub language_id: Option, + #[cfg_attr(feature = "full", ts(type = "string"))] + /// Instead of fetching a thumbnail, use a custom one. + pub custom_thumbnail: Option, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -114,6 +117,9 @@ pub struct EditPost { pub body: Option, pub nsfw: Option, pub language_id: Option, + #[cfg_attr(feature = "full", ts(type = "string"))] + /// Instead of fetching a thumbnail, use a custom one. + pub custom_thumbnail: Option, } #[derive(Debug, Serialize, Deserialize, Clone, Copy, Default, PartialEq, Eq, Hash)] diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 910454d5b..a5a9c013f 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -57,10 +57,12 @@ pub async fn create_post( let data_url = data.url.as_ref(); let url = data_url.map(clean_url_params); // TODO no good way to handle a "clear" + let custom_thumbnail = data.custom_thumbnail.as_ref().map(clean_url_params); is_valid_post_title(&data.name)?; is_valid_body_field(&body, true)?; - check_url_scheme(&data.url)?; + check_url_scheme(&url)?; + check_url_scheme(&custom_thumbnail)?; check_community_user_action( &local_user_view.person, @@ -84,9 +86,17 @@ pub async fn create_post( } } + // Only generate the thumbnail if there's no custom thumbnail provided, + // otherwise it will save it in pictrs + let generate_thumbnail = custom_thumbnail.is_none(); + // Fetch post links and pictrs cached image - let metadata = fetch_link_metadata_opt(url.as_ref(), true, &context).await; + let metadata = fetch_link_metadata_opt(url.as_ref(), generate_thumbnail, &context).await; let url = proxy_image_link_opt_apub(url, &context).await?; + let thumbnail_url = proxy_image_link_opt_apub(custom_thumbnail, &context) + .await? + .map(Into::into) + .or(metadata.thumbnail); // Only need to check if language is allowed in case user set it explicitly. When using default // language, it already only returns allowed languages. @@ -121,7 +131,7 @@ pub async fn create_post( .embed_description(metadata.opengraph_data.description) .embed_video_url(metadata.opengraph_data.embed_video_url) .language_id(language_id) - .thumbnail_url(metadata.thumbnail) + .thumbnail_url(thumbnail_url) .build(); let inserted_post = Post::create(&mut context.pool(), &post_form) diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index c36718615..e858d9b30 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -43,6 +43,7 @@ pub async fn update_post( // TODO No good way to handle a clear. // Issue link: https://github.com/LemmyNet/lemmy/issues/2287 let url = data.url.as_ref().map(clean_url_params); + let custom_thumbnail = data.custom_thumbnail.as_ref().map(clean_url_params); let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.name, &slur_regex)?; @@ -53,7 +54,8 @@ pub async fn update_post( } is_valid_body_field(&body, true)?; - check_url_scheme(&data.url)?; + check_url_scheme(&url)?; + check_url_scheme(&custom_thumbnail)?; let post_id = data.post_id; let orig_post = Post::read(&mut context.pool(), post_id).await?; @@ -70,10 +72,14 @@ pub async fn update_post( Err(LemmyErrorType::NoPostEditAllowed)? } - // Fetch post links and Pictrs cached image if url was updated - let (embed_title, embed_description, embed_video_url, thumbnail_url) = match &url { + // Fetch post links and thumbnail if url was updated + let (embed_title, embed_description, embed_video_url, metadata_thumbnail) = match &url { Some(url) => { - let metadata = fetch_link_metadata(url, true, &context).await?; + // Only generate the thumbnail if there's no custom thumbnail provided, + // otherwise it will save it in pictrs + let generate_thumbnail = custom_thumbnail.is_none(); + + let metadata = fetch_link_metadata(url, generate_thumbnail, &context).await?; ( Some(metadata.opengraph_data.title), Some(metadata.opengraph_data.description), @@ -83,11 +89,21 @@ pub async fn update_post( } _ => Default::default(), }; + let url = match url { Some(url) => Some(proxy_image_link_opt_apub(Some(url), &context).await?), _ => Default::default(), }; + let custom_thumbnail = match custom_thumbnail { + Some(custom_thumbnail) => { + Some(proxy_image_link_opt_apub(Some(custom_thumbnail), &context).await?) + } + _ => Default::default(), + }; + + let thumbnail_url = custom_thumbnail.or(metadata_thumbnail); + let language_id = data.language_id; CommunityLanguage::is_allowed_community_language( &mut context.pool(), diff --git a/crates/db_schema/src/impls/local_user.rs b/crates/db_schema/src/impls/local_user.rs index 58c37ff3c..14da24bae 100644 --- a/crates/db_schema/src/impls/local_user.rs +++ b/crates/db_schema/src/impls/local_user.rs @@ -1,12 +1,6 @@ use crate::{ newtypes::{DbUrl, LocalUserId, PersonId}, - schema::local_user::dsl::{ - accepted_application, - email, - email_verified, - local_user, - password_encrypted, - }, + schema::{local_user, person, registration_application}, source::{ actor_language::{LocalUserLanguage, SiteLanguage}, local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, @@ -15,11 +9,18 @@ use crate::{ utils::{ functions::{coalesce, lower}, get_conn, + now, DbPool, }, }; use bcrypt::{hash, DEFAULT_COST}; -use diesel::{dsl::insert_into, result::Error, ExpressionMethods, JoinOnDsl, QueryDsl}; +use diesel::{ + dsl::{insert_into, not, IntervalDsl}, + result::Error, + ExpressionMethods, + JoinOnDsl, + QueryDsl, +}; use diesel_async::RunQueryDsl; impl LocalUser { @@ -31,16 +32,16 @@ impl LocalUser { let conn = &mut get_conn(pool).await?; let password_hash = hash(new_password, DEFAULT_COST).expect("Couldn't hash password"); - diesel::update(local_user.find(local_user_id)) - .set((password_encrypted.eq(password_hash),)) + diesel::update(local_user::table.find(local_user_id)) + .set((local_user::password_encrypted.eq(password_hash),)) .get_result::(conn) .await } pub async fn set_all_users_email_verified(pool: &mut DbPool<'_>) -> Result, Error> { let conn = &mut get_conn(pool).await?; - diesel::update(local_user) - .set(email_verified.eq(true)) + diesel::update(local_user::table) + .set(local_user::email_verified.eq(true)) .get_results::(conn) .await } @@ -49,18 +50,43 @@ impl LocalUser { pool: &mut DbPool<'_>, ) -> Result, Error> { let conn = &mut get_conn(pool).await?; - diesel::update(local_user) - .set(accepted_application.eq(true)) + diesel::update(local_user::table) + .set(local_user::accepted_application.eq(true)) .get_results::(conn) .await } - pub async fn is_email_taken(pool: &mut DbPool<'_>, email_: &str) -> Result { + pub async fn delete_old_denied_local_users(pool: &mut DbPool<'_>) -> Result { + let conn = &mut get_conn(pool).await?; + + // Make sure: + // - The deny reason exists + // - The app is older than a week + // - The accepted_application is false + let old_denied_registrations = registration_application::table + .filter(registration_application::deny_reason.is_not_null()) + .filter(registration_application::published.lt(now() - 1.week())) + .select(registration_application::local_user_id); + + // Delete based on join logic is here: + // https://stackoverflow.com/questions/60836040/how-do-i-perform-a-delete-with-sub-query-in-diesel-against-a-postgres-database + let local_users = local_user::table + .filter(local_user::id.eq_any(old_denied_registrations)) + .filter(not(local_user::accepted_application)) + .select(local_user::person_id); + + // Delete the person rows, which should automatically clear the local_user ones + let persons = person::table.filter(person::id.eq_any(local_users)); + + diesel::delete(persons).execute(conn).await + } + + pub async fn is_email_taken(pool: &mut DbPool<'_>, email: &str) -> Result { use diesel::dsl::{exists, select}; let conn = &mut get_conn(pool).await?; - select(exists( - local_user.filter(lower(coalesce(email, "")).eq(email_.to_lowercase())), - )) + select(exists(local_user::table.filter( + lower(coalesce(local_user::email, "")).eq(email.to_lowercase()), + ))) .get_result(conn) .await } @@ -78,7 +104,6 @@ impl LocalUser { community_follower, instance, instance_block, - person, person_block, post, post_saved, @@ -171,7 +196,7 @@ impl Crud for LocalUser { hash(&form.password_encrypted, DEFAULT_COST).expect("Couldn't hash password"); form_with_encrypted_password.password_encrypted = password_hash; - let local_user_ = insert_into(local_user) + let local_user_ = insert_into(local_user::table) .values(form_with_encrypted_password) .get_result::(conn) .await?; @@ -194,7 +219,7 @@ impl Crud for LocalUser { form: &Self::UpdateForm, ) -> Result { let conn = &mut get_conn(pool).await?; - diesel::update(local_user.find(local_user_id)) + diesel::update(local_user::table.find(local_user_id)) .set(form) .get_result::(conn) .await diff --git a/docker/Dockerfile b/docker/Dockerfile index e33b99cfb..1bbf4ddbd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.6 -ARG RUST_VERSION=1.75 +ARG RUST_VERSION=1.76 ARG CARGO_BUILD_FEATURES=default ARG RUST_RELEASE_MODE=debug diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 7433c9638..38eb4ece2 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -22,7 +22,10 @@ use lemmy_db_schema::{ received_activity, sent_activity, }, - source::instance::{Instance, InstanceForm}, + source::{ + instance::{Instance, InstanceForm}, + local_user::LocalUser, + }, utils::{get_conn, naive_now, now, DbPool, DELETED_REPLACEMENT_TEXT}, }; use lemmy_routes::nodeinfo::NodeInfo; @@ -79,24 +82,19 @@ pub async fn setup(context: LemmyContext) -> Result<(), LemmyError> { }); let context_1 = context.clone(); - // Overwrite deleted & removed posts and comments every day + // Daily tasks: + // - Overwrite deleted & removed posts and comments every day + // - Delete old denied users + // - Update instance software scheduler.every(CTimeUnits::days(1)).run(move || { let context = context_1.clone(); async move { overwrite_deleted_posts_and_comments(&mut context.pool()).await; - } - }); - - let context_1 = context.clone(); - // Update the Instance Software - scheduler.every(CTimeUnits::days(1)).run(move || { - let context = context_1.clone(); - - async move { + delete_old_denied_users(&mut context.pool()).await; update_instance_software(&mut context.pool(), context.client()) .await - .map_err(|e| warn!("Failed to update instance software: {e}")) + .inspect_err(|e| warn!("Failed to update instance software: {e}")) .ok(); } }); @@ -115,6 +113,7 @@ async fn startup_jobs(pool: &mut DbPool<'_>) { update_banned_when_expired(pool).await; clear_old_activities(pool).await; overwrite_deleted_posts_and_comments(pool).await; + delete_old_denied_users(pool).await; } /// Update the hot_rank columns for the aggregates tables @@ -277,10 +276,10 @@ async fn delete_expired_captcha_answers(pool: &mut DbPool<'_>) { ) .execute(&mut conn) .await - .map(|_| { + .inspect(|_| { info!("Done."); }) - .map_err(|e| error!("Failed to clear old captcha answers: {e}")) + .inspect_err(|e| error!("Failed to clear old captcha answers: {e}")) .ok(); } Err(e) => { @@ -301,7 +300,7 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) { ) .execute(&mut conn) .await - .map_err(|e| error!("Failed to clear old sent activities: {e}")) + .inspect_err(|e| error!("Failed to clear old sent activities: {e}")) .ok(); diesel::delete( @@ -310,8 +309,8 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) { ) .execute(&mut conn) .await - .map(|_| info!("Done.")) - .map_err(|e| error!("Failed to clear old received activities: {e}")) + .inspect(|_| info!("Done.")) + .inspect_err(|e| error!("Failed to clear old received activities: {e}")) .ok(); } Err(e) => { @@ -320,6 +319,16 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) { } } +async fn delete_old_denied_users(pool: &mut DbPool<'_>) { + LocalUser::delete_old_denied_local_users(pool) + .await + .inspect(|_| { + info!("Done."); + }) + .inspect(|e| error!("Failed to deleted old denied users: {e}")) + .ok(); +} + /// overwrite posts and comments 30d after deletion async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) { info!("Overwriting deleted posts..."); @@ -339,10 +348,10 @@ async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) { )) .execute(&mut conn) .await - .map(|_| { + .inspect(|_| { info!("Done."); }) - .map_err(|e| error!("Failed to overwrite deleted posts: {e}")) + .inspect_err(|e| error!("Failed to overwrite deleted posts: {e}")) .ok(); info!("Overwriting deleted comments..."); @@ -355,10 +364,10 @@ async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) { .set(comment::content.eq(DELETED_REPLACEMENT_TEXT)) .execute(&mut conn) .await - .map(|_| { + .inspect(|_| { info!("Done."); }) - .map_err(|e| error!("Failed to overwrite deleted comments: {e}")) + .inspect_err(|e| error!("Failed to overwrite deleted comments: {e}")) .ok(); } Err(e) => { @@ -390,14 +399,14 @@ async fn active_counts(pool: &mut DbPool<'_>) { sql_query(update_site_stmt) .execute(&mut conn) .await - .map_err(|e| error!("Failed to update site stats: {e}")) + .inspect_err(|e| error!("Failed to update site stats: {e}")) .ok(); let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", i.1, i.0); sql_query(update_community_stmt) .execute(&mut conn) .await - .map_err(|e| error!("Failed to update community stats: {e}")) + .inspect_err(|e| error!("Failed to update community stats: {e}")) .ok(); } @@ -424,7 +433,7 @@ async fn update_banned_when_expired(pool: &mut DbPool<'_>) { .set(person::banned.eq(false)) .execute(&mut conn) .await - .map_err(|e| error!("Failed to update person.banned when expires: {e}")) + .inspect_err(|e| error!("Failed to update person.banned when expires: {e}")) .ok(); diesel::delete( @@ -432,7 +441,7 @@ async fn update_banned_when_expired(pool: &mut DbPool<'_>) { ) .execute(&mut conn) .await - .map_err(|e| error!("Failed to remove community_ban expired rows: {e}")) + .inspect_err(|e| error!("Failed to remove community_ban expired rows: {e}")) .ok(); } Err(e) => {