Merge branch 'main' into auto_resolve_reports_1

auto_resolve_reports_1
Dessalines 2024-02-15 08:30:18 -05:00 committed by GitHub
commit f2e2bfcc42
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 121 additions and 55 deletions

View File

@ -2,7 +2,7 @@
# See https://github.com/woodpecker-ci/woodpecker/issues/1677 # See https://github.com/woodpecker-ci/woodpecker/issues/1677
variables: variables:
- &rust_image "rust:1.75" - &rust_image "rust:1.76"
- &install_pnpm "corepack enable pnpm" - &install_pnpm "corepack enable pnpm"
- &slow_check_paths - &slow_check_paths
- path: - path:

View File

@ -28,6 +28,9 @@ pub struct CreatePost {
pub honeypot: Option<String>, pub honeypot: Option<String>,
pub nsfw: Option<bool>, pub nsfw: Option<bool>,
pub language_id: Option<LanguageId>, pub language_id: Option<LanguageId>,
#[cfg_attr(feature = "full", ts(type = "string"))]
/// Instead of fetching a thumbnail, use a custom one.
pub custom_thumbnail: Option<Url>,
} }
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
@ -114,6 +117,9 @@ pub struct EditPost {
pub body: Option<String>, pub body: Option<String>,
pub nsfw: Option<bool>, pub nsfw: Option<bool>,
pub language_id: Option<LanguageId>, pub language_id: Option<LanguageId>,
#[cfg_attr(feature = "full", ts(type = "string"))]
/// Instead of fetching a thumbnail, use a custom one.
pub custom_thumbnail: Option<Url>,
} }
#[derive(Debug, Serialize, Deserialize, Clone, Copy, Default, PartialEq, Eq, Hash)] #[derive(Debug, Serialize, Deserialize, Clone, Copy, Default, PartialEq, Eq, Hash)]

View File

@ -57,10 +57,12 @@ pub async fn create_post(
let data_url = data.url.as_ref(); let data_url = data.url.as_ref();
let url = data_url.map(clean_url_params); // TODO no good way to handle a "clear" let url = data_url.map(clean_url_params); // TODO no good way to handle a "clear"
let custom_thumbnail = data.custom_thumbnail.as_ref().map(clean_url_params);
is_valid_post_title(&data.name)?; is_valid_post_title(&data.name)?;
is_valid_body_field(&body, true)?; is_valid_body_field(&body, true)?;
check_url_scheme(&data.url)?; check_url_scheme(&url)?;
check_url_scheme(&custom_thumbnail)?;
check_community_user_action( check_community_user_action(
&local_user_view.person, &local_user_view.person,
@ -84,9 +86,17 @@ pub async fn create_post(
} }
} }
// Only generate the thumbnail if there's no custom thumbnail provided,
// otherwise it will save it in pictrs
let generate_thumbnail = custom_thumbnail.is_none();
// Fetch post links and pictrs cached image // Fetch post links and pictrs cached image
let metadata = fetch_link_metadata_opt(url.as_ref(), true, &context).await; let metadata = fetch_link_metadata_opt(url.as_ref(), generate_thumbnail, &context).await;
let url = proxy_image_link_opt_apub(url, &context).await?; let url = proxy_image_link_opt_apub(url, &context).await?;
let thumbnail_url = proxy_image_link_opt_apub(custom_thumbnail, &context)
.await?
.map(Into::into)
.or(metadata.thumbnail);
// Only need to check if language is allowed in case user set it explicitly. When using default // Only need to check if language is allowed in case user set it explicitly. When using default
// language, it already only returns allowed languages. // language, it already only returns allowed languages.
@ -121,7 +131,7 @@ pub async fn create_post(
.embed_description(metadata.opengraph_data.description) .embed_description(metadata.opengraph_data.description)
.embed_video_url(metadata.opengraph_data.embed_video_url) .embed_video_url(metadata.opengraph_data.embed_video_url)
.language_id(language_id) .language_id(language_id)
.thumbnail_url(metadata.thumbnail) .thumbnail_url(thumbnail_url)
.build(); .build();
let inserted_post = Post::create(&mut context.pool(), &post_form) let inserted_post = Post::create(&mut context.pool(), &post_form)

View File

@ -43,6 +43,7 @@ pub async fn update_post(
// TODO No good way to handle a clear. // TODO No good way to handle a clear.
// Issue link: https://github.com/LemmyNet/lemmy/issues/2287 // Issue link: https://github.com/LemmyNet/lemmy/issues/2287
let url = data.url.as_ref().map(clean_url_params); let url = data.url.as_ref().map(clean_url_params);
let custom_thumbnail = data.custom_thumbnail.as_ref().map(clean_url_params);
let slur_regex = local_site_to_slur_regex(&local_site); let slur_regex = local_site_to_slur_regex(&local_site);
check_slurs_opt(&data.name, &slur_regex)?; check_slurs_opt(&data.name, &slur_regex)?;
@ -53,7 +54,8 @@ pub async fn update_post(
} }
is_valid_body_field(&body, true)?; is_valid_body_field(&body, true)?;
check_url_scheme(&data.url)?; check_url_scheme(&url)?;
check_url_scheme(&custom_thumbnail)?;
let post_id = data.post_id; let post_id = data.post_id;
let orig_post = Post::read(&mut context.pool(), post_id).await?; let orig_post = Post::read(&mut context.pool(), post_id).await?;
@ -70,10 +72,14 @@ pub async fn update_post(
Err(LemmyErrorType::NoPostEditAllowed)? Err(LemmyErrorType::NoPostEditAllowed)?
} }
// Fetch post links and Pictrs cached image if url was updated // Fetch post links and thumbnail if url was updated
let (embed_title, embed_description, embed_video_url, thumbnail_url) = match &url { let (embed_title, embed_description, embed_video_url, metadata_thumbnail) = match &url {
Some(url) => { Some(url) => {
let metadata = fetch_link_metadata(url, true, &context).await?; // Only generate the thumbnail if there's no custom thumbnail provided,
// otherwise it will save it in pictrs
let generate_thumbnail = custom_thumbnail.is_none();
let metadata = fetch_link_metadata(url, generate_thumbnail, &context).await?;
( (
Some(metadata.opengraph_data.title), Some(metadata.opengraph_data.title),
Some(metadata.opengraph_data.description), Some(metadata.opengraph_data.description),
@ -83,11 +89,21 @@ pub async fn update_post(
} }
_ => Default::default(), _ => Default::default(),
}; };
let url = match url { let url = match url {
Some(url) => Some(proxy_image_link_opt_apub(Some(url), &context).await?), Some(url) => Some(proxy_image_link_opt_apub(Some(url), &context).await?),
_ => Default::default(), _ => Default::default(),
}; };
let custom_thumbnail = match custom_thumbnail {
Some(custom_thumbnail) => {
Some(proxy_image_link_opt_apub(Some(custom_thumbnail), &context).await?)
}
_ => Default::default(),
};
let thumbnail_url = custom_thumbnail.or(metadata_thumbnail);
let language_id = data.language_id; let language_id = data.language_id;
CommunityLanguage::is_allowed_community_language( CommunityLanguage::is_allowed_community_language(
&mut context.pool(), &mut context.pool(),

View File

@ -1,12 +1,6 @@
use crate::{ use crate::{
newtypes::{DbUrl, LocalUserId, PersonId}, newtypes::{DbUrl, LocalUserId, PersonId},
schema::local_user::dsl::{ schema::{local_user, person, registration_application},
accepted_application,
email,
email_verified,
local_user,
password_encrypted,
},
source::{ source::{
actor_language::{LocalUserLanguage, SiteLanguage}, actor_language::{LocalUserLanguage, SiteLanguage},
local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm},
@ -15,11 +9,18 @@ use crate::{
utils::{ utils::{
functions::{coalesce, lower}, functions::{coalesce, lower},
get_conn, get_conn,
now,
DbPool, DbPool,
}, },
}; };
use bcrypt::{hash, DEFAULT_COST}; use bcrypt::{hash, DEFAULT_COST};
use diesel::{dsl::insert_into, result::Error, ExpressionMethods, JoinOnDsl, QueryDsl}; use diesel::{
dsl::{insert_into, not, IntervalDsl},
result::Error,
ExpressionMethods,
JoinOnDsl,
QueryDsl,
};
use diesel_async::RunQueryDsl; use diesel_async::RunQueryDsl;
impl LocalUser { impl LocalUser {
@ -31,16 +32,16 @@ impl LocalUser {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
let password_hash = hash(new_password, DEFAULT_COST).expect("Couldn't hash password"); let password_hash = hash(new_password, DEFAULT_COST).expect("Couldn't hash password");
diesel::update(local_user.find(local_user_id)) diesel::update(local_user::table.find(local_user_id))
.set((password_encrypted.eq(password_hash),)) .set((local_user::password_encrypted.eq(password_hash),))
.get_result::<Self>(conn) .get_result::<Self>(conn)
.await .await
} }
pub async fn set_all_users_email_verified(pool: &mut DbPool<'_>) -> Result<Vec<Self>, Error> { pub async fn set_all_users_email_verified(pool: &mut DbPool<'_>) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
diesel::update(local_user) diesel::update(local_user::table)
.set(email_verified.eq(true)) .set(local_user::email_verified.eq(true))
.get_results::<Self>(conn) .get_results::<Self>(conn)
.await .await
} }
@ -49,18 +50,43 @@ impl LocalUser {
pool: &mut DbPool<'_>, pool: &mut DbPool<'_>,
) -> Result<Vec<Self>, Error> { ) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
diesel::update(local_user) diesel::update(local_user::table)
.set(accepted_application.eq(true)) .set(local_user::accepted_application.eq(true))
.get_results::<Self>(conn) .get_results::<Self>(conn)
.await .await
} }
pub async fn is_email_taken(pool: &mut DbPool<'_>, email_: &str) -> Result<bool, Error> { pub async fn delete_old_denied_local_users(pool: &mut DbPool<'_>) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
// Make sure:
// - The deny reason exists
// - The app is older than a week
// - The accepted_application is false
let old_denied_registrations = registration_application::table
.filter(registration_application::deny_reason.is_not_null())
.filter(registration_application::published.lt(now() - 1.week()))
.select(registration_application::local_user_id);
// Delete based on join logic is here:
// https://stackoverflow.com/questions/60836040/how-do-i-perform-a-delete-with-sub-query-in-diesel-against-a-postgres-database
let local_users = local_user::table
.filter(local_user::id.eq_any(old_denied_registrations))
.filter(not(local_user::accepted_application))
.select(local_user::person_id);
// Delete the person rows, which should automatically clear the local_user ones
let persons = person::table.filter(person::id.eq_any(local_users));
diesel::delete(persons).execute(conn).await
}
pub async fn is_email_taken(pool: &mut DbPool<'_>, email: &str) -> Result<bool, Error> {
use diesel::dsl::{exists, select}; use diesel::dsl::{exists, select};
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
select(exists( select(exists(local_user::table.filter(
local_user.filter(lower(coalesce(email, "")).eq(email_.to_lowercase())), lower(coalesce(local_user::email, "")).eq(email.to_lowercase()),
)) )))
.get_result(conn) .get_result(conn)
.await .await
} }
@ -78,7 +104,6 @@ impl LocalUser {
community_follower, community_follower,
instance, instance,
instance_block, instance_block,
person,
person_block, person_block,
post, post,
post_saved, post_saved,
@ -171,7 +196,7 @@ impl Crud for LocalUser {
hash(&form.password_encrypted, DEFAULT_COST).expect("Couldn't hash password"); hash(&form.password_encrypted, DEFAULT_COST).expect("Couldn't hash password");
form_with_encrypted_password.password_encrypted = password_hash; form_with_encrypted_password.password_encrypted = password_hash;
let local_user_ = insert_into(local_user) let local_user_ = insert_into(local_user::table)
.values(form_with_encrypted_password) .values(form_with_encrypted_password)
.get_result::<Self>(conn) .get_result::<Self>(conn)
.await?; .await?;
@ -194,7 +219,7 @@ impl Crud for LocalUser {
form: &Self::UpdateForm, form: &Self::UpdateForm,
) -> Result<Self, Error> { ) -> Result<Self, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
diesel::update(local_user.find(local_user_id)) diesel::update(local_user::table.find(local_user_id))
.set(form) .set(form)
.get_result::<Self>(conn) .get_result::<Self>(conn)
.await .await

View File

@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1.6 # syntax=docker/dockerfile:1.6
ARG RUST_VERSION=1.75 ARG RUST_VERSION=1.76
ARG CARGO_BUILD_FEATURES=default ARG CARGO_BUILD_FEATURES=default
ARG RUST_RELEASE_MODE=debug ARG RUST_RELEASE_MODE=debug

View File

@ -22,7 +22,10 @@ use lemmy_db_schema::{
received_activity, received_activity,
sent_activity, sent_activity,
}, },
source::instance::{Instance, InstanceForm}, source::{
instance::{Instance, InstanceForm},
local_user::LocalUser,
},
utils::{get_conn, naive_now, now, DbPool, DELETED_REPLACEMENT_TEXT}, utils::{get_conn, naive_now, now, DbPool, DELETED_REPLACEMENT_TEXT},
}; };
use lemmy_routes::nodeinfo::NodeInfo; use lemmy_routes::nodeinfo::NodeInfo;
@ -79,24 +82,19 @@ pub async fn setup(context: LemmyContext) -> Result<(), LemmyError> {
}); });
let context_1 = context.clone(); let context_1 = context.clone();
// Overwrite deleted & removed posts and comments every day // Daily tasks:
// - Overwrite deleted & removed posts and comments every day
// - Delete old denied users
// - Update instance software
scheduler.every(CTimeUnits::days(1)).run(move || { scheduler.every(CTimeUnits::days(1)).run(move || {
let context = context_1.clone(); let context = context_1.clone();
async move { async move {
overwrite_deleted_posts_and_comments(&mut context.pool()).await; overwrite_deleted_posts_and_comments(&mut context.pool()).await;
} delete_old_denied_users(&mut context.pool()).await;
});
let context_1 = context.clone();
// Update the Instance Software
scheduler.every(CTimeUnits::days(1)).run(move || {
let context = context_1.clone();
async move {
update_instance_software(&mut context.pool(), context.client()) update_instance_software(&mut context.pool(), context.client())
.await .await
.map_err(|e| warn!("Failed to update instance software: {e}")) .inspect_err(|e| warn!("Failed to update instance software: {e}"))
.ok(); .ok();
} }
}); });
@ -115,6 +113,7 @@ async fn startup_jobs(pool: &mut DbPool<'_>) {
update_banned_when_expired(pool).await; update_banned_when_expired(pool).await;
clear_old_activities(pool).await; clear_old_activities(pool).await;
overwrite_deleted_posts_and_comments(pool).await; overwrite_deleted_posts_and_comments(pool).await;
delete_old_denied_users(pool).await;
} }
/// Update the hot_rank columns for the aggregates tables /// Update the hot_rank columns for the aggregates tables
@ -277,10 +276,10 @@ async fn delete_expired_captcha_answers(pool: &mut DbPool<'_>) {
) )
.execute(&mut conn) .execute(&mut conn)
.await .await
.map(|_| { .inspect(|_| {
info!("Done."); info!("Done.");
}) })
.map_err(|e| error!("Failed to clear old captcha answers: {e}")) .inspect_err(|e| error!("Failed to clear old captcha answers: {e}"))
.ok(); .ok();
} }
Err(e) => { Err(e) => {
@ -301,7 +300,7 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) {
) )
.execute(&mut conn) .execute(&mut conn)
.await .await
.map_err(|e| error!("Failed to clear old sent activities: {e}")) .inspect_err(|e| error!("Failed to clear old sent activities: {e}"))
.ok(); .ok();
diesel::delete( diesel::delete(
@ -310,8 +309,8 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) {
) )
.execute(&mut conn) .execute(&mut conn)
.await .await
.map(|_| info!("Done.")) .inspect(|_| info!("Done."))
.map_err(|e| error!("Failed to clear old received activities: {e}")) .inspect_err(|e| error!("Failed to clear old received activities: {e}"))
.ok(); .ok();
} }
Err(e) => { Err(e) => {
@ -320,6 +319,16 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) {
} }
} }
async fn delete_old_denied_users(pool: &mut DbPool<'_>) {
LocalUser::delete_old_denied_local_users(pool)
.await
.inspect(|_| {
info!("Done.");
})
.inspect(|e| error!("Failed to deleted old denied users: {e}"))
.ok();
}
/// overwrite posts and comments 30d after deletion /// overwrite posts and comments 30d after deletion
async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) { async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) {
info!("Overwriting deleted posts..."); info!("Overwriting deleted posts...");
@ -339,10 +348,10 @@ async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) {
)) ))
.execute(&mut conn) .execute(&mut conn)
.await .await
.map(|_| { .inspect(|_| {
info!("Done."); info!("Done.");
}) })
.map_err(|e| error!("Failed to overwrite deleted posts: {e}")) .inspect_err(|e| error!("Failed to overwrite deleted posts: {e}"))
.ok(); .ok();
info!("Overwriting deleted comments..."); info!("Overwriting deleted comments...");
@ -355,10 +364,10 @@ async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) {
.set(comment::content.eq(DELETED_REPLACEMENT_TEXT)) .set(comment::content.eq(DELETED_REPLACEMENT_TEXT))
.execute(&mut conn) .execute(&mut conn)
.await .await
.map(|_| { .inspect(|_| {
info!("Done."); info!("Done.");
}) })
.map_err(|e| error!("Failed to overwrite deleted comments: {e}")) .inspect_err(|e| error!("Failed to overwrite deleted comments: {e}"))
.ok(); .ok();
} }
Err(e) => { Err(e) => {
@ -390,14 +399,14 @@ async fn active_counts(pool: &mut DbPool<'_>) {
sql_query(update_site_stmt) sql_query(update_site_stmt)
.execute(&mut conn) .execute(&mut conn)
.await .await
.map_err(|e| error!("Failed to update site stats: {e}")) .inspect_err(|e| error!("Failed to update site stats: {e}"))
.ok(); .ok();
let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", i.1, i.0); let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", i.1, i.0);
sql_query(update_community_stmt) sql_query(update_community_stmt)
.execute(&mut conn) .execute(&mut conn)
.await .await
.map_err(|e| error!("Failed to update community stats: {e}")) .inspect_err(|e| error!("Failed to update community stats: {e}"))
.ok(); .ok();
} }
@ -424,7 +433,7 @@ async fn update_banned_when_expired(pool: &mut DbPool<'_>) {
.set(person::banned.eq(false)) .set(person::banned.eq(false))
.execute(&mut conn) .execute(&mut conn)
.await .await
.map_err(|e| error!("Failed to update person.banned when expires: {e}")) .inspect_err(|e| error!("Failed to update person.banned when expires: {e}"))
.ok(); .ok();
diesel::delete( diesel::delete(
@ -432,7 +441,7 @@ async fn update_banned_when_expired(pool: &mut DbPool<'_>) {
) )
.execute(&mut conn) .execute(&mut conn)
.await .await
.map_err(|e| error!("Failed to remove community_ban expired rows: {e}")) .inspect_err(|e| error!("Failed to remove community_ban expired rows: {e}"))
.ok(); .ok();
} }
Err(e) => { Err(e) => {