From 0ebd83081445821768267c5f794540bce3ca8657 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 25 Sep 2020 11:16:49 -0400 Subject: [PATCH 01/17] More overwriteable fields (#1155) * Adding more overwriteable fields for user. Fixes #1154 * Adding a note for bio. --- lemmy_api/src/user.rs | 35 ++++++++++++++--------------------- lemmy_apub/src/user.rs | 7 +++++-- lemmy_db/src/user.rs | 6 +++--- src/code_migrations.rs | 6 +++--- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/lemmy_api/src/user.rs b/lemmy_api/src/user.rs index 4e6269268..4324c9368 100644 --- a/lemmy_api/src/user.rs +++ b/lemmy_api/src/user.rs @@ -335,31 +335,24 @@ impl Perform for SaveUserSettings { let user_id = user.id; let read_user = blocking(context.pool(), move |conn| User_::read(conn, user_id)).await??; - let bio = match &data.bio { - Some(bio) => { - if bio.chars().count() <= 300 { - Some(bio.to_owned()) - } else { - return Err(APIError::err("bio_length_overflow").into()); - } - } - None => read_user.bio, - }; - let avatar = diesel_option_overwrite(&data.avatar); let banner = diesel_option_overwrite(&data.banner); let email = diesel_option_overwrite(&data.email); + let bio = diesel_option_overwrite(&data.bio); + let preferred_username = diesel_option_overwrite(&data.preferred_username); + let matrix_user_id = diesel_option_overwrite(&data.matrix_user_id); - // The DB constraint should stop too many characters - let preferred_username = match &data.preferred_username { - Some(preferred_username) => { - if !is_valid_preferred_username(preferred_username.trim()) { - return Err(APIError::err("invalid_username").into()); - } - Some(preferred_username.trim().to_string()) + if let Some(Some(bio)) = &bio { + if bio.chars().count() > 300 { + return Err(APIError::err("bio_length_overflow").into()); } - None => read_user.preferred_username, - }; + } + + if let Some(Some(preferred_username)) = &preferred_username { + if !is_valid_preferred_username(preferred_username.trim()) { + return Err(APIError::err("invalid_username").into()); + } + } let password_encrypted = match &data.new_password { Some(new_password) => { @@ -397,7 +390,7 @@ impl Perform for SaveUserSettings { let user_form = UserForm { name: read_user.name, email, - matrix_user_id: data.matrix_user_id.to_owned(), + matrix_user_id, avatar, banner, password_encrypted, diff --git a/lemmy_apub/src/user.rs b/lemmy_apub/src/user.rs index 950f59a18..60af834ca 100644 --- a/lemmy_apub/src/user.rs +++ b/lemmy_apub/src/user.rs @@ -241,6 +241,9 @@ impl FromApub for UserForm { .context(location_info!())? .to_string(); let preferred_username = person.inner.preferred_username().map(|u| u.to_string()); + + // TODO a limit check (like the API does) might need to be done + // here when we federate to other platforms. Same for preferred_username let bio = person .inner .summary() @@ -253,7 +256,7 @@ impl FromApub for UserForm { Ok(UserForm { name, - preferred_username, + preferred_username: Some(preferred_username), password_encrypted: "".to_string(), admin: false, banned: false, @@ -271,7 +274,7 @@ impl FromApub for UserForm { send_notifications_to_email: false, matrix_user_id: None, actor_id: Some(check_actor_domain(person, expected_domain)?), - bio, + bio: Some(bio), local: false, private_key: None, public_key: Some(person.ext_one.public_key.to_owned().public_key_pem), diff --git a/lemmy_db/src/user.rs b/lemmy_db/src/user.rs index 15fb592c7..83f0559ab 100644 --- a/lemmy_db/src/user.rs +++ b/lemmy_db/src/user.rs @@ -43,7 +43,7 @@ pub struct User_ { #[table_name = "user_"] pub struct UserForm { pub name: String, - pub preferred_username: Option, + pub preferred_username: Option>, pub password_encrypted: String, pub admin: bool, pub banned: bool, @@ -58,9 +58,9 @@ pub struct UserForm { pub lang: String, pub show_avatars: bool, pub send_notifications_to_email: bool, - pub matrix_user_id: Option, + pub matrix_user_id: Option>, pub actor_id: Option, - pub bio: Option, + pub bio: Option>, pub local: bool, pub private_key: Option, pub public_key: Option, diff --git a/src/code_migrations.rs b/src/code_migrations.rs index 6743fb298..adc4ae49a 100644 --- a/src/code_migrations.rs +++ b/src/code_migrations.rs @@ -49,11 +49,11 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { let form = UserForm { name: cuser.name.to_owned(), email: Some(cuser.email.to_owned()), - matrix_user_id: cuser.matrix_user_id.to_owned(), + matrix_user_id: Some(cuser.matrix_user_id.to_owned()), avatar: Some(cuser.avatar.to_owned()), banner: Some(cuser.banner.to_owned()), password_encrypted: cuser.password_encrypted.to_owned(), - preferred_username: cuser.preferred_username.to_owned(), + preferred_username: Some(cuser.preferred_username.to_owned()), published: Some(cuser.published), updated: None, admin: cuser.admin, @@ -66,7 +66,7 @@ fn user_updates_2020_04_02(conn: &PgConnection) -> Result<(), LemmyError> { show_avatars: cuser.show_avatars, send_notifications_to_email: cuser.send_notifications_to_email, actor_id: Some(make_apub_endpoint(EndpointType::User, &cuser.name).to_string()), - bio: cuser.bio.to_owned(), + bio: Some(cuser.bio.to_owned()), local: cuser.local, private_key: Some(keypair.private_key), public_key: Some(keypair.public_key), From b074a963fe09c6a621b4670e6eb2587645e25e4e Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 25 Sep 2020 16:26:19 -0500 Subject: [PATCH 02/17] Version v0.7.61 --- ansible/VERSION | 2 +- docker/prod/docker-compose.yml | 2 +- docker/travis/docker_push.sh | 4 ++-- lemmy_api/src/version.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/VERSION b/ansible/VERSION index 90ff00a0d..ddf4dd99c 100644 --- a/ansible/VERSION +++ b/ansible/VERSION @@ -1 +1 @@ -v0.7.59 +v0.7.61 diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 385fdc7c6..0b64dcd05 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -12,7 +12,7 @@ services: restart: always lemmy: - image: dessalines/lemmy:v0.7.59 + image: dessalines/lemmy:v0.7.61 ports: - "127.0.0.1:8536:8536" restart: always diff --git a/docker/travis/docker_push.sh b/docker/travis/docker_push.sh index 364ff90aa..0d53e6896 100644 --- a/docker/travis/docker_push.sh +++ b/docker/travis/docker_push.sh @@ -1,5 +1,5 @@ #!/bin/sh echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin docker tag dessalines/lemmy:travis \ - dessalines/lemmy:v0.7.59 -docker push dessalines/lemmy:v0.7.59 + dessalines/lemmy:v0.7.61 +docker push dessalines/lemmy:v0.7.61 diff --git a/lemmy_api/src/version.rs b/lemmy_api/src/version.rs index e1a081ad8..63213d54a 100644 --- a/lemmy_api/src/version.rs +++ b/lemmy_api/src/version.rs @@ -1 +1 @@ -pub const VERSION: &str = "v0.7.59"; +pub const VERSION: &str = "v0.7.61"; From a4cb067130f34f4be82698ab648ea5e418a351e6 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 30 Sep 2020 18:19:14 +0200 Subject: [PATCH 03/17] Dont send to blocked instances, rewrite activity_sender --- lemmy_apub/src/activities.rs | 49 ---------- lemmy_apub/src/activity_queue.rs | 149 ++++++++++++++++++++++++++++-- lemmy_apub/src/comment.rs | 72 +++------------ lemmy_apub/src/community.rs | 68 ++++---------- lemmy_apub/src/lib.rs | 15 ++- lemmy_apub/src/post.rs | 84 +++-------------- lemmy_apub/src/private_message.rs | 21 ++--- lemmy_apub/src/user.rs | 13 +-- 8 files changed, 206 insertions(+), 265 deletions(-) delete mode 100644 lemmy_apub/src/activities.rs diff --git a/lemmy_apub/src/activities.rs b/lemmy_apub/src/activities.rs deleted file mode 100644 index 18781ef42..000000000 --- a/lemmy_apub/src/activities.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::{activity_queue::send_activity, community::do_announce, insert_activity}; -use activitystreams::{ - base::{Extends, ExtendsExt}, - object::AsObject, -}; -use lemmy_db::{community::Community, user::User_}; -use lemmy_utils::{settings::Settings, LemmyError}; -use lemmy_websocket::LemmyContext; -use serde::{export::fmt::Debug, Serialize}; -use url::{ParseError, Url}; -use uuid::Uuid; - -pub async fn send_activity_to_community( - creator: &User_, - community: &Community, - to: Vec, - activity: T, - context: &LemmyContext, -) -> Result<(), LemmyError> -where - T: AsObject + Extends + Serialize + Debug + Send + Clone + 'static, - Kind: Serialize, - >::Error: From + Send + Sync + 'static, -{ - // TODO: looks like call this sometimes with activity, and sometimes with any_base - insert_activity(creator.id, activity.clone(), true, context.pool()).await?; - - // if this is a local community, we need to do an announce from the community instead - if community.local { - do_announce(activity.into_any_base()?, &community, creator, context).await?; - } else { - send_activity(context.activity_queue(), activity, creator, to)?; - } - - Ok(()) -} - -pub(in crate) fn generate_activity_id(kind: T) -> Result -where - T: ToString, -{ - let id = format!( - "{}/activities/{}/{}", - Settings::get().get_protocol_and_hostname(), - kind.to_string().to_lowercase(), - Uuid::new_v4() - ); - Url::parse(&id) -} diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index ece782c5d..960e126b8 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -1,4 +1,10 @@ -use crate::{check_is_apub_id_valid, extensions::signatures::sign, ActorType}; +use crate::{ + check_is_apub_id_valid, + community::do_announce, + extensions::signatures::sign, + insert_activity, + ActorType, +}; use activitystreams::{ base::{Extends, ExtendsExt}, object::AsObject, @@ -13,22 +19,148 @@ use background_jobs::{ QueueHandle, WorkerConfig, }; +use itertools::Itertools; +use lemmy_db::{community::Community, user::User_, DbPool}; use lemmy_utils::{location_info, settings::Settings, LemmyError}; +use lemmy_websocket::LemmyContext; use log::warn; use reqwest::Client; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, future::Future, pin::Pin}; use url::Url; -pub fn send_activity( +pub async fn send_activity_single_dest( + activity: T, + creator: &dyn ActorType, + to: Url, + context: &LemmyContext, +) -> Result<(), LemmyError> +where + T: AsObject + Extends, + Kind: Serialize, + >::Error: From + Send + Sync + 'static, +{ + if check_is_apub_id_valid(&to).is_ok() { + send_activity_internal( + context.activity_queue(), + activity, + creator, + vec![to], + context.pool(), + ) + .await?; + } + + Ok(()) +} + +pub async fn send_to_community_followers( + activity: T, + community: &Community, + context: &LemmyContext, + sender_shared_inbox: Option, +) -> Result<(), LemmyError> +where + T: AsObject + Extends, + Kind: Serialize, + >::Error: From + Send + Sync + 'static, +{ + // dont send to the local instance, nor to the instance where the activity originally came from, + // because that would result in a database error (same data inserted twice) + let community_shared_inbox = community.get_shared_inbox_url()?; + let to: Vec = community + .get_follower_inboxes(context.pool()) + .await? + .iter() + .filter(|inbox| Some(inbox) != sender_shared_inbox.as_ref().as_ref()) + .filter(|inbox| inbox != &&community_shared_inbox) + .filter(|inbox| check_is_apub_id_valid(inbox).is_ok()) + .unique() + .map(|inbox| inbox.to_owned()) + .collect(); + + send_activity_internal( + context.activity_queue(), + activity, + community, + to, + context.pool(), + ) + .await?; + + Ok(()) +} + +pub async fn send_to_community( + creator: &User_, + community: &Community, + activity: T, + context: &LemmyContext, +) -> Result<(), LemmyError> +where + T: AsObject + Extends, + Kind: Serialize, + >::Error: From + Send + Sync + 'static, +{ + // if this is a local community, we need to do an announce from the community instead + if community.local { + do_announce(activity.into_any_base()?, &community, creator, context).await?; + } else { + let inbox = community.get_shared_inbox_url()?; + check_is_apub_id_valid(&inbox)?; + send_activity_internal( + context.activity_queue(), + activity, + creator, + vec![inbox], + context.pool(), + ) + .await?; + } + + Ok(()) +} + +pub async fn send_comment_mentions( + creator: &User_, + mentions: Vec, + activity: T, + context: &LemmyContext, +) -> Result<(), LemmyError> +where + T: AsObject + Extends, + Kind: Serialize, + >::Error: From + Send + Sync + 'static, +{ + let mentions = mentions + .iter() + .filter(|inbox| check_is_apub_id_valid(inbox).is_ok()) + .map(|i| i.to_owned()) + .collect(); + send_activity_internal( + context.activity_queue(), + activity, + creator, + mentions, + context.pool(), + ) + .await?; + Ok(()) +} + +/// Asynchronously sends the given `activity` from `actor` to every inbox URL in `to`. +/// +/// The caller of this function needs to remove any blocked domains from `to`, +/// using `check_is_apub_id_valid()`. +async fn send_activity_internal( activity_sender: &QueueHandle, activity: T, actor: &dyn ActorType, to: Vec, + pool: &DbPool, ) -> Result<(), LemmyError> where - T: AsObject, - T: Extends, + T: AsObject + Extends, Kind: Serialize, >::Error: From + Send + Sync + 'static, { @@ -36,12 +168,13 @@ where return Ok(()); } + for to_url in &to { + assert!(check_is_apub_id_valid(&to_url).is_ok()); + } + let activity = activity.into_any_base()?; let serialised_activity = serde_json::to_string(&activity)?; - - for to_url in &to { - check_is_apub_id_valid(&to_url)?; - } + insert_activity(actor.user_id(), serialised_activity.clone(), true, pool).await?; // TODO: it would make sense to create a separate task for each destination server let message = SendActivityTask { diff --git a/lemmy_apub/src/comment.rs b/lemmy_apub/src/comment.rs index 4e5c173f8..7f6885a19 100644 --- a/lemmy_apub/src/comment.rs +++ b/lemmy_apub/src/comment.rs @@ -1,5 +1,5 @@ use crate::{ - activities::{generate_activity_id, send_activity_to_community}, + activity_queue::{send_comment_mentions, send_to_community}, check_actor_domain, create_apub_response, create_apub_tombstone_response, @@ -10,6 +10,7 @@ use crate::{ get_or_fetch_and_insert_post, get_or_fetch_and_upsert_user, }, + generate_activity_id, ActorType, ApubLikeableType, ApubObjectType, @@ -219,7 +220,8 @@ impl ApubObjectType for Comment { // Set the mention tags .set_many_tags(maa.get_tags()?); - send_activity_to_community(&creator, &community, maa.inboxes, create, context).await?; + send_to_community(&creator, &community, create.clone(), context).await?; + send_comment_mentions(&creator, maa.inboxes, create, context).await?; Ok(()) } @@ -247,7 +249,8 @@ impl ApubObjectType for Comment { // Set the mention tags .set_many_tags(maa.get_tags()?); - send_activity_to_community(&creator, &community, maa.inboxes, update, context).await?; + send_to_community(&creator, &community, update.clone(), context).await?; + send_comment_mentions(&creator, maa.inboxes, update, context).await?; Ok(()) } @@ -270,14 +273,7 @@ impl ApubObjectType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - delete, - context, - ) - .await?; + send_to_community(&creator, &community, delete, context).await?; Ok(()) } @@ -313,14 +309,7 @@ impl ApubObjectType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - undo, - context, - ) - .await?; + send_to_community(&creator, &community, undo, context).await?; Ok(()) } @@ -343,14 +332,7 @@ impl ApubObjectType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &mod_, - &community, - vec![community.get_shared_inbox_url()?], - remove, - context, - ) - .await?; + send_to_community(&mod_, &community, remove, context).await?; Ok(()) } @@ -382,14 +364,7 @@ impl ApubObjectType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &mod_, - &community, - vec![community.get_shared_inbox_url()?], - undo, - context, - ) - .await?; + send_to_community(&mod_, &community, undo, context).await?; Ok(()) } } @@ -415,14 +390,7 @@ impl ApubLikeableType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - like, - context, - ) - .await?; + send_to_community(&creator, &community, like, context).await?; Ok(()) } @@ -445,14 +413,7 @@ impl ApubLikeableType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - dislike, - context, - ) - .await?; + send_to_community(&creator, &community, dislike, context).await?; Ok(()) } @@ -487,14 +448,7 @@ impl ApubLikeableType for Comment { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - undo, - context, - ) - .await?; + send_to_community(&creator, &community, undo, context).await?; Ok(()) } } diff --git a/lemmy_apub/src/community.rs b/lemmy_apub/src/community.rs index 715b765b3..44f5e6e10 100644 --- a/lemmy_apub/src/community.rs +++ b/lemmy_apub/src/community.rs @@ -1,13 +1,13 @@ use crate::{ - activities::generate_activity_id, - activity_queue::send_activity, + activity_queue::{send_activity_single_dest, send_to_community_followers}, check_actor_domain, + check_is_apub_id_valid, create_apub_response, create_apub_tombstone_response, create_tombstone, extensions::group_extensions::GroupExtension, fetcher::{get_or_fetch_and_upsert_actor, get_or_fetch_and_upsert_user}, - insert_activity, + generate_activity_id, ActorType, FromApub, GroupExt, @@ -167,9 +167,7 @@ impl ActorType for Community { .set_id(generate_activity_id(AcceptType::Accept)?) .set_to(to.clone()); - insert_activity(self.creator_id, accept.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), accept, self, vec![to])?; + send_activity_single_dest(accept, self, to, context).await?; Ok(()) } @@ -183,14 +181,7 @@ impl ActorType for Community { .set_to(public()) .set_many_ccs(vec![self.get_followers_url()?]); - insert_activity(self.creator_id, delete.clone(), true, context.pool()).await?; - - let inboxes = self.get_follower_inboxes(context.pool()).await?; - - // Note: For an accept, since it was automatic, no one pushed a button, - // the community was the actor. - // But for delete, the creator is the actor, and does the signing - send_activity(context.activity_queue(), delete, creator, inboxes)?; + send_to_community_followers(delete, self, context, None).await?; Ok(()) } @@ -215,14 +206,7 @@ impl ActorType for Community { .set_to(public()) .set_many_ccs(vec![self.get_followers_url()?]); - insert_activity(self.creator_id, undo.clone(), true, context.pool()).await?; - - let inboxes = self.get_follower_inboxes(context.pool()).await?; - - // Note: For an accept, since it was automatic, no one pushed a button, - // the community was the actor. - // But for delete, the creator is the actor, and does the signing - send_activity(context.activity_queue(), undo, creator, inboxes)?; + send_to_community_followers(undo, self, context, None).await?; Ok(()) } @@ -236,14 +220,7 @@ impl ActorType for Community { .set_to(public()) .set_many_ccs(vec![self.get_followers_url()?]); - insert_activity(mod_.id, remove.clone(), true, context.pool()).await?; - - let inboxes = self.get_follower_inboxes(context.pool()).await?; - - // Note: For an accept, since it was automatic, no one pushed a button, - // the community was the actor. - // But for delete, the creator is the actor, and does the signing - send_activity(context.activity_queue(), remove, mod_, inboxes)?; + send_to_community_followers(remove, self, context, None).await?; Ok(()) } @@ -265,14 +242,7 @@ impl ActorType for Community { .set_to(public()) .set_many_ccs(vec![self.get_followers_url()?]); - insert_activity(mod_.id, undo.clone(), true, context.pool()).await?; - - let inboxes = self.get_follower_inboxes(context.pool()).await?; - - // Note: For an accept, since it was automatic, no one pushed a button, - // the community was the actor. - // But for remove , the creator is the actor, and does the signing - send_activity(context.activity_queue(), undo, mod_, inboxes)?; + send_to_community_followers(undo, self, context, None).await?; Ok(()) } @@ -305,6 +275,8 @@ impl ActorType for Community { ))?) }) .filter_map(Result::ok) + // Don't send to blocked instances + .filter(|inbox| check_is_apub_id_valid(inbox).is_ok()) .unique() .collect(); @@ -513,19 +485,13 @@ pub async fn do_announce( .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - insert_activity(community.creator_id, announce.clone(), true, context.pool()).await?; - - let mut to: Vec = community.get_follower_inboxes(context.pool()).await?; - - // dont send to the local instance, nor to the instance where the activity originally came from, - // because that would result in a database error (same data inserted twice) - // this seems to be the "easiest" stable alternative for remove_item() - let sender_shared_inbox = sender.get_shared_inbox_url()?; - to.retain(|x| x != &sender_shared_inbox); - let community_shared_inbox = community.get_shared_inbox_url()?; - to.retain(|x| x != &community_shared_inbox); - - send_activity(context.activity_queue(), announce, community, to)?; + send_to_community_followers( + announce, + community, + context, + Some(sender.get_shared_inbox_url()?), + ) + .await?; Ok(()) } diff --git a/lemmy_apub/src/lib.rs b/lemmy_apub/src/lib.rs index 3f37c5d3c..1f6e75e4c 100644 --- a/lemmy_apub/src/lib.rs +++ b/lemmy_apub/src/lib.rs @@ -1,7 +1,6 @@ #[macro_use] extern crate lazy_static; -pub mod activities; pub mod activity_queue; pub mod comment; pub mod community; @@ -43,6 +42,7 @@ use log::debug; use reqwest::Client; use serde::Serialize; use url::{ParseError, Url}; +use uuid::Uuid; type GroupExt = Ext2, GroupExtension, PublicKeyExtension>; type PersonExt = Ext1, PublicKeyExtension>; @@ -360,3 +360,16 @@ where .await??; Ok(()) } + +pub(in crate) fn generate_activity_id(kind: T) -> Result +where + T: ToString, +{ + let id = format!( + "{}/activities/{}/{}", + Settings::get().get_protocol_and_hostname(), + kind.to_string().to_lowercase(), + Uuid::new_v4() + ); + Url::parse(&id) +} diff --git a/lemmy_apub/src/post.rs b/lemmy_apub/src/post.rs index 8f5ffbcb8..2d615ea1b 100644 --- a/lemmy_apub/src/post.rs +++ b/lemmy_apub/src/post.rs @@ -1,11 +1,12 @@ use crate::{ - activities::{generate_activity_id, send_activity_to_community}, + activity_queue::send_to_community, check_actor_domain, create_apub_response, create_apub_tombstone_response, create_tombstone, extensions::page_extension::PageExtension, fetcher::{get_or_fetch_and_upsert_community, get_or_fetch_and_upsert_user}, + generate_activity_id, ActorType, ApubLikeableType, ApubObjectType, @@ -257,14 +258,7 @@ impl ApubObjectType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - creator, - &community, - vec![community.get_shared_inbox_url()?], - create, - context, - ) - .await?; + send_to_community(creator, &community, create, context).await?; Ok(()) } @@ -285,14 +279,7 @@ impl ApubObjectType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - creator, - &community, - vec![community.get_shared_inbox_url()?], - update, - context, - ) - .await?; + send_to_community(creator, &community, update, context).await?; Ok(()) } @@ -312,14 +299,7 @@ impl ApubObjectType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - creator, - &community, - vec![community.get_shared_inbox_url()?], - delete, - context, - ) - .await?; + send_to_community(creator, &community, delete, context).await?; Ok(()) } @@ -351,14 +331,7 @@ impl ApubObjectType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - creator, - &community, - vec![community.get_shared_inbox_url()?], - undo, - context, - ) - .await?; + send_to_community(creator, &community, undo, context).await?; Ok(()) } @@ -378,14 +351,7 @@ impl ApubObjectType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - mod_, - &community, - vec![community.get_shared_inbox_url()?], - remove, - context, - ) - .await?; + send_to_community(mod_, &community, remove, context).await?; Ok(()) } @@ -413,14 +379,7 @@ impl ApubObjectType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - mod_, - &community, - vec![community.get_shared_inbox_url()?], - undo, - context, - ) - .await?; + send_to_community(mod_, &community, undo, context).await?; Ok(()) } } @@ -443,14 +402,7 @@ impl ApubLikeableType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - like, - context, - ) - .await?; + send_to_community(&creator, &community, like, context).await?; Ok(()) } @@ -470,14 +422,7 @@ impl ApubLikeableType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - dislike, - context, - ) - .await?; + send_to_community(&creator, &community, dislike, context).await?; Ok(()) } @@ -509,14 +454,7 @@ impl ApubLikeableType for Post { .set_to(public()) .set_many_ccs(vec![community.get_followers_url()?]); - send_activity_to_community( - &creator, - &community, - vec![community.get_shared_inbox_url()?], - undo, - context, - ) - .await?; + send_to_community(&creator, &community, undo, context).await?; Ok(()) } } diff --git a/lemmy_apub/src/private_message.rs b/lemmy_apub/src/private_message.rs index d61a7771e..fd8e6c6b0 100644 --- a/lemmy_apub/src/private_message.rs +++ b/lemmy_apub/src/private_message.rs @@ -1,11 +1,10 @@ use crate::{ - activities::generate_activity_id, - activity_queue::send_activity, + activity_queue::send_activity_single_dest, check_actor_domain, check_is_apub_id_valid, create_tombstone, fetcher::get_or_fetch_and_upsert_user, - insert_activity, + generate_activity_id, ActorType, ApubObjectType, FromApub, @@ -130,9 +129,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(CreateType::Create)?) .set_to(to.clone()); - insert_activity(creator.id, create.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), create, creator, vec![to])?; + send_activity_single_dest(create, creator, to, context).await?; Ok(()) } @@ -150,9 +147,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(UpdateType::Update)?) .set_to(to.clone()); - insert_activity(creator.id, update.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), update, creator, vec![to])?; + send_activity_single_dest(update, creator, to, context).await?; Ok(()) } @@ -169,9 +164,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(DeleteType::Delete)?) .set_to(to.clone()); - insert_activity(creator.id, delete.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), delete, creator, vec![to])?; + send_activity_single_dest(delete, creator, to, context).await?; Ok(()) } @@ -199,9 +192,7 @@ impl ApubObjectType for PrivateMessage { .set_id(generate_activity_id(UndoType::Undo)?) .set_to(to.clone()); - insert_activity(creator.id, undo.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), undo, creator, vec![to])?; + send_activity_single_dest(undo, creator, to, context).await?; Ok(()) } diff --git a/lemmy_apub/src/user.rs b/lemmy_apub/src/user.rs index 60af834ca..3f6e6971c 100644 --- a/lemmy_apub/src/user.rs +++ b/lemmy_apub/src/user.rs @@ -1,10 +1,9 @@ use crate::{ - activities::generate_activity_id, - activity_queue::send_activity, + activity_queue::send_activity_single_dest, check_actor_domain, create_apub_response, fetcher::get_or_fetch_and_upsert_actor, - insert_activity, + generate_activity_id, ActorType, FromApub, PersonExt, @@ -126,9 +125,7 @@ impl ActorType for User_ { let follow_actor = get_or_fetch_and_upsert_actor(follow_actor_id, context).await?; let to = follow_actor.get_inbox_url()?; - insert_activity(self.id, follow.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), follow, self, vec![to])?; + send_activity_single_dest(follow, self, to, context).await?; Ok(()) } @@ -151,9 +148,7 @@ impl ActorType for User_ { .set_context(activitystreams::context()) .set_id(generate_activity_id(UndoType::Undo)?); - insert_activity(self.id, undo.clone(), true, context.pool()).await?; - - send_activity(context.activity_queue(), undo, self, vec![to])?; + send_activity_single_dest(undo, self, to, context).await?; Ok(()) } From eef0a5c7e83636d5d0579d3f0a41b5a22fc18b06 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 2 Oct 2020 09:21:14 -0500 Subject: [PATCH 04/17] Adding pretty print for activities. --- lemmy_apub/src/lib.rs | 5 +++-- lemmy_db/src/activity.rs | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lemmy_apub/src/lib.rs b/lemmy_apub/src/lib.rs index 1f6e75e4c..555bdccc7 100644 --- a/lemmy_apub/src/lib.rs +++ b/lemmy_apub/src/lib.rs @@ -102,8 +102,9 @@ fn check_is_apub_id_valid(apub_id: &Url) -> Result<(), LemmyError> { let mut allowed_instances = Settings::get().get_allowed_instances(); let blocked_instances = Settings::get().get_blocked_instances(); - - if !allowed_instances.is_empty() { + if allowed_instances.is_empty() && blocked_instances.is_empty() { + Ok(()) + } else if !allowed_instances.is_empty() { // need to allow this explicitly because apub activities might contain objects from our local // instance. split is needed to remove the port in our federation test setup. allowed_instances.push(local_instance); diff --git a/lemmy_db/src/activity.rs b/lemmy_db/src/activity.rs index c3f0ab679..af2a1ec8d 100644 --- a/lemmy_db/src/activity.rs +++ b/lemmy_db/src/activity.rs @@ -62,7 +62,11 @@ pub fn do_insert_activity( where T: Serialize + Debug, { - debug!("inserting activity for user {}, data {:?}", user_id, &data); + debug!( + "inserting activity for user {}, data {:?}", + user_id, + serde_json::to_string_pretty(&data).unwrap() + ); let activity_form = ActivityForm { user_id, data: serde_json::to_value(&data)?, From 2e2b6eacd7db07f8809d7717003cfa8160218a1b Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 2 Oct 2020 10:15:26 -0500 Subject: [PATCH 05/17] Fixing pretty print again. --- lemmy_apub/src/community.rs | 1 + lemmy_apub/src/inbox/shared_inbox.rs | 2 +- lemmy_db/src/activity.rs | 7 ++----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lemmy_apub/src/community.rs b/lemmy_apub/src/community.rs index 44f5e6e10..c2b4571fa 100644 --- a/lemmy_apub/src/community.rs +++ b/lemmy_apub/src/community.rs @@ -259,6 +259,7 @@ impl ActorType for Community { .await??; let inboxes = inboxes .into_iter() + .filter(|i| !i.user_local) .map(|u| -> Result { let url = Url::parse(&u.user_actor_id)?; let domain = url.domain().context(location_info!())?; diff --git a/lemmy_apub/src/inbox/shared_inbox.rs b/lemmy_apub/src/inbox/shared_inbox.rs index b9077ebe9..677ceadf9 100644 --- a/lemmy_apub/src/inbox/shared_inbox.rs +++ b/lemmy_apub/src/inbox/shared_inbox.rs @@ -60,7 +60,7 @@ pub async fn shared_inbox( ) -> Result { let activity = input.into_inner(); - let json = serde_json::to_string(&activity)?; + let json = serde_json::to_string_pretty(&activity)?; debug!("Shared inbox received activity: {}", json); // TODO: if we already received an activity with identical ID, then ignore this (same in other inboxes) diff --git a/lemmy_db/src/activity.rs b/lemmy_db/src/activity.rs index af2a1ec8d..18b6ef005 100644 --- a/lemmy_db/src/activity.rs +++ b/lemmy_db/src/activity.rs @@ -62,11 +62,8 @@ pub fn do_insert_activity( where T: Serialize + Debug, { - debug!( - "inserting activity for user {}, data {:?}", - user_id, - serde_json::to_string_pretty(&data).unwrap() - ); + debug!("inserting activity for user {}: ", user_id); + debug!("{}", serde_json::to_string_pretty(&serde_json::to_value(&data)?)?); let activity_form = ActivityForm { user_id, data: serde_json::to_value(&data)?, From 9fa2092a21af18f38fb01206fd81fd958b717446 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Sat, 3 Oct 2020 09:47:06 -0500 Subject: [PATCH 06/17] Adding some logging. --- lemmy_apub/src/activity_queue.rs | 2 +- lemmy_apub/src/community.rs | 5 +++++ lemmy_db/src/activity.rs | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index 7602fe2f7..68b5c3ab0 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -174,7 +174,7 @@ where let activity = activity.into_any_base()?; let serialised_activity = serde_json::to_string(&activity)?; - insert_activity(actor.user_id(), serialised_activity.clone(), true, pool).await?; + insert_activity(actor.user_id(), activity.clone(), true, pool).await?; // TODO: it would make sense to create a separate task for each destination server let message = SendActivityTask { diff --git a/lemmy_apub/src/community.rs b/lemmy_apub/src/community.rs index c2b4571fa..ad9abaaa2 100644 --- a/lemmy_apub/src/community.rs +++ b/lemmy_apub/src/community.rs @@ -1,3 +1,4 @@ +use log::debug; use crate::{ activity_queue::{send_activity_single_dest, send_to_community_followers}, check_actor_domain, @@ -281,6 +282,10 @@ impl ActorType for Community { .unique() .collect(); + // TODO remove this logging line + let inboxes_json = serde_json::to_string_pretty(&inboxes)?; + debug!("Community follower inboxes: {}", inboxes_json); + Ok(inboxes) } diff --git a/lemmy_db/src/activity.rs b/lemmy_db/src/activity.rs index 18b6ef005..6830d566d 100644 --- a/lemmy_db/src/activity.rs +++ b/lemmy_db/src/activity.rs @@ -63,7 +63,7 @@ where T: Serialize + Debug, { debug!("inserting activity for user {}: ", user_id); - debug!("{}", serde_json::to_string_pretty(&serde_json::to_value(&data)?)?); + debug!("{}", serde_json::to_string_pretty(&data)?); let activity_form = ActivityForm { user_id, data: serde_json::to_value(&data)?, From 048fe287c23d4714caeeb8205c1f4c78709cdca8 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Sat, 3 Oct 2020 09:50:06 -0500 Subject: [PATCH 07/17] Running cargo fmt. --- lemmy_apub/src/community.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemmy_apub/src/community.rs b/lemmy_apub/src/community.rs index ad9abaaa2..b1c28d75b 100644 --- a/lemmy_apub/src/community.rs +++ b/lemmy_apub/src/community.rs @@ -1,4 +1,3 @@ -use log::debug; use crate::{ activity_queue::{send_activity_single_dest, send_to_community_followers}, check_actor_domain, @@ -51,6 +50,7 @@ use lemmy_utils::{ LemmyError, }; use lemmy_websocket::LemmyContext; +use log::debug; use serde::Deserialize; use url::Url; From 9e84fe20e64b4cf33d181cef6b37c81ee4a65cca Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 6 Oct 2020 14:58:37 +0200 Subject: [PATCH 08/17] Dont send mentions to inbox of local community also, dont start SendActivityTask for empty `to`, and remove useless comment --- lemmy_apub/src/activity_queue.rs | 2 +- lemmy_apub/src/comment.rs | 5 ++++- lemmy_apub/src/fetcher.rs | 3 --- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index 68b5c3ab0..9f7f38cf1 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -164,7 +164,7 @@ where Kind: Serialize, >::Error: From + Send + Sync + 'static, { - if !Settings::get().federation.enabled { + if !Settings::get().federation.enabled || to.is_empty() { return Ok(()); } diff --git a/lemmy_apub/src/comment.rs b/lemmy_apub/src/comment.rs index 7f6885a19..35be5522e 100644 --- a/lemmy_apub/src/comment.rs +++ b/lemmy_apub/src/comment.rs @@ -506,7 +506,10 @@ async fn collect_non_local_mentions_and_addresses( } } - let mut inboxes = vec![community.get_shared_inbox_url()?]; + let mut inboxes: Vec = vec![]; + if !community.local { + inboxes.push(community.get_shared_inbox_url()?); + } inboxes.extend(mention_inboxes); inboxes = inboxes.into_iter().unique().collect(); diff --git a/lemmy_apub/src/fetcher.rs b/lemmy_apub/src/fetcher.rs index 3f2109ecb..226569f67 100644 --- a/lemmy_apub/src/fetcher.rs +++ b/lemmy_apub/src/fetcher.rs @@ -55,9 +55,6 @@ where let timeout = Duration::from_secs(60); - // speed up tests - // before: 305s - // after: 240s let json = retry(|| { client .get(url.as_str()) From ca4868cefdbe4035c4c9e4e7770df8d7ad308fdb Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 6 Oct 2020 10:19:01 -0500 Subject: [PATCH 09/17] Adding a boolean check to send_activity_internal --- lemmy_apub/src/activity_queue.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index 9f7f38cf1..2f11024c4 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -47,6 +47,7 @@ where creator, vec![to], context.pool(), + true, ) .await?; } @@ -85,6 +86,7 @@ where community, to, context.pool(), + true, ) .await?; @@ -114,6 +116,7 @@ where creator, vec![inbox], context.pool(), + true, ) .await?; } @@ -143,6 +146,7 @@ where creator, mentions, context.pool(), + false, // Don't create a new DB row ) .await?; Ok(()) @@ -158,6 +162,7 @@ async fn send_activity_internal( actor: &dyn ActorType, to: Vec, pool: &DbPool, + insert_into_db: bool, ) -> Result<(), LemmyError> where T: AsObject + Extends, @@ -174,7 +179,12 @@ where let activity = activity.into_any_base()?; let serialised_activity = serde_json::to_string(&activity)?; - insert_activity(actor.user_id(), activity.clone(), true, pool).await?; + + // This is necessary because send_comment and send_comment_mentions + // might send the same ap_id + if insert_into_db { + insert_activity(actor.user_id(), activity.clone(), true, pool).await?; + } // TODO: it would make sense to create a separate task for each destination server let message = SendActivityTask { From 60730e81d9ef6b069ae7e63b79740366d2d44645 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 6 Oct 2020 18:28:31 +0200 Subject: [PATCH 10/17] Avoid duplicate comment send, better activity logging --- lemmy_apub/src/activity_queue.rs | 33 +++++++++++++++++++------ lemmy_apub/src/comment.rs | 7 +----- lemmy_apub/src/inbox/community_inbox.rs | 10 +++++--- lemmy_apub/src/inbox/shared_inbox.rs | 10 ++++---- lemmy_apub/src/inbox/user_inbox.rs | 7 +++++- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index 2f11024c4..846756593 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -6,7 +6,7 @@ use crate::{ ActorType, }; use activitystreams::{ - base::{Extends, ExtendsExt}, + base::{BaseExt, Extends, ExtendsExt}, object::AsObject, }; use anyhow::{anyhow, Context, Error}; @@ -23,9 +23,9 @@ use itertools::Itertools; use lemmy_db::{community::Community, user::User_, DbPool}; use lemmy_utils::{location_info, settings::Settings, LemmyError}; use lemmy_websocket::LemmyContext; -use log::warn; +use log::{debug, warn}; use reqwest::Client; -use serde::{Deserialize, Serialize}; +use serde::{export::fmt::Debug, Deserialize, Serialize}; use std::{collections::BTreeMap, future::Future, pin::Pin}; use url::Url; @@ -36,11 +36,12 @@ pub async fn send_activity_single_dest( context: &LemmyContext, ) -> Result<(), LemmyError> where - T: AsObject + Extends, + T: AsObject + Extends + Debug + BaseExt, Kind: Serialize, >::Error: From + Send + Sync + 'static, { if check_is_apub_id_valid(&to).is_ok() { + debug!("Sending activity {:?} to {}", &activity.id_unchecked(), &to); send_activity_internal( context.activity_queue(), activity, @@ -62,7 +63,7 @@ pub async fn send_to_community_followers( sender_shared_inbox: Option, ) -> Result<(), LemmyError> where - T: AsObject + Extends, + T: AsObject + Extends + Debug + BaseExt, Kind: Serialize, >::Error: From + Send + Sync + 'static, { @@ -79,6 +80,11 @@ where .unique() .map(|inbox| inbox.to_owned()) .collect(); + debug!( + "Sending activity {:?} to followers of {}", + &activity.id_unchecked(), + &community.actor_id + ); send_activity_internal( context.activity_queue(), @@ -100,7 +106,7 @@ pub async fn send_to_community( context: &LemmyContext, ) -> Result<(), LemmyError> where - T: AsObject + Extends, + T: AsObject + Extends + Debug + BaseExt, Kind: Serialize, >::Error: From + Send + Sync + 'static, { @@ -110,6 +116,11 @@ where } else { let inbox = community.get_shared_inbox_url()?; check_is_apub_id_valid(&inbox)?; + debug!( + "Sending activity {:?} to community {}", + &activity.id_unchecked(), + &community.actor_id + ); send_activity_internal( context.activity_queue(), activity, @@ -131,10 +142,16 @@ pub async fn send_comment_mentions( context: &LemmyContext, ) -> Result<(), LemmyError> where - T: AsObject + Extends, + T: AsObject + Extends + Debug + BaseExt, Kind: Serialize, >::Error: From + Send + Sync + 'static, { + dbg!(&mentions, &activity.id_unchecked()); + debug!( + "Sending mentions activity {:?} to {:?}", + &activity.id_unchecked(), + &mentions + ); let mentions = mentions .iter() .filter(|inbox| check_is_apub_id_valid(inbox).is_ok()) @@ -165,7 +182,7 @@ async fn send_activity_internal( insert_into_db: bool, ) -> Result<(), LemmyError> where - T: AsObject + Extends, + T: AsObject + Extends + Debug, Kind: Serialize, >::Error: From + Send + Sync + 'static, { diff --git a/lemmy_apub/src/comment.rs b/lemmy_apub/src/comment.rs index 35be5522e..33ad1d8a1 100644 --- a/lemmy_apub/src/comment.rs +++ b/lemmy_apub/src/comment.rs @@ -506,12 +506,7 @@ async fn collect_non_local_mentions_and_addresses( } } - let mut inboxes: Vec = vec![]; - if !community.local { - inboxes.push(community.get_shared_inbox_url()?); - } - inboxes.extend(mention_inboxes); - inboxes = inboxes.into_iter().unique().collect(); + let inboxes = mention_inboxes.into_iter().unique().collect(); Ok(MentionsAndAddresses { addressed_ccs, diff --git a/lemmy_apub/src/inbox/community_inbox.rs b/lemmy_apub/src/inbox/community_inbox.rs index ee75fa005..d979e27fe 100644 --- a/lemmy_apub/src/inbox/community_inbox.rs +++ b/lemmy_apub/src/inbox/community_inbox.rs @@ -57,14 +57,16 @@ pub async fn community_inbox( .into(), ); } - debug!( - "Community {} received activity {:?}", - &community.name, &activity - ); let user_uri = activity .actor()? .as_single_xsd_any_uri() .context(location_info!())?; + debug!( + "Community {} inbox received activity {:?} from {}", + community.name, + &activity.id_unchecked(), + &user_uri + ); check_is_apub_id_valid(user_uri)?; let user = get_or_fetch_and_upsert_user(&user_uri, &context).await?; diff --git a/lemmy_apub/src/inbox/shared_inbox.rs b/lemmy_apub/src/inbox/shared_inbox.rs index 677ceadf9..2d69336fd 100644 --- a/lemmy_apub/src/inbox/shared_inbox.rs +++ b/lemmy_apub/src/inbox/shared_inbox.rs @@ -60,17 +60,17 @@ pub async fn shared_inbox( ) -> Result { let activity = input.into_inner(); - let json = serde_json::to_string_pretty(&activity)?; - debug!("Shared inbox received activity: {}", json); - - // TODO: if we already received an activity with identical ID, then ignore this (same in other inboxes) - let sender = &activity .actor()? .to_owned() .single_xsd_any_uri() .context(location_info!())?; let community = get_community_id_from_activity(&activity)?; + debug!( + "Shared inbox received activity {:?} from {}", + &activity.id_unchecked(), + &sender + ); check_is_apub_id_valid(sender)?; check_is_apub_id_valid(&community)?; diff --git a/lemmy_apub/src/inbox/user_inbox.rs b/lemmy_apub/src/inbox/user_inbox.rs index a7050c45c..c411f5b7a 100644 --- a/lemmy_apub/src/inbox/user_inbox.rs +++ b/lemmy_apub/src/inbox/user_inbox.rs @@ -50,12 +50,17 @@ pub async fn user_inbox( ) -> Result { let activity = input.into_inner(); let username = path.into_inner(); - debug!("User {} received activity: {:?}", &username, &activity); let actor_uri = activity .actor()? .as_single_xsd_any_uri() .context(location_info!())?; + debug!( + "User {} inbox received activity {:?} from {}", + username, + &activity.id_unchecked(), + &actor_uri + ); check_is_apub_id_valid(actor_uri)?; From 26883208cd6fe91f200902e86f9ae0591912eadb Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Tue, 6 Oct 2020 19:19:53 +0200 Subject: [PATCH 11/17] Create separate SendActivityTask for each destination --- lemmy_apub/src/activity_queue.rs | 58 +++++++++++++++----------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index 846756593..a887eb93f 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -203,15 +203,15 @@ where insert_activity(actor.user_id(), activity.clone(), true, pool).await?; } - // TODO: it would make sense to create a separate task for each destination server - let message = SendActivityTask { - activity: serialised_activity, - to, - actor_id: actor.actor_id()?, - private_key: actor.private_key().context(location_info!())?, - }; - - activity_sender.queue::(message)?; + for t in to { + let message = SendActivityTask { + activity: serialised_activity.to_owned(), + to: t, + actor_id: actor.actor_id()?, + private_key: actor.private_key().context(location_info!())?, + }; + activity_sender.queue::(message)?; + } Ok(()) } @@ -219,7 +219,7 @@ where #[derive(Clone, Debug, Deserialize, Serialize)] struct SendActivityTask { activity: String, - to: Vec, + to: Url, actor_id: Url, private_key: String, } @@ -234,27 +234,25 @@ impl ActixJob for SendActivityTask { fn run(self, state: Self::State) -> Self::Future { Box::pin(async move { - for to_url in &self.to { - let mut headers = BTreeMap::::new(); - headers.insert("Content-Type".into(), "application/json".into()); - let result = sign_and_send( - &state.client, - headers, - to_url, - self.activity.clone(), - &self.actor_id, - self.private_key.to_owned(), - ) - .await; + let mut headers = BTreeMap::::new(); + headers.insert("Content-Type".into(), "application/json".into()); + let result = sign_and_send( + &state.client, + headers, + &self.to, + self.activity.clone(), + &self.actor_id, + self.private_key.to_owned(), + ) + .await; - if let Err(e) = result { - warn!("{}", e); - return Err(anyhow!( - "Failed to send activity {} to {}", - &self.activity, - to_url - )); - } + if let Err(e) = result { + warn!("{}", e); + return Err(anyhow!( + "Failed to send activity {} to {}", + &self.activity, + self.to + )); } Ok(()) }) From db7027a3674cb1c9f07822d3e56743af4133f523 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 6 Oct 2020 12:31:16 -0500 Subject: [PATCH 12/17] Removing on_conflict as it may not work with table triggers (user_fast, etc) --- lemmy_db/src/comment.rs | 16 +++++++++------- lemmy_db/src/community.rs | 19 ++++++++++++------- lemmy_db/src/post.rs | 16 +++++++++------- lemmy_db/src/private_message.rs | 19 ++++++++++++------- lemmy_db/src/user.rs | 15 +++++++++------ 5 files changed, 51 insertions(+), 34 deletions(-) diff --git a/lemmy_db/src/comment.rs b/lemmy_db/src/comment.rs index 398ea78bf..ff0c61203 100644 --- a/lemmy_db/src/comment.rs +++ b/lemmy_db/src/comment.rs @@ -170,13 +170,15 @@ impl Comment { } pub fn upsert(conn: &PgConnection, comment_form: &CommentForm) -> Result { - use crate::schema::comment::dsl::*; - insert_into(comment) - .values(comment_form) - .on_conflict(ap_id) - .do_update() - .set(comment_form) - .get_result::(conn) + let existing = Self::read_from_apub_id( + conn, + comment_form.ap_id.as_ref().unwrap_or(&"none".to_string()), + ); + match existing { + Err(NotFound {}) => Ok(Self::create(conn, &comment_form)?), + Ok(p) => Ok(Self::update(conn, p.id, &comment_form)?), + Err(e) => Err(e), + } } } diff --git a/lemmy_db/src/community.rs b/lemmy_db/src/community.rs index 24cf7e32f..ece96b0c7 100644 --- a/lemmy_db/src/community.rs +++ b/lemmy_db/src/community.rs @@ -166,13 +166,18 @@ impl Community { } pub fn upsert(conn: &PgConnection, community_form: &CommunityForm) -> Result { - use crate::schema::community::dsl::*; - insert_into(community) - .values(community_form) - .on_conflict(actor_id) - .do_update() - .set(community_form) - .get_result::(conn) + let existing = Self::read_from_actor_id( + conn, + community_form + .actor_id + .as_ref() + .unwrap_or(&"none".to_string()), + ); + match existing { + Err(NotFound {}) => Ok(Self::create(conn, &community_form)?), + Ok(p) => Ok(Self::update(conn, p.id, &community_form)?), + Err(e) => Err(e), + } } } diff --git a/lemmy_db/src/post.rs b/lemmy_db/src/post.rs index 724c342c6..1ca38b310 100644 --- a/lemmy_db/src/post.rs +++ b/lemmy_db/src/post.rs @@ -179,13 +179,15 @@ impl Post { } pub fn upsert(conn: &PgConnection, post_form: &PostForm) -> Result { - use crate::schema::post::dsl::*; - insert_into(post) - .values(post_form) - .on_conflict(ap_id) - .do_update() - .set(post_form) - .get_result::(conn) + let existing = Self::read_from_apub_id( + conn, + post_form.ap_id.as_ref().unwrap_or(&"none".to_string()), + ); + match existing { + Err(NotFound {}) => Ok(Self::create(conn, &post_form)?), + Ok(p) => Ok(Self::update(conn, p.id, &post_form)?), + Err(e) => Err(e), + } } } diff --git a/lemmy_db/src/private_message.rs b/lemmy_db/src/private_message.rs index 988d97d3b..36aa7aa8a 100644 --- a/lemmy_db/src/private_message.rs +++ b/lemmy_db/src/private_message.rs @@ -124,13 +124,18 @@ impl PrivateMessage { conn: &PgConnection, private_message_form: &PrivateMessageForm, ) -> Result { - use crate::schema::private_message::dsl::*; - insert_into(private_message) - .values(private_message_form) - .on_conflict(ap_id) - .do_update() - .set(private_message_form) - .get_result::(conn) + let existing = Self::read_from_apub_id( + conn, + private_message_form + .ap_id + .as_ref() + .unwrap_or(&"none".to_string()), + ); + match existing { + Err(NotFound {}) => Ok(Self::create(conn, &private_message_form)?), + Ok(p) => Ok(Self::update(conn, p.id, &private_message_form)?), + Err(e) => Err(e), + } } } diff --git a/lemmy_db/src/user.rs b/lemmy_db/src/user.rs index 83f0559ab..88bccf6a5 100644 --- a/lemmy_db/src/user.rs +++ b/lemmy_db/src/user.rs @@ -161,12 +161,15 @@ impl User_ { } pub fn upsert(conn: &PgConnection, user_form: &UserForm) -> Result { - insert_into(user_) - .values(user_form) - .on_conflict(actor_id) - .do_update() - .set(user_form) - .get_result::(conn) + let existing = Self::read_from_actor_id( + conn, + user_form.actor_id.as_ref().unwrap_or(&"none".to_string()), + ); + match existing { + Err(NotFound {}) => Ok(Self::create(conn, &user_form)?), + Ok(p) => Ok(Self::update(conn, p.id, &user_form)?), + Err(e) => Err(e), + } } } From 9fe3efcb322d6de8b8558b3032b3394837a78efe Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 7 Oct 2020 09:40:36 -0500 Subject: [PATCH 13/17] Trimming allowed and blocked instances --- lemmy_utils/src/settings.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemmy_utils/src/settings.rs b/lemmy_utils/src/settings.rs index d3775c4b4..797c6fba1 100644 --- a/lemmy_utils/src/settings.rs +++ b/lemmy_utils/src/settings.rs @@ -134,7 +134,7 @@ impl Settings { .federation .allowed_instances .split(',') - .map(|d| d.to_string()) + .map(|d| d.trim().to_string()) .collect(); // The defaults.hjson config always returns a [""] @@ -148,7 +148,7 @@ impl Settings { .federation .blocked_instances .split(',') - .map(|d| d.to_string()) + .map(|d| d.trim().to_string()) .collect(); // The defaults.hjson config always returns a [""] From fd257a6d39c1e0631d3a7142a11b6dde8bf049eb Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 7 Oct 2020 19:05:46 -0500 Subject: [PATCH 14/17] Adding no conflict triggers. Fixes #1179 --- docker/federation/docker-compose.yml | 10 +- .../down.sql | 128 +++++++++++++++++ .../up.sql | 132 ++++++++++++++++++ 3 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 migrations/2020-10-07-234221_fix_fast_triggers/down.sql create mode 100644 migrations/2020-10-07-234221_fix_fast_triggers/up.sql diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 8f01eadf7..91a816efe 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -29,7 +29,7 @@ services: - ./volumes/pictrs_alpha:/mnt lemmy-alpha-ui: - image: dessalines/lemmy-ui:v0.0.14 + image: dessalines/lemmy-ui:dev environment: - LEMMY_INTERNAL_HOST=lemmy-alpha:8541 - LEMMY_EXTERNAL_HOST=localhost:8541 @@ -68,7 +68,7 @@ services: - ./volumes/postgres_alpha:/var/lib/postgresql/data lemmy-beta-ui: - image: dessalines/lemmy-ui:v0.0.14 + image: dessalines/lemmy-ui:dev environment: - LEMMY_INTERNAL_HOST=lemmy-beta:8551 - LEMMY_EXTERNAL_HOST=localhost:8551 @@ -107,7 +107,7 @@ services: - ./volumes/postgres_beta:/var/lib/postgresql/data lemmy-gamma-ui: - image: dessalines/lemmy-ui:v0.0.14 + image: dessalines/lemmy-ui:dev environment: - LEMMY_INTERNAL_HOST=lemmy-gamma:8561 - LEMMY_EXTERNAL_HOST=localhost:8561 @@ -147,7 +147,7 @@ services: # An instance with only an allowlist for beta lemmy-delta-ui: - image: dessalines/lemmy-ui:v0.0.14 + image: dessalines/lemmy-ui:dev environment: - LEMMY_INTERNAL_HOST=lemmy-delta:8571 - LEMMY_EXTERNAL_HOST=localhost:8571 @@ -187,7 +187,7 @@ services: # An instance who has a blocklist, with lemmy-alpha blocked lemmy-epsilon-ui: - image: dessalines/lemmy-ui:v0.0.14 + image: dessalines/lemmy-ui:dev environment: - LEMMY_INTERNAL_HOST=lemmy-epsilon:8581 - LEMMY_EXTERNAL_HOST=localhost:8581 diff --git a/migrations/2020-10-07-234221_fix_fast_triggers/down.sql b/migrations/2020-10-07-234221_fix_fast_triggers/down.sql new file mode 100644 index 000000000..6b44e8a40 --- /dev/null +++ b/migrations/2020-10-07-234221_fix_fast_triggers/down.sql @@ -0,0 +1,128 @@ +create or replace function refresh_community() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from community_aggregates_fast where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from community_aggregates_fast where id = OLD.id; + insert into community_aggregates_fast select * from community_aggregates_view where id = NEW.id; + + -- Update user view due to owner changes + delete from user_fast where id = NEW.creator_id; + insert into user_fast select * from user_view where id = NEW.creator_id; + + -- Update post view due to community changes + delete from post_aggregates_fast where community_id = NEW.id; + insert into post_aggregates_fast select * from post_aggregates_view where community_id = NEW.id; + + -- TODO make sure this shows up in the users page ? + ELSIF (TG_OP = 'INSERT') THEN + insert into community_aggregates_fast select * from community_aggregates_view where id = NEW.id; + END IF; + + return null; +end $$; + +create or replace function refresh_user() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from user_fast where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from user_fast where id = OLD.id; + insert into user_fast select * from user_view where id = NEW.id; + + -- Refresh post_fast, cause of user info changes + delete from post_aggregates_fast where creator_id = NEW.id; + insert into post_aggregates_fast select * from post_aggregates_view where creator_id = NEW.id; + + delete from comment_aggregates_fast where creator_id = NEW.id; + insert into comment_aggregates_fast select * from comment_aggregates_view where creator_id = NEW.id; + + ELSIF (TG_OP = 'INSERT') THEN + insert into user_fast select * from user_view where id = NEW.id; + END IF; + + return null; +end $$; + +create or replace function refresh_post() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from post_aggregates_fast where id = OLD.id; + + -- Update community number of posts + update community_aggregates_fast set number_of_posts = number_of_posts - 1 where id = OLD.community_id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from post_aggregates_fast where id = OLD.id; + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.id; + + -- Update that users number of posts, post score + delete from user_fast where id = NEW.creator_id; + insert into user_fast select * from user_view where id = NEW.creator_id; + + -- Update community number of posts + update community_aggregates_fast set number_of_posts = number_of_posts + 1 where id = NEW.community_id; + + -- Update the hot rank on the post table + -- TODO this might not correctly update it, using a 1 week interval + update post_aggregates_fast as paf + set hot_rank = pav.hot_rank + from post_aggregates_view as pav + where paf.id = pav.id and (pav.published > ('now'::timestamp - '1 week'::interval)); + END IF; + + return null; +end $$; + +create or replace function refresh_comment() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from comment_aggregates_fast where id = OLD.id; + + -- Update community number of comments + update community_aggregates_fast as caf + set number_of_comments = number_of_comments - 1 + from post as p + where caf.id = p.community_id and p.id = OLD.post_id; + + ELSIF (TG_OP = 'UPDATE') THEN + delete from comment_aggregates_fast where id = OLD.id; + insert into comment_aggregates_fast select * from comment_aggregates_view where id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + insert into comment_aggregates_fast select * from comment_aggregates_view where id = NEW.id; + + -- Update user view due to comment count + update user_fast + set number_of_comments = number_of_comments + 1 + where id = NEW.creator_id; + + -- Update post view due to comment count, new comment activity time, but only on new posts + -- TODO this could be done more efficiently + delete from post_aggregates_fast where id = NEW.post_id; + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.post_id; + + -- Force the hot rank as zero on week-older posts + update post_aggregates_fast as paf + set hot_rank = 0 + where paf.id = NEW.post_id and (paf.published < ('now'::timestamp - '1 week'::interval)); + + -- Update community number of comments + update community_aggregates_fast as caf + set number_of_comments = number_of_comments + 1 + from post as p + where caf.id = p.community_id and p.id = NEW.post_id; + + END IF; + + return null; +end $$; + diff --git a/migrations/2020-10-07-234221_fix_fast_triggers/up.sql b/migrations/2020-10-07-234221_fix_fast_triggers/up.sql new file mode 100644 index 000000000..39f774470 --- /dev/null +++ b/migrations/2020-10-07-234221_fix_fast_triggers/up.sql @@ -0,0 +1,132 @@ +-- This adds on conflict do nothing triggers to all the insert_intos +-- Github issue: https://github.com/LemmyNet/lemmy/issues/1179 + +create or replace function refresh_community() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from community_aggregates_fast where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from community_aggregates_fast where id = OLD.id; + insert into community_aggregates_fast select * from community_aggregates_view where id = NEW.id on conflict (id) do nothing; + + -- Update user view due to owner changes + delete from user_fast where id = NEW.creator_id; + insert into user_fast select * from user_view where id = NEW.creator_id on conflict (id) do nothing; + + -- Update post view due to community changes + delete from post_aggregates_fast where community_id = NEW.id; + insert into post_aggregates_fast select * from post_aggregates_view where community_id = NEW.id on conflict (id) do nothing; + + -- TODO make sure this shows up in the users page ? + ELSIF (TG_OP = 'INSERT') THEN + insert into community_aggregates_fast select * from community_aggregates_view where id = NEW.id; + END IF; + + return null; +end $$; + +create or replace function refresh_user() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from user_fast where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from user_fast where id = OLD.id; + insert into user_fast select * from user_view where id = NEW.id on conflict(id) do nothing; + + -- Refresh post_fast, cause of user info changes + delete from post_aggregates_fast where creator_id = NEW.id; + insert into post_aggregates_fast select * from post_aggregates_view where creator_id = NEW.id on conflict (id) do nothing; + + delete from comment_aggregates_fast where creator_id = NEW.id; + insert into comment_aggregates_fast select * from comment_aggregates_view where creator_id = NEW.id on conflict (id) do nothing; + + ELSIF (TG_OP = 'INSERT') THEN + insert into user_fast select * from user_view where id = NEW.id; + END IF; + + return null; +end $$; + +create or replace function refresh_post() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from post_aggregates_fast where id = OLD.id; + + -- Update community number of posts + update community_aggregates_fast set number_of_posts = number_of_posts - 1 where id = OLD.community_id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from post_aggregates_fast where id = OLD.id; + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.id on conflict (id) do nothing; + + ELSIF (TG_OP = 'INSERT') THEN + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.id; + + -- Update that users number of posts, post score + delete from user_fast where id = NEW.creator_id; + insert into user_fast select * from user_view where id = NEW.creator_id on conflict (id) do nothing; + + -- Update community number of posts + update community_aggregates_fast set number_of_posts = number_of_posts + 1 where id = NEW.community_id; + + -- Update the hot rank on the post table + -- TODO this might not correctly update it, using a 1 week interval + update post_aggregates_fast as paf + set hot_rank = pav.hot_rank + from post_aggregates_view as pav + where paf.id = pav.id and (pav.published > ('now'::timestamp - '1 week'::interval)); + END IF; + + return null; +end $$; + + +create or replace function refresh_comment() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from comment_aggregates_fast where id = OLD.id; + + -- Update community number of comments + update community_aggregates_fast as caf + set number_of_comments = number_of_comments - 1 + from post as p + where caf.id = p.community_id and p.id = OLD.post_id; + + ELSIF (TG_OP = 'UPDATE') THEN + delete from comment_aggregates_fast where id = OLD.id; + insert into comment_aggregates_fast select * from comment_aggregates_view where id = NEW.id on conflict (id) do nothing; + ELSIF (TG_OP = 'INSERT') THEN + insert into comment_aggregates_fast select * from comment_aggregates_view where id = NEW.id; + + -- Update user view due to comment count + update user_fast + set number_of_comments = number_of_comments + 1 + where id = NEW.creator_id; + + -- Update post view due to comment count, new comment activity time, but only on new posts + -- TODO this could be done more efficiently + delete from post_aggregates_fast where id = NEW.post_id; + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.post_id on conflict (id) do nothing; + + -- Force the hot rank as zero on week-older posts + update post_aggregates_fast as paf + set hot_rank = 0 + where paf.id = NEW.post_id and (paf.published < ('now'::timestamp - '1 week'::interval)); + + -- Update community number of comments + update community_aggregates_fast as caf + set number_of_comments = number_of_comments + 1 + from post as p + where caf.id = p.community_id and p.id = NEW.post_id; + + END IF; + + return null; +end $$; From b4c730e537bc9b44478f4396e34338607be6d6b5 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 7 Oct 2020 19:08:06 -0500 Subject: [PATCH 15/17] Revert "Removing on_conflict as it may not work with table triggers (user_fast, etc)" This reverts commit db7027a3674cb1c9f07822d3e56743af4133f523. --- lemmy_db/src/comment.rs | 16 +++++++--------- lemmy_db/src/community.rs | 19 +++++++------------ lemmy_db/src/post.rs | 16 +++++++--------- lemmy_db/src/private_message.rs | 19 +++++++------------ lemmy_db/src/user.rs | 15 ++++++--------- 5 files changed, 34 insertions(+), 51 deletions(-) diff --git a/lemmy_db/src/comment.rs b/lemmy_db/src/comment.rs index ff0c61203..398ea78bf 100644 --- a/lemmy_db/src/comment.rs +++ b/lemmy_db/src/comment.rs @@ -170,15 +170,13 @@ impl Comment { } pub fn upsert(conn: &PgConnection, comment_form: &CommentForm) -> Result { - let existing = Self::read_from_apub_id( - conn, - comment_form.ap_id.as_ref().unwrap_or(&"none".to_string()), - ); - match existing { - Err(NotFound {}) => Ok(Self::create(conn, &comment_form)?), - Ok(p) => Ok(Self::update(conn, p.id, &comment_form)?), - Err(e) => Err(e), - } + use crate::schema::comment::dsl::*; + insert_into(comment) + .values(comment_form) + .on_conflict(ap_id) + .do_update() + .set(comment_form) + .get_result::(conn) } } diff --git a/lemmy_db/src/community.rs b/lemmy_db/src/community.rs index ece96b0c7..24cf7e32f 100644 --- a/lemmy_db/src/community.rs +++ b/lemmy_db/src/community.rs @@ -166,18 +166,13 @@ impl Community { } pub fn upsert(conn: &PgConnection, community_form: &CommunityForm) -> Result { - let existing = Self::read_from_actor_id( - conn, - community_form - .actor_id - .as_ref() - .unwrap_or(&"none".to_string()), - ); - match existing { - Err(NotFound {}) => Ok(Self::create(conn, &community_form)?), - Ok(p) => Ok(Self::update(conn, p.id, &community_form)?), - Err(e) => Err(e), - } + use crate::schema::community::dsl::*; + insert_into(community) + .values(community_form) + .on_conflict(actor_id) + .do_update() + .set(community_form) + .get_result::(conn) } } diff --git a/lemmy_db/src/post.rs b/lemmy_db/src/post.rs index 1ca38b310..724c342c6 100644 --- a/lemmy_db/src/post.rs +++ b/lemmy_db/src/post.rs @@ -179,15 +179,13 @@ impl Post { } pub fn upsert(conn: &PgConnection, post_form: &PostForm) -> Result { - let existing = Self::read_from_apub_id( - conn, - post_form.ap_id.as_ref().unwrap_or(&"none".to_string()), - ); - match existing { - Err(NotFound {}) => Ok(Self::create(conn, &post_form)?), - Ok(p) => Ok(Self::update(conn, p.id, &post_form)?), - Err(e) => Err(e), - } + use crate::schema::post::dsl::*; + insert_into(post) + .values(post_form) + .on_conflict(ap_id) + .do_update() + .set(post_form) + .get_result::(conn) } } diff --git a/lemmy_db/src/private_message.rs b/lemmy_db/src/private_message.rs index 36aa7aa8a..988d97d3b 100644 --- a/lemmy_db/src/private_message.rs +++ b/lemmy_db/src/private_message.rs @@ -124,18 +124,13 @@ impl PrivateMessage { conn: &PgConnection, private_message_form: &PrivateMessageForm, ) -> Result { - let existing = Self::read_from_apub_id( - conn, - private_message_form - .ap_id - .as_ref() - .unwrap_or(&"none".to_string()), - ); - match existing { - Err(NotFound {}) => Ok(Self::create(conn, &private_message_form)?), - Ok(p) => Ok(Self::update(conn, p.id, &private_message_form)?), - Err(e) => Err(e), - } + use crate::schema::private_message::dsl::*; + insert_into(private_message) + .values(private_message_form) + .on_conflict(ap_id) + .do_update() + .set(private_message_form) + .get_result::(conn) } } diff --git a/lemmy_db/src/user.rs b/lemmy_db/src/user.rs index 88bccf6a5..83f0559ab 100644 --- a/lemmy_db/src/user.rs +++ b/lemmy_db/src/user.rs @@ -161,15 +161,12 @@ impl User_ { } pub fn upsert(conn: &PgConnection, user_form: &UserForm) -> Result { - let existing = Self::read_from_actor_id( - conn, - user_form.actor_id.as_ref().unwrap_or(&"none".to_string()), - ); - match existing { - Err(NotFound {}) => Ok(Self::create(conn, &user_form)?), - Ok(p) => Ok(Self::update(conn, p.id, &user_form)?), - Err(e) => Err(e), - } + insert_into(user_) + .values(user_form) + .on_conflict(actor_id) + .do_update() + .set(user_form) + .get_result::(conn) } } From e9ce14069e95991027b9731dbd668dac229ffd72 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 7 Oct 2020 21:55:15 -0500 Subject: [PATCH 16/17] Removing some unecessary logging. --- lemmy_apub/src/community.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemmy_apub/src/community.rs b/lemmy_apub/src/community.rs index b1c28d75b..c2b4571fa 100644 --- a/lemmy_apub/src/community.rs +++ b/lemmy_apub/src/community.rs @@ -50,7 +50,6 @@ use lemmy_utils::{ LemmyError, }; use lemmy_websocket::LemmyContext; -use log::debug; use serde::Deserialize; use url::Url; @@ -282,10 +281,6 @@ impl ActorType for Community { .unique() .collect(); - // TODO remove this logging line - let inboxes_json = serde_json::to_string_pretty(&inboxes)?; - debug!("Community follower inboxes: {}", inboxes_json); - Ok(inboxes) } From 7fbad900d7ef5d77cf62f02d8ebd61f963abd342 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Thu, 8 Oct 2020 12:38:44 -0500 Subject: [PATCH 17/17] Addressing a few comments. --- lemmy_apub/src/activity_queue.rs | 3 +-- lemmy_apub/src/inbox/community_inbox.rs | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lemmy_apub/src/activity_queue.rs b/lemmy_apub/src/activity_queue.rs index a887eb93f..988f1a477 100644 --- a/lemmy_apub/src/activity_queue.rs +++ b/lemmy_apub/src/activity_queue.rs @@ -146,7 +146,6 @@ where Kind: Serialize, >::Error: From + Send + Sync + 'static, { - dbg!(&mentions, &activity.id_unchecked()); debug!( "Sending mentions activity {:?} to {:?}", &activity.id_unchecked(), @@ -191,7 +190,7 @@ where } for to_url in &to { - assert!(check_is_apub_id_valid(&to_url).is_ok()); + check_is_apub_id_valid(&to_url)?; } let activity = activity.into_any_base()?; diff --git a/lemmy_apub/src/inbox/community_inbox.rs b/lemmy_apub/src/inbox/community_inbox.rs index e4ee7b375..9ac589c70 100644 --- a/lemmy_apub/src/inbox/community_inbox.rs +++ b/lemmy_apub/src/inbox/community_inbox.rs @@ -20,7 +20,7 @@ use lemmy_db::{ use lemmy_structs::blocking; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; -use log::debug; +use log::info; use serde::{Deserialize, Serialize}; use std::fmt::Debug; @@ -57,7 +57,7 @@ pub async fn community_inbox( .into(), ); } - debug!( + info!( "Community {} received activity {:?}", &community.name, &activity ); @@ -65,7 +65,7 @@ pub async fn community_inbox( .actor()? .as_single_xsd_any_uri() .context(location_info!())?; - debug!( + info!( "Community {} inbox received activity {:?} from {}", community.name, &activity.id_unchecked(),