From 27e409442a98a6cdfb0ee3c93561c18a5e762d5c Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 22 Sep 2021 15:58:34 +0200 Subject: [PATCH] rewrite person fetch to use new fetcher --- crates/apub/src/activities/comment/mod.rs | 6 +- .../apub/src/activities/community/add_mod.rs | 4 +- .../src/activities/community/block_user.rs | 5 +- .../src/activities/community/remove_mod.rs | 5 +- .../activities/community/undo_block_user.rs | 5 +- crates/apub/src/activities/deletion/delete.rs | 4 +- crates/apub/src/activities/deletion/mod.rs | 4 +- .../apub/src/activities/following/accept.rs | 4 +- .../apub/src/activities/following/follow.rs | 4 +- crates/apub/src/activities/following/undo.rs | 4 +- crates/apub/src/activities/mod.rs | 6 +- .../src/activities/post/create_or_update.rs | 4 +- .../apub/src/activities/voting/undo_vote.rs | 4 +- crates/apub/src/activities/voting/vote.rs | 4 +- crates/apub/src/fetcher/community.rs | 9 ++- crates/apub/src/fetcher/mod.rs | 7 +- crates/apub/src/fetcher/new_fetcher.rs | 25 +++++-- crates/apub/src/fetcher/person.rs | 70 ------------------- crates/apub/src/fetcher/post_or_comment.rs | 5 ++ crates/apub/src/fetcher/search.rs | 4 +- crates/apub/src/objects/comment.rs | 5 +- crates/apub/src/objects/mod.rs | 1 - crates/apub/src/objects/post.rs | 5 +- crates/apub/src/objects/private_message.rs | 10 ++- crates/db_queries/src/lib.rs | 4 ++ crates/db_queries/src/source/activity.rs | 2 - crates/db_queries/src/source/comment.rs | 6 ++ crates/db_queries/src/source/community.rs | 6 ++ crates/db_queries/src/source/person.rs | 6 ++ crates/db_queries/src/source/post.rs | 6 ++ .../db_queries/src/source/private_message.rs | 6 ++ 31 files changed, 109 insertions(+), 131 deletions(-) delete mode 100644 crates/apub/src/fetcher/person.rs diff --git a/crates/apub/src/activities/comment/mod.rs b/crates/apub/src/activities/comment/mod.rs index 44b8d245a..44a6ea405 100644 --- a/crates/apub/src/activities/comment/mod.rs +++ b/crates/apub/src/activities/comment/mod.rs @@ -1,4 +1,4 @@ -use crate::{fetcher::person::get_or_fetch_and_upsert_person, ActorType}; +use crate::{fetcher::new_fetcher::dereference, ActorType}; use activitystreams::{ base::BaseExt, link::{LinkExt, Mention}, @@ -33,7 +33,7 @@ async fn get_notif_recipients( ) -> Result, LemmyError> { let post_id = comment.post_id; let post = blocking(context.pool(), move |conn| Post::read(conn, post_id)).await??; - let actor = get_or_fetch_and_upsert_person(actor, context, request_counter).await?; + let actor = dereference::(actor, context, request_counter).await?; // Note: // Although mentions could be gotten from the post tags (they are included there), or the ccs, @@ -79,7 +79,7 @@ pub async fn collect_non_local_mentions( debug!("mention actor_id: {}", actor_id); addressed_ccs.push(actor_id.to_owned().to_string().parse()?); - let mention_person = get_or_fetch_and_upsert_person(&actor_id, context, &mut 0).await?; + let mention_person = dereference::(&actor_id, context, &mut 0).await?; inboxes.push(mention_person.get_shared_inbox_or_inbox_url()); let mut mention_tag = Mention::new(); diff --git a/crates/apub/src/activities/community/add_mod.rs b/crates/apub/src/activities/community/add_mod.rs index 56013d4ae..e1b6864cd 100644 --- a/crates/apub/src/activities/community/add_mod.rs +++ b/crates/apub/src/activities/community/add_mod.rs @@ -9,7 +9,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, generate_moderators_url, ActorType, }; @@ -95,7 +95,7 @@ impl ActivityHandler for AddMod { ) -> Result<(), LemmyError> { let community = get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; - let new_mod = get_or_fetch_and_upsert_person(&self.object, context, request_counter).await?; + let new_mod = dereference::(&self.object, context, request_counter).await?; // If we had to refetch the community while parsing the activity, then the new mod has already // been added. Skip it here as it would result in a duplicate key error. diff --git a/crates/apub/src/activities/community/block_user.rs b/crates/apub/src/activities/community/block_user.rs index 777faf9a9..b84c1304b 100644 --- a/crates/apub/src/activities/community/block_user.rs +++ b/crates/apub/src/activities/community/block_user.rs @@ -8,7 +8,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, ActorType, }; use activitystreams::{ @@ -104,8 +104,7 @@ impl ActivityHandler for BlockUserFromCommunity { ) -> Result<(), LemmyError> { let community = get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; - let blocked_user = - get_or_fetch_and_upsert_person(&self.object, context, request_counter).await?; + let blocked_user = dereference::(&self.object, context, request_counter).await?; let community_user_ban_form = CommunityPersonBanForm { community_id: community.id, diff --git a/crates/apub/src/activities/community/remove_mod.rs b/crates/apub/src/activities/community/remove_mod.rs index c71755678..916da64b5 100644 --- a/crates/apub/src/activities/community/remove_mod.rs +++ b/crates/apub/src/activities/community/remove_mod.rs @@ -10,7 +10,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, generate_moderators_url, ActorType, }; @@ -110,8 +110,7 @@ impl ActivityHandler for RemoveMod { if self.target.is_some() { let community = get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; - let remove_mod = - get_or_fetch_and_upsert_person(&self.object, context, request_counter).await?; + let remove_mod = dereference::(&self.object, context, request_counter).await?; let form = CommunityModeratorForm { community_id: community.id, diff --git a/crates/apub/src/activities/community/undo_block_user.rs b/crates/apub/src/activities/community/undo_block_user.rs index 9c12dd3ae..764380d7b 100644 --- a/crates/apub/src/activities/community/undo_block_user.rs +++ b/crates/apub/src/activities/community/undo_block_user.rs @@ -8,7 +8,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, ActorType, }; use activitystreams::{ @@ -93,8 +93,7 @@ impl ActivityHandler for UndoBlockUserFromCommunity { ) -> Result<(), LemmyError> { let community = get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; - let blocked_user = - get_or_fetch_and_upsert_person(&self.object.object, context, request_counter).await?; + let blocked_user = dereference::(&self.object.object, context, request_counter).await?; let community_user_ban_form = CommunityPersonBanForm { community_id: community.id, diff --git a/crates/apub/src/activities/deletion/delete.rs b/crates/apub/src/activities/deletion/delete.rs index cb1fdbd93..1bd8bc0bf 100644 --- a/crates/apub/src/activities/deletion/delete.rs +++ b/crates/apub/src/activities/deletion/delete.rs @@ -12,7 +12,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::person::get_or_fetch_and_upsert_person, + fetcher::new_fetcher::dereference, ActorType, }; use activitystreams::{ @@ -171,7 +171,7 @@ pub(in crate::activities) async fn receive_remove_action( context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_person(actor, context, request_counter).await?; + let actor = dereference::(actor, context, request_counter).await?; use UserOperationCrud::*; match DeletableObjects::read_from_db(object, context).await? { DeletableObjects::Community(community) => { diff --git a/crates/apub/src/activities/deletion/mod.rs b/crates/apub/src/activities/deletion/mod.rs index 350773f42..41f5f09c6 100644 --- a/crates/apub/src/activities/deletion/mod.rs +++ b/crates/apub/src/activities/deletion/mod.rs @@ -4,7 +4,7 @@ use crate::{ verify_mod_action, verify_person_in_community, }, - fetcher::person::get_or_fetch_and_upsert_person, + fetcher::new_fetcher::dereference, ActorType, }; use lemmy_api_common::blocking; @@ -180,7 +180,7 @@ async fn receive_delete_action( match DeletableObjects::read_from_db(object, context).await? { DeletableObjects::Community(community) => { if community.local { - let mod_ = get_or_fetch_and_upsert_person(actor, context, request_counter).await?; + let mod_ = dereference::(actor, context, request_counter).await?; let object = community.actor_id(); send_apub_delete(&mod_, &community.clone(), object, true, context).await?; } diff --git a/crates/apub/src/activities/following/accept.rs b/crates/apub/src/activities/following/accept.rs index c76263cc0..ec05318f9 100644 --- a/crates/apub/src/activities/following/accept.rs +++ b/crates/apub/src/activities/following/accept.rs @@ -7,7 +7,7 @@ use crate::{ }, activity_queue::send_activity_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, ActorType, }; use activitystreams::{ @@ -91,7 +91,7 @@ impl ActivityHandler for AcceptFollowCommunity { request_counter: &mut i32, ) -> Result<(), LemmyError> { let actor = get_or_fetch_and_upsert_community(&self.actor, context, request_counter).await?; - let to = get_or_fetch_and_upsert_person(&self.to, context, request_counter).await?; + let to = dereference::(&self.to, context, request_counter).await?; // This will throw an error if no follow was requested blocking(context.pool(), move |conn| { CommunityFollower::follow_accepted(conn, actor.id, to.id) diff --git a/crates/apub/src/activities/following/follow.rs b/crates/apub/src/activities/following/follow.rs index e6ca747a0..7b2008089 100644 --- a/crates/apub/src/activities/following/follow.rs +++ b/crates/apub/src/activities/following/follow.rs @@ -7,7 +7,7 @@ use crate::{ }, activity_queue::send_activity_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, ActorType, }; use activitystreams::{ @@ -97,7 +97,7 @@ impl ActivityHandler for FollowCommunity { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_person(&self.actor, context, request_counter).await?; + let actor = dereference::(&self.actor, context, request_counter).await?; let community = get_or_fetch_and_upsert_community(&self.object, context, request_counter).await?; let community_follower_form = CommunityFollowerForm { diff --git a/crates/apub/src/activities/following/undo.rs b/crates/apub/src/activities/following/undo.rs index 092036bb4..025bb5d5d 100644 --- a/crates/apub/src/activities/following/undo.rs +++ b/crates/apub/src/activities/following/undo.rs @@ -7,7 +7,7 @@ use crate::{ }, activity_queue::send_activity_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, ActorType, }; use activitystreams::{ @@ -84,7 +84,7 @@ impl ActivityHandler for UndoFollowCommunity { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_person(&self.actor, context, request_counter).await?; + let actor = dereference::(&self.actor, context, request_counter).await?; let community = get_or_fetch_and_upsert_community(&self.to, context, request_counter).await?; let community_follower_form = CommunityFollowerForm { diff --git a/crates/apub/src/activities/mod.rs b/crates/apub/src/activities/mod.rs index a846a0e70..5c135ee54 100644 --- a/crates/apub/src/activities/mod.rs +++ b/crates/apub/src/activities/mod.rs @@ -1,7 +1,7 @@ use crate::{ check_community_or_site_ban, check_is_apub_id_valid, - fetcher::{community::get_or_fetch_and_upsert_community, person::get_or_fetch_and_upsert_person}, + fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, generate_moderators_url, }; use anyhow::anyhow; @@ -43,7 +43,7 @@ async fn verify_person( context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let person = get_or_fetch_and_upsert_person(person_id, context, request_counter).await?; + let person = dereference::(person_id, context, request_counter).await?; if person.banned { return Err(anyhow!("Person {} is banned", person_id).into()); } @@ -76,7 +76,7 @@ pub(crate) async fn verify_person_in_community( request_counter: &mut i32, ) -> Result<(), LemmyError> { let community = get_or_fetch_and_upsert_community(community_id, context, request_counter).await?; - let person = get_or_fetch_and_upsert_person(person_id, context, request_counter).await?; + let person = dereference::(person_id, context, request_counter).await?; check_community_or_site_ban(&person, community.id, context.pool()).await } diff --git a/crates/apub/src/activities/post/create_or_update.rs b/crates/apub/src/activities/post/create_or_update.rs index eff56ce21..d515e2e1e 100644 --- a/crates/apub/src/activities/post/create_or_update.rs +++ b/crates/apub/src/activities/post/create_or_update.rs @@ -10,7 +10,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::person::get_or_fetch_and_upsert_person, + fetcher::new_fetcher::dereference, objects::{post::Page, FromApub, ToApub}, ActorType, }; @@ -121,7 +121,7 @@ impl ActivityHandler for CreateOrUpdatePost { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_person(&self.actor, context, request_counter).await?; + let actor = dereference::(&self.actor, context, request_counter).await?; let post = Post::from_apub(&self.object, context, &actor.actor_id(), request_counter).await?; let notif_type = match self.kind { diff --git a/crates/apub/src/activities/voting/undo_vote.rs b/crates/apub/src/activities/voting/undo_vote.rs index bf0fa71e3..62bf60268 100644 --- a/crates/apub/src/activities/voting/undo_vote.rs +++ b/crates/apub/src/activities/voting/undo_vote.rs @@ -12,7 +12,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{new_fetcher::dereference, person::get_or_fetch_and_upsert_person}, + fetcher::new_fetcher::dereference, ActorType, PostOrComment, }; @@ -100,7 +100,7 @@ impl ActivityHandler for UndoVote { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_person(&self.actor, context, request_counter).await?; + let actor = dereference::(&self.actor, context, request_counter).await?; let object = dereference::(&self.object.object, context, request_counter).await?; match object { diff --git a/crates/apub/src/activities/voting/vote.rs b/crates/apub/src/activities/voting/vote.rs index 0dbc82a82..c1c7fdc39 100644 --- a/crates/apub/src/activities/voting/vote.rs +++ b/crates/apub/src/activities/voting/vote.rs @@ -8,7 +8,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{new_fetcher::dereference, person::get_or_fetch_and_upsert_person}, + fetcher::new_fetcher::dereference, ActorType, PostOrComment, }; @@ -126,7 +126,7 @@ impl ActivityHandler for Vote { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_person(&self.actor, context, request_counter).await?; + let actor = dereference::(&self.actor, context, request_counter).await?; let object = dereference::(&self.object, context, request_counter).await?; match object { PostOrComment::Post(p) => vote_post(&self.kind, actor, p.deref(), context).await, diff --git a/crates/apub/src/fetcher/community.rs b/crates/apub/src/fetcher/community.rs index 83aa7b51e..161d741c0 100644 --- a/crates/apub/src/fetcher/community.rs +++ b/crates/apub/src/fetcher/community.rs @@ -3,7 +3,7 @@ use crate::{ fetcher::{ fetch::fetch_remote_object, is_deleted, - person::get_or_fetch_and_upsert_person, + new_fetcher::dereference, should_refetch_actor, }, objects::{community::Group, FromApub}, @@ -14,7 +14,10 @@ use diesel::result::Error::NotFound; use lemmy_api_common::blocking; use lemmy_apub_lib::ActivityHandler; use lemmy_db_queries::{source::community::Community_, ApubObject, Joinable}; -use lemmy_db_schema::source::community::{Community, CommunityModerator, CommunityModeratorForm}; +use lemmy_db_schema::source::{ + community::{Community, CommunityModerator, CommunityModeratorForm}, + person::Person, +}; use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; @@ -114,7 +117,7 @@ async fn update_community_mods( // Add new mods to database which have been added to moderators collection for mod_uri in new_moderators { - let mod_user = get_or_fetch_and_upsert_person(&mod_uri, context, request_counter).await?; + let mod_user = dereference::(&mod_uri, context, request_counter).await?; if !current_moderators .clone() diff --git a/crates/apub/src/fetcher/mod.rs b/crates/apub/src/fetcher/mod.rs index bf470bac4..1b2ec107c 100644 --- a/crates/apub/src/fetcher/mod.rs +++ b/crates/apub/src/fetcher/mod.rs @@ -1,7 +1,6 @@ pub mod community; mod fetch; pub mod new_fetcher; -pub mod person; pub mod post_or_comment; pub mod search; @@ -9,13 +8,13 @@ use crate::{ fetcher::{ community::get_or_fetch_and_upsert_community, fetch::FetchError, - person::get_or_fetch_and_upsert_person, + new_fetcher::dereference, }, ActorType, }; use chrono::NaiveDateTime; use http::StatusCode; -use lemmy_db_schema::naive_now; +use lemmy_db_schema::{naive_now, source::person::Person}; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use serde::Deserialize; @@ -51,7 +50,7 @@ pub(crate) async fn get_or_fetch_and_upsert_actor( let community = get_or_fetch_and_upsert_community(apub_id, context, recursion_counter).await; let actor: Box = match community { Ok(c) => Box::new(c), - Err(_) => Box::new(get_or_fetch_and_upsert_person(apub_id, context, recursion_counter).await?), + Err(_) => Box::new(dereference::(apub_id, context, recursion_counter).await?), }; Ok(actor) } diff --git a/crates/apub/src/fetcher/new_fetcher.rs b/crates/apub/src/fetcher/new_fetcher.rs index 3b6822098..c071a60cc 100644 --- a/crates/apub/src/fetcher/new_fetcher.rs +++ b/crates/apub/src/fetcher/new_fetcher.rs @@ -1,10 +1,11 @@ -use crate::{objects::FromApub, APUB_JSON_CONTENT_TYPE}; +use crate::{fetcher::should_refetch_actor, objects::FromApub, APUB_JSON_CONTENT_TYPE}; use anyhow::anyhow; use diesel::NotFound; use lemmy_api_common::blocking; use lemmy_db_queries::{ApubObject, DbPool}; use lemmy_utils::{request::retry, settings::structs::Settings, LemmyError}; use lemmy_websocket::LemmyContext; +use log::debug; use reqwest::StatusCode; use std::time::Duration; use url::Url; @@ -23,11 +24,27 @@ where Kind: FromApub + ApubObject + Send + 'static, for<'de> ::ApubType: serde::Deserialize<'de>, { - let local_object = dereference_locally(id.clone(), context.pool()).await?; - if let Some(object) = local_object { - // TODO: for actors, also refetch after 24 hours + let db_object = dereference_locally::(id.clone(), context.pool()).await?; + // if its a local object, only fetch it from the database and not over http + if id.domain() == Some(&Settings::get().get_hostname_without_port()?) { + dbg!("is local object", db_object.is_some()); + return match db_object { + None => Err(NotFound {}.into()), + Some(o) => Ok(o), + }; + } + + if let Some(object) = db_object { + if let Some(last_refreshed_at) = object.last_refreshed_at() { + // TODO: rename to should_refetch_object() + if should_refetch_actor(last_refreshed_at) { + debug!("Refetching remote object {}", id); + return dereference_remotely(id, context, request_counter).await; + } + } Ok(object) } else { + debug!("Fetching remote object {}", id); dereference_remotely(id, context, request_counter).await } } diff --git a/crates/apub/src/fetcher/person.rs b/crates/apub/src/fetcher/person.rs deleted file mode 100644 index ed3ca057f..000000000 --- a/crates/apub/src/fetcher/person.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::{ - fetcher::{fetch::fetch_remote_object, is_deleted, should_refetch_actor}, - objects::{person::Person as ApubPerson, FromApub}, -}; -use anyhow::anyhow; -use diesel::result::Error::NotFound; -use lemmy_api_common::blocking; -use lemmy_db_queries::{source::person::Person_, ApubObject}; -use lemmy_db_schema::source::person::Person; -use lemmy_utils::LemmyError; -use lemmy_websocket::LemmyContext; -use log::debug; -use url::Url; - -/// Get a person from its apub ID. -/// -/// If it exists locally and `!should_refetch_actor()`, it is returned directly from the database. -/// Otherwise it is fetched from the remote instance, stored and returned. -pub(crate) async fn get_or_fetch_and_upsert_person( - apub_id: &Url, - context: &LemmyContext, - recursion_counter: &mut i32, -) -> Result { - let apub_id_owned = apub_id.to_owned(); - let person = blocking(context.pool(), move |conn| { - Person::read_from_apub_id(conn, &apub_id_owned.into()) - }) - .await?; - - match person { - // If its older than a day, re-fetch it - Ok(u) if !u.local && should_refetch_actor(u.last_refreshed_at) => { - debug!("Fetching and updating from remote person: {}", apub_id); - let person = - fetch_remote_object::(context.client(), apub_id, recursion_counter).await; - - if is_deleted(&person) { - // TODO: use Person::update_deleted() once implemented - blocking(context.pool(), move |conn| { - Person::delete_account(conn, u.id) - }) - .await??; - return Err(anyhow!("Person was deleted by remote instance").into()); - } else if person.is_err() { - return Ok(u); - } - - let person = Person::from_apub(&person?, context, apub_id, recursion_counter).await?; - - let person_id = person.id; - blocking(context.pool(), move |conn| { - Person::mark_as_updated(conn, person_id) - }) - .await??; - - Ok(person) - } - Ok(u) => Ok(u), - Err(NotFound {}) => { - debug!("Fetching and creating remote person: {}", apub_id); - let person = - fetch_remote_object::(context.client(), apub_id, recursion_counter).await?; - - let person = Person::from_apub(&person, context, apub_id, recursion_counter).await?; - - Ok(person) - } - Err(e) => Err(e.into()), - } -} diff --git a/crates/apub/src/fetcher/post_or_comment.rs b/crates/apub/src/fetcher/post_or_comment.rs index 176aeb1b1..bc95f8a89 100644 --- a/crates/apub/src/fetcher/post_or_comment.rs +++ b/crates/apub/src/fetcher/post_or_comment.rs @@ -1,4 +1,5 @@ use crate::objects::{comment::Note, post::Page, FromApub}; +use activitystreams::chrono::NaiveDateTime; use diesel::{result::Error, PgConnection}; use lemmy_db_queries::ApubObject; use lemmy_db_schema::{ @@ -33,6 +34,10 @@ pub enum PageOrNote { impl ApubObject for PostOrComment { type Form = PostOrCommentForm; + fn last_refreshed_at(&self) -> Option { + None + } + // TODO: this can probably be implemented using a single sql query fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result where diff --git a/crates/apub/src/fetcher/search.rs b/crates/apub/src/fetcher/search.rs index 70e7c40c1..ffab035d0 100644 --- a/crates/apub/src/fetcher/search.rs +++ b/crates/apub/src/fetcher/search.rs @@ -3,7 +3,7 @@ use crate::{ community::get_or_fetch_and_upsert_community, fetch::fetch_remote_object, is_deleted, - person::get_or_fetch_and_upsert_person, + new_fetcher::dereference, }, find_object_by_id, objects::{comment::Note, community::Group, person::Person as ApubPerson, post::Page, FromApub}, @@ -131,7 +131,7 @@ async fn build_response( SearchAcceptedObjects::Person(p) => { let person_uri = p.id(&query_url)?; - let person = get_or_fetch_and_upsert_person(person_uri, context, recursion_counter).await?; + let person = dereference::(person_uri, context, recursion_counter).await?; ROR { person: blocking(context.pool(), move |conn| { PersonViewSafe::read(conn, person.id) diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index 25100e8fe..93abdf896 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -3,7 +3,7 @@ use crate::{ extensions::context::lemmy_context, fetcher::new_fetcher::dereference, migrations::CommentInReplyToMigration, - objects::{create_tombstone, get_or_fetch_and_upsert_person, FromApub, Source, ToApub}, + objects::{create_tombstone, FromApub, Source, ToApub}, ActorType, PostOrComment, }; @@ -216,8 +216,7 @@ impl FromApub for Comment { request_counter: &mut i32, ) -> Result { let ap_id = Some(note.id(expected_domain)?.clone().into()); - let creator = - get_or_fetch_and_upsert_person(¬e.attributed_to, context, request_counter).await?; + let creator = dereference::(¬e.attributed_to, context, request_counter).await?; let (post, parent_comment_id) = note.get_parents(context, request_counter).await?; if post.locked { return Err(anyhow!("Post is locked").into()); diff --git a/crates/apub/src/objects/mod.rs b/crates/apub/src/objects/mod.rs index 33a93915a..860f18401 100644 --- a/crates/apub/src/objects/mod.rs +++ b/crates/apub/src/objects/mod.rs @@ -1,4 +1,3 @@ -use crate::fetcher::person::get_or_fetch_and_upsert_person; use activitystreams::{ base::BaseExt, object::{kind::ImageType, Tombstone, TombstoneExt}, diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 4773f8c56..f86dee293 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -1,7 +1,7 @@ use crate::{ activities::{extract_community, verify_person_in_community}, extensions::context::lemmy_context, - fetcher::person::get_or_fetch_and_upsert_person, + fetcher::new_fetcher::dereference, objects::{create_tombstone, FromApub, ImageObject, Source, ToApub}, ActorType, }; @@ -183,8 +183,7 @@ impl FromApub for Post { page.id(expected_domain)? }; let ap_id = Some(ap_id.clone().into()); - let creator = - get_or_fetch_and_upsert_person(&page.attributed_to, context, request_counter).await?; + let creator = dereference::(&page.attributed_to, context, request_counter).await?; let community = extract_community(&page.to, context, request_counter).await?; let thumbnail_url: Option = page.image.clone().map(|i| i.url); diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 02cf12eba..92fc0370d 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -1,6 +1,6 @@ use crate::{ extensions::context::lemmy_context, - fetcher::person::get_or_fetch_and_upsert_person, + fetcher::new_fetcher::dereference, objects::{create_tombstone, FromApub, Source, ToApub}, }; use activitystreams::{ @@ -61,8 +61,7 @@ impl Note { request_counter: &mut i32, ) -> Result<(), LemmyError> { verify_domains_match(&self.attributed_to, &self.id)?; - let person = - get_or_fetch_and_upsert_person(&self.attributed_to, context, request_counter).await?; + let person = dereference::(&self.attributed_to, context, request_counter).await?; if person.banned { return Err(anyhow!("Person is banned from site").into()); } @@ -121,9 +120,8 @@ impl FromApub for PrivateMessage { request_counter: &mut i32, ) -> Result { let ap_id = Some(note.id(expected_domain)?.clone().into()); - let creator = - get_or_fetch_and_upsert_person(¬e.attributed_to, context, request_counter).await?; - let recipient = get_or_fetch_and_upsert_person(¬e.to, context, request_counter).await?; + let creator = dereference::(¬e.attributed_to, context, request_counter).await?; + let recipient = dereference::(¬e.to, context, request_counter).await?; let form = PrivateMessageForm { creator_id: creator.id, diff --git a/crates/db_queries/src/lib.rs b/crates/db_queries/src/lib.rs index e62124b18..6334cbca3 100644 --- a/crates/db_queries/src/lib.rs +++ b/crates/db_queries/src/lib.rs @@ -12,6 +12,7 @@ extern crate diesel_migrations; #[cfg(test)] extern crate serial_test; +use chrono::NaiveDateTime; use diesel::{result::Error, *}; use lemmy_db_schema::{CommunityId, DbUrl, PersonId}; use lemmy_utils::ApiError; @@ -147,6 +148,9 @@ pub trait DeleteableOrRemoveable { pub trait ApubObject { type Form; + /// If this object should be refetched after a certain interval, it should return the last refresh + /// time here. This is mainly used to update remote actors. + fn last_refreshed_at(&self) -> Option; fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result where Self: Sized; diff --git a/crates/db_queries/src/source/activity.rs b/crates/db_queries/src/source/activity.rs index 1f158370e..3008c38f8 100644 --- a/crates/db_queries/src/source/activity.rs +++ b/crates/db_queries/src/source/activity.rs @@ -1,7 +1,6 @@ use crate::Crud; use diesel::{dsl::*, result::Error, sql_types::Text, *}; use lemmy_db_schema::{source::activity::*, DbUrl}; -use log::debug; use serde::Serialize; use serde_json::Value; use std::{ @@ -72,7 +71,6 @@ impl Activity_ for Activity { where T: Serialize + Debug, { - debug!("{}", serde_json::to_string_pretty(&data)?); let activity_form = ActivityForm { ap_id, data: serde_json::to_value(&data)?, diff --git a/crates/db_queries/src/source/comment.rs b/crates/db_queries/src/source/comment.rs index 1e9722714..0b6adf29d 100644 --- a/crates/db_queries/src/source/comment.rs +++ b/crates/db_queries/src/source/comment.rs @@ -1,4 +1,5 @@ use crate::{ApubObject, Crud, DeleteableOrRemoveable, Likeable, Saveable}; +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use lemmy_db_schema::{ naive_now, @@ -169,6 +170,11 @@ impl Crud for Comment { impl ApubObject for Comment { type Form = CommentForm; + + fn last_refreshed_at(&self) -> Option { + None + } + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::comment::dsl::*; comment.filter(ap_id.eq(object_id)).first::(conn) diff --git a/crates/db_queries/src/source/community.rs b/crates/db_queries/src/source/community.rs index eaaa2ba29..fdd108941 100644 --- a/crates/db_queries/src/source/community.rs +++ b/crates/db_queries/src/source/community.rs @@ -1,4 +1,5 @@ use crate::{ApubObject, Bannable, Crud, DeleteableOrRemoveable, Followable, Joinable}; +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use lemmy_db_schema::{ naive_now, @@ -94,6 +95,11 @@ impl Crud for Community { impl ApubObject for Community { type Form = CommunityForm; + + fn last_refreshed_at(&self) -> Option { + Some(self.last_refreshed_at) + } + fn read_from_apub_id(conn: &PgConnection, for_actor_id: &DbUrl) -> Result { use lemmy_db_schema::schema::community::dsl::*; community diff --git a/crates/db_queries/src/source/person.rs b/crates/db_queries/src/source/person.rs index 6172b4e1e..2449b349d 100644 --- a/crates/db_queries/src/source/person.rs +++ b/crates/db_queries/src/source/person.rs @@ -1,4 +1,5 @@ use crate::{ApubObject, Crud}; +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use lemmy_db_schema::{ naive_now, @@ -182,6 +183,11 @@ impl Crud for Person { impl ApubObject for Person { type Form = PersonForm; + + fn last_refreshed_at(&self) -> Option { + Some(self.last_refreshed_at) + } + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::person::dsl::*; person diff --git a/crates/db_queries/src/source/post.rs b/crates/db_queries/src/source/post.rs index 02ae4d6e2..814c4650e 100644 --- a/crates/db_queries/src/source/post.rs +++ b/crates/db_queries/src/source/post.rs @@ -1,4 +1,5 @@ use crate::{ApubObject, Crud, DeleteableOrRemoveable, Likeable, Readable, Saveable}; +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use lemmy_db_schema::{ naive_now, @@ -183,6 +184,11 @@ impl Post_ for Post { impl ApubObject for Post { type Form = PostForm; + + fn last_refreshed_at(&self) -> Option { + None + } + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result { use lemmy_db_schema::schema::post::dsl::*; post.filter(ap_id.eq(object_id)).first::(conn) diff --git a/crates/db_queries/src/source/private_message.rs b/crates/db_queries/src/source/private_message.rs index c1138b979..5525224fc 100644 --- a/crates/db_queries/src/source/private_message.rs +++ b/crates/db_queries/src/source/private_message.rs @@ -1,4 +1,5 @@ use crate::{ApubObject, Crud, DeleteableOrRemoveable}; +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use lemmy_db_schema::{naive_now, source::private_message::*, DbUrl, PersonId, PrivateMessageId}; @@ -31,6 +32,11 @@ impl Crud for PrivateMessage { impl ApubObject for PrivateMessage { type Form = PrivateMessageForm; + + fn last_refreshed_at(&self) -> Option { + None + } + fn read_from_apub_id(conn: &PgConnection, object_id: &DbUrl) -> Result where Self: Sized,