From 1013d1a46d17ed3e1c07c12ae03fd454fca8e8e4 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 22 Sep 2021 16:15:20 +0200 Subject: [PATCH] rewrite community to use new fetcher --- .../apub/src/activities/community/add_mod.rs | 5 +- .../src/activities/community/block_user.rs | 5 +- .../src/activities/community/remove_mod.rs | 5 +- .../activities/community/undo_block_user.rs | 5 +- .../apub/src/activities/following/accept.rs | 4 +- .../apub/src/activities/following/follow.rs | 5 +- crates/apub/src/activities/following/undo.rs | 4 +- crates/apub/src/activities/mod.rs | 8 +- crates/apub/src/fetcher/community.rs | 82 ++----------------- crates/apub/src/fetcher/mod.rs | 13 ++- crates/apub/src/fetcher/new_fetcher.rs | 1 - crates/apub/src/fetcher/search.rs | 10 +-- crates/apub/src/objects/community.rs | 7 +- 13 files changed, 37 insertions(+), 117 deletions(-) diff --git a/crates/apub/src/activities/community/add_mod.rs b/crates/apub/src/activities/community/add_mod.rs index e1b6864cd..135dae4df 100644 --- a/crates/apub/src/activities/community/add_mod.rs +++ b/crates/apub/src/activities/community/add_mod.rs @@ -9,7 +9,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, generate_moderators_url, ActorType, }; @@ -93,8 +93,7 @@ impl ActivityHandler for AddMod { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let community = - get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; + let community = dereference::(&self.cc[0], context, request_counter).await?; let new_mod = dereference::(&self.object, context, request_counter).await?; // If we had to refetch the community while parsing the activity, then the new mod has already diff --git a/crates/apub/src/activities/community/block_user.rs b/crates/apub/src/activities/community/block_user.rs index b84c1304b..f0e4681ab 100644 --- a/crates/apub/src/activities/community/block_user.rs +++ b/crates/apub/src/activities/community/block_user.rs @@ -8,7 +8,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, ActorType, }; use activitystreams::{ @@ -102,8 +102,7 @@ impl ActivityHandler for BlockUserFromCommunity { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let community = - get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; + let community = dereference::(&self.cc[0], context, request_counter).await?; let blocked_user = dereference::(&self.object, context, request_counter).await?; let community_user_ban_form = CommunityPersonBanForm { diff --git a/crates/apub/src/activities/community/remove_mod.rs b/crates/apub/src/activities/community/remove_mod.rs index 916da64b5..6746a52d9 100644 --- a/crates/apub/src/activities/community/remove_mod.rs +++ b/crates/apub/src/activities/community/remove_mod.rs @@ -10,7 +10,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, generate_moderators_url, ActorType, }; @@ -108,8 +108,7 @@ impl ActivityHandler for RemoveMod { request_counter: &mut i32, ) -> Result<(), LemmyError> { if self.target.is_some() { - let community = - get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; + let community = dereference::(&self.cc[0], context, request_counter).await?; let remove_mod = dereference::(&self.object, context, request_counter).await?; let form = CommunityModeratorForm { diff --git a/crates/apub/src/activities/community/undo_block_user.rs b/crates/apub/src/activities/community/undo_block_user.rs index 764380d7b..2220a56f2 100644 --- a/crates/apub/src/activities/community/undo_block_user.rs +++ b/crates/apub/src/activities/community/undo_block_user.rs @@ -8,7 +8,7 @@ use crate::{ }, activity_queue::send_to_community_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, ActorType, }; use activitystreams::{ @@ -91,8 +91,7 @@ impl ActivityHandler for UndoBlockUserFromCommunity { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let community = - get_or_fetch_and_upsert_community(&self.cc[0], context, request_counter).await?; + let community = dereference::(&self.cc[0], context, request_counter).await?; let blocked_user = dereference::(&self.object.object, context, request_counter).await?; let community_user_ban_form = CommunityPersonBanForm { diff --git a/crates/apub/src/activities/following/accept.rs b/crates/apub/src/activities/following/accept.rs index ec05318f9..6c8412dcd 100644 --- a/crates/apub/src/activities/following/accept.rs +++ b/crates/apub/src/activities/following/accept.rs @@ -7,7 +7,7 @@ use crate::{ }, activity_queue::send_activity_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, ActorType, }; use activitystreams::{ @@ -90,7 +90,7 @@ impl ActivityHandler for AcceptFollowCommunity { context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let actor = get_or_fetch_and_upsert_community(&self.actor, context, request_counter).await?; + let actor = dereference::(&self.actor, context, request_counter).await?; let to = dereference::(&self.to, context, request_counter).await?; // This will throw an error if no follow was requested blocking(context.pool(), move |conn| { diff --git a/crates/apub/src/activities/following/follow.rs b/crates/apub/src/activities/following/follow.rs index 7b2008089..17943a062 100644 --- a/crates/apub/src/activities/following/follow.rs +++ b/crates/apub/src/activities/following/follow.rs @@ -7,7 +7,7 @@ use crate::{ }, activity_queue::send_activity_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, ActorType, }; use activitystreams::{ @@ -98,8 +98,7 @@ impl ActivityHandler for FollowCommunity { request_counter: &mut i32, ) -> Result<(), LemmyError> { let actor = dereference::(&self.actor, context, request_counter).await?; - let community = - get_or_fetch_and_upsert_community(&self.object, context, request_counter).await?; + let community = dereference::(&self.object, context, request_counter).await?; let community_follower_form = CommunityFollowerForm { community_id: community.id, person_id: actor.id, diff --git a/crates/apub/src/activities/following/undo.rs b/crates/apub/src/activities/following/undo.rs index 025bb5d5d..d09eae04a 100644 --- a/crates/apub/src/activities/following/undo.rs +++ b/crates/apub/src/activities/following/undo.rs @@ -7,7 +7,7 @@ use crate::{ }, activity_queue::send_activity_new, extensions::context::lemmy_context, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, ActorType, }; use activitystreams::{ @@ -85,7 +85,7 @@ impl ActivityHandler for UndoFollowCommunity { request_counter: &mut i32, ) -> Result<(), LemmyError> { let actor = dereference::(&self.actor, context, request_counter).await?; - let community = get_or_fetch_and_upsert_community(&self.to, context, request_counter).await?; + let community = dereference::(&self.to, context, request_counter).await?; let community_follower_form = CommunityFollowerForm { community_id: community.id, diff --git a/crates/apub/src/activities/mod.rs b/crates/apub/src/activities/mod.rs index 5c135ee54..52ae90ccb 100644 --- a/crates/apub/src/activities/mod.rs +++ b/crates/apub/src/activities/mod.rs @@ -1,7 +1,7 @@ use crate::{ check_community_or_site_ban, check_is_apub_id_valid, - fetcher::{community::get_or_fetch_and_upsert_community, new_fetcher::dereference}, + fetcher::new_fetcher::dereference, generate_moderators_url, }; use anyhow::anyhow; @@ -58,7 +58,7 @@ pub(crate) async fn extract_community( let mut cc_iter = cc.iter(); loop { if let Some(cid) = cc_iter.next() { - if let Ok(c) = get_or_fetch_and_upsert_community(cid, context, request_counter).await { + if let Ok(c) = dereference(cid, context, request_counter).await { break Ok(c); } } else { @@ -75,7 +75,7 @@ pub(crate) async fn verify_person_in_community( context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - let community = get_or_fetch_and_upsert_community(community_id, context, request_counter).await?; + let community = dereference::(community_id, context, request_counter).await?; let person = dereference::(person_id, context, request_counter).await?; check_community_or_site_ban(&person, community.id, context.pool()).await } @@ -86,7 +86,7 @@ async fn verify_community( context: &LemmyContext, request_counter: &mut i32, ) -> Result<(), LemmyError> { - get_or_fetch_and_upsert_community(community_id, context, request_counter).await?; + dereference::(community_id, context, request_counter).await?; Ok(()) } diff --git a/crates/apub/src/fetcher/community.rs b/crates/apub/src/fetcher/community.rs index 161d741c0..949e457cd 100644 --- a/crates/apub/src/fetcher/community.rs +++ b/crates/apub/src/fetcher/community.rs @@ -1,19 +1,13 @@ use crate::{ activities::community::announce::AnnounceActivity, - fetcher::{ - fetch::fetch_remote_object, - is_deleted, - new_fetcher::dereference, - should_refetch_actor, - }, - objects::{community::Group, FromApub}, + fetcher::{fetch::fetch_remote_object, new_fetcher::dereference}, + objects::community::Group, }; use activitystreams::collection::{CollectionExt, OrderedCollection}; use anyhow::Context; -use diesel::result::Error::NotFound; use lemmy_api_common::blocking; use lemmy_apub_lib::ActivityHandler; -use lemmy_db_queries::{source::community::Community_, ApubObject, Joinable}; +use lemmy_db_queries::Joinable; use lemmy_db_schema::source::{ community::{Community, CommunityModerator, CommunityModeratorForm}, person::Person, @@ -21,75 +15,9 @@ use lemmy_db_schema::source::{ use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView; use lemmy_utils::{location_info, LemmyError}; use lemmy_websocket::LemmyContext; -use log::debug; use url::Url; -/// Get a community from its apub ID. -/// -/// If it exists locally and `!should_refetch_actor()`, it is returned directly from the database. -/// Otherwise it is fetched from the remote instance, stored and returned. -pub(crate) async fn get_or_fetch_and_upsert_community( - apub_id: &Url, - context: &LemmyContext, - recursion_counter: &mut i32, -) -> Result { - let apub_id_owned = apub_id.to_owned(); - let community = blocking(context.pool(), move |conn| { - Community::read_from_apub_id(conn, &apub_id_owned.into()) - }) - .await?; - - match community { - Ok(c) if !c.local && should_refetch_actor(c.last_refreshed_at) => { - debug!("Fetching and updating from remote community: {}", apub_id); - fetch_remote_community(apub_id, context, Some(c), recursion_counter).await - } - Ok(c) => Ok(c), - Err(NotFound {}) => { - debug!("Fetching and creating remote community: {}", apub_id); - fetch_remote_community(apub_id, context, None, recursion_counter).await - } - Err(e) => Err(e.into()), - } -} - -/// Request a community by apub ID from a remote instance, including moderators. If `old_community`, -/// is set, this is an update for a community which is already known locally. If not, we don't know -/// the community yet and also pull the outbox, to get some initial posts. -async fn fetch_remote_community( - apub_id: &Url, - context: &LemmyContext, - old_community: Option, - request_counter: &mut i32, -) -> Result { - let group = fetch_remote_object::(context.client(), apub_id, request_counter).await; - - if let Some(c) = old_community.to_owned() { - if is_deleted(&group) { - blocking(context.pool(), move |conn| { - Community::update_deleted(conn, c.id, true) - }) - .await??; - } else if group.is_err() { - // If fetching failed, return the existing data. - return Ok(c); - } - } - - let group = group?; - let community = Community::from_apub(&group, context, apub_id, request_counter).await?; - - update_community_mods(&group, &community, context, request_counter).await?; - - // only fetch outbox for new communities, otherwise this can create an infinite loop - if old_community.is_none() { - fetch_community_outbox(context, &group.outbox, request_counter).await? - } - - Ok(community) -} - -async fn update_community_mods( +pub(crate) async fn update_community_mods( group: &Group, community: &Community, context: &LemmyContext, @@ -139,7 +67,7 @@ async fn update_community_mods( Ok(()) } -async fn fetch_community_outbox( +pub(crate) async fn fetch_community_outbox( context: &LemmyContext, outbox: &Url, recursion_counter: &mut i32, diff --git a/crates/apub/src/fetcher/mod.rs b/crates/apub/src/fetcher/mod.rs index 1b2ec107c..e8b1caa11 100644 --- a/crates/apub/src/fetcher/mod.rs +++ b/crates/apub/src/fetcher/mod.rs @@ -5,16 +5,15 @@ pub mod post_or_comment; pub mod search; use crate::{ - fetcher::{ - community::get_or_fetch_and_upsert_community, - fetch::FetchError, - new_fetcher::dereference, - }, + fetcher::{fetch::FetchError, new_fetcher::dereference}, ActorType, }; use chrono::NaiveDateTime; use http::StatusCode; -use lemmy_db_schema::{naive_now, source::person::Person}; +use lemmy_db_schema::{ + naive_now, + source::{community::Community, person::Person}, +}; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use serde::Deserialize; @@ -47,7 +46,7 @@ pub(crate) async fn get_or_fetch_and_upsert_actor( context: &LemmyContext, recursion_counter: &mut i32, ) -> Result, LemmyError> { - let community = get_or_fetch_and_upsert_community(apub_id, context, recursion_counter).await; + let community = dereference::(apub_id, context, recursion_counter).await; let actor: Box = match community { Ok(c) => Box::new(c), Err(_) => Box::new(dereference::(apub_id, context, recursion_counter).await?), diff --git a/crates/apub/src/fetcher/new_fetcher.rs b/crates/apub/src/fetcher/new_fetcher.rs index c071a60cc..002a43009 100644 --- a/crates/apub/src/fetcher/new_fetcher.rs +++ b/crates/apub/src/fetcher/new_fetcher.rs @@ -27,7 +27,6 @@ where let db_object = dereference_locally::(id.clone(), context.pool()).await?; // if its a local object, only fetch it from the database and not over http if id.domain() == Some(&Settings::get().get_hostname_without_port()?) { - dbg!("is local object", db_object.is_some()); return match db_object { None => Err(NotFound {}.into()), Some(o) => Ok(o), diff --git a/crates/apub/src/fetcher/search.rs b/crates/apub/src/fetcher/search.rs index ffab035d0..f0c86bb06 100644 --- a/crates/apub/src/fetcher/search.rs +++ b/crates/apub/src/fetcher/search.rs @@ -1,10 +1,5 @@ use crate::{ - fetcher::{ - community::get_or_fetch_and_upsert_community, - fetch::fetch_remote_object, - is_deleted, - new_fetcher::dereference, - }, + fetcher::{fetch::fetch_remote_object, is_deleted, new_fetcher::dereference}, find_object_by_id, objects::{comment::Note, community::Group, person::Person as ApubPerson, post::Page, FromApub}, Object, @@ -143,8 +138,7 @@ async fn build_response( } SearchAcceptedObjects::Group(g) => { let community_uri = g.id(&query_url)?; - let community = - get_or_fetch_and_upsert_community(community_uri, context, recursion_counter).await?; + let community = dereference::(community_uri, context, recursion_counter).await?; ROR { community: blocking(context.pool(), move |conn| { CommunityView::read(conn, community.id, None) diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 5269d7663..fbd702073 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -1,6 +1,6 @@ use crate::{ extensions::{context::lemmy_context, signatures::PublicKey}, - fetcher::community::fetch_community_mods, + fetcher::community::{fetch_community_mods, fetch_community_outbox, update_community_mods}, generate_moderators_url, objects::{create_tombstone, FromApub, ImageObject, Source, ToApub}, ActorType, @@ -179,6 +179,11 @@ impl FromApub for Community { let form = Group::from_apub_to_form(group, expected_domain).await?; let community = blocking(context.pool(), move |conn| Community::upsert(conn, &form)).await??; + update_community_mods(group, &community, context, request_counter).await?; + + // TODO: doing this unconditionally might cause infinite loop for some reason + fetch_community_outbox(context, &group.outbox, request_counter).await?; + Ok(community) } }