From b439cb36aaa31c9ac760404ee78513ff254495d6 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Wed, 27 Oct 2021 12:56:07 +0200 Subject: [PATCH] Rewrite community moderators collection --- .../src/collections/community_moderators.rs | 141 ++++++++++++++++++ .../apub/src/collections/community_outbox.rs | 18 +-- crates/apub/src/collections/mod.rs | 6 + crates/apub/src/fetcher/community.rs | 94 ------------ crates/apub/src/fetcher/fetch.rs | 49 ------ crates/apub/src/fetcher/mod.rs | 2 - crates/apub/src/fetcher/object_id.rs | 2 + crates/apub/src/http/community.rs | 68 +++------ crates/apub/src/http/person.rs | 16 -- crates/apub/src/http/routes.rs | 8 +- crates/apub/src/objects/community.rs | 33 ++-- crates/apub/src/objects/person.rs | 2 +- 12 files changed, 197 insertions(+), 242 deletions(-) create mode 100644 crates/apub/src/collections/community_moderators.rs delete mode 100644 crates/apub/src/fetcher/community.rs delete mode 100644 crates/apub/src/fetcher/fetch.rs diff --git a/crates/apub/src/collections/community_moderators.rs b/crates/apub/src/collections/community_moderators.rs new file mode 100644 index 000000000..dc1d57985 --- /dev/null +++ b/crates/apub/src/collections/community_moderators.rs @@ -0,0 +1,141 @@ +use crate::{ + collections::CommunityContext, + context::lemmy_context, + fetcher::object_id::ObjectId, + generate_moderators_url, + objects::person::ApubPerson, +}; +use activitystreams::{ + base::AnyBase, + chrono::NaiveDateTime, + collection::kind::OrderedCollectionType, + primitives::OneOrMany, + url::Url, +}; +use lemmy_api_common::blocking; +use lemmy_apub_lib::{traits::ApubObject, verify::verify_domains_match}; +use lemmy_db_schema::{ + source::community::{CommunityModerator, CommunityModeratorForm}, + traits::Joinable, +}; +use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView; +use lemmy_utils::LemmyError; +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; + +#[skip_serializing_none] +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GroupModerators { + #[serde(rename = "@context")] + context: OneOrMany, + r#type: OrderedCollectionType, + id: Url, + ordered_items: Vec>, +} + +#[derive(Clone, Debug)] +pub(crate) struct ApubCommunityModerators(pub(crate) Vec); + +#[async_trait::async_trait(?Send)] +impl ApubObject for ApubCommunityModerators { + type DataType = CommunityContext; + type TombstoneType = (); + type ApubType = GroupModerators; + + fn last_refreshed_at(&self) -> Option { + None + } + + async fn read_from_apub_id( + _object_id: Url, + data: &Self::DataType, + ) -> Result, LemmyError> { + // Only read from database if its a local community, otherwise fetch over http + if data.0.local { + let cid = data.0.id; + let moderators = blocking(data.1.pool(), move |conn| { + CommunityModeratorView::for_community(conn, cid) + }) + .await??; + Ok(Some(ApubCommunityModerators { 0: moderators })) + } else { + Ok(None) + } + } + + async fn delete(self, _data: &Self::DataType) -> Result<(), LemmyError> { + unimplemented!() + } + + async fn to_apub(&self, data: &Self::DataType) -> Result { + let ordered_items = self + .0 + .iter() + .map(|m| ObjectId::::new(m.moderator.actor_id.clone().into_inner())) + .collect(); + Ok(GroupModerators { + context: lemmy_context(), + r#type: OrderedCollectionType::OrderedCollection, + id: generate_moderators_url(&data.0.actor_id)?.into(), + ordered_items, + }) + } + + fn to_tombstone(&self) -> Result { + unimplemented!() + } + + async fn from_apub( + apub: &Self::ApubType, + data: &Self::DataType, + expected_domain: &Url, + request_counter: &mut i32, + ) -> Result { + verify_domains_match(expected_domain, &apub.id)?; + let community_id = data.0.id; + let current_moderators = blocking(data.1.pool(), move |conn| { + CommunityModeratorView::for_community(conn, community_id) + }) + .await??; + // Remove old mods from database which arent in the moderators collection anymore + for mod_user in ¤t_moderators { + let mod_id = ObjectId::new(mod_user.moderator.actor_id.clone().into_inner()); + if !apub.ordered_items.contains(&mod_id) { + let community_moderator_form = CommunityModeratorForm { + community_id: mod_user.community.id, + person_id: mod_user.moderator.id, + }; + blocking(data.1.pool(), move |conn| { + CommunityModerator::leave(conn, &community_moderator_form) + }) + .await??; + } + } + + // Add new mods to database which have been added to moderators collection + for mod_id in &apub.ordered_items { + let mod_id = ObjectId::new(mod_id.clone()); + let mod_user: ApubPerson = mod_id.dereference(&data.1, request_counter).await?; + + if !current_moderators + .clone() + .iter() + .map(|c| c.moderator.actor_id.clone()) + .any(|x| x == mod_user.actor_id) + { + let community_moderator_form = CommunityModeratorForm { + community_id: data.0.id, + person_id: mod_user.id, + }; + blocking(data.1.pool(), move |conn| { + CommunityModerator::join(conn, &community_moderator_form) + }) + .await??; + } + } + + // This return value is unused, so just set an empty vec + Ok(ApubCommunityModerators { 0: vec![] }) + } +} diff --git a/crates/apub/src/collections/community_outbox.rs b/crates/apub/src/collections/community_outbox.rs index 6f5eb1b31..24465c95c 100644 --- a/crates/apub/src/collections/community_outbox.rs +++ b/crates/apub/src/collections/community_outbox.rs @@ -1,14 +1,14 @@ use crate::{ activities::{post::create_or_update::CreateOrUpdatePost, CreateOrUpdateType}, + collections::CommunityContext, context::lemmy_context, generate_outbox_url, - objects::{community::ApubCommunity, person::ApubPerson, post::ApubPost}, + objects::{person::ApubPerson, post::ApubPost}, }; use activitystreams::{ base::AnyBase, chrono::NaiveDateTime, collection::kind::OrderedCollectionType, - object::Tombstone, primitives::OneOrMany, url::Url, }; @@ -23,14 +23,13 @@ use lemmy_db_schema::{ traits::Crud, }; use lemmy_utils::LemmyError; -use lemmy_websocket::LemmyContext; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; #[skip_serializing_none] #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -pub struct CommunityOutbox { +pub struct GroupOutbox { #[serde(rename = "@context")] context: OneOrMany, r#type: OrderedCollectionType, @@ -41,14 +40,11 @@ pub struct CommunityOutbox { #[derive(Clone, Debug)] pub(crate) struct ApubCommunityOutbox(Vec); -/// Put community in the data, so we dont have to read it again from the database. -pub(crate) struct OutboxData(pub ApubCommunity, pub LemmyContext); - #[async_trait::async_trait(?Send)] impl ApubObject for ApubCommunityOutbox { - type DataType = OutboxData; - type TombstoneType = Tombstone; - type ApubType = CommunityOutbox; + type DataType = CommunityContext; + type TombstoneType = (); + type ApubType = GroupOutbox; fn last_refreshed_at(&self) -> Option { None @@ -91,7 +87,7 @@ impl ApubObject for ApubCommunityOutbox { ordered_items.push(a); } - Ok(CommunityOutbox { + Ok(GroupOutbox { context: lemmy_context(), r#type: OrderedCollectionType::OrderedCollection, id: generate_outbox_url(&data.0.actor_id)?.into(), diff --git a/crates/apub/src/collections/mod.rs b/crates/apub/src/collections/mod.rs index 43a48af33..948824e22 100644 --- a/crates/apub/src/collections/mod.rs +++ b/crates/apub/src/collections/mod.rs @@ -1 +1,7 @@ +use crate::objects::community::ApubCommunity; +use lemmy_websocket::LemmyContext; +pub(crate) mod community_moderators; pub(crate) mod community_outbox; + +/// Put community in the data, so we dont have to read it again from the database. +pub(crate) struct CommunityContext(pub ApubCommunity, pub LemmyContext); diff --git a/crates/apub/src/fetcher/community.rs b/crates/apub/src/fetcher/community.rs deleted file mode 100644 index 1c153b5fa..000000000 --- a/crates/apub/src/fetcher/community.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::{ - fetcher::{fetch::fetch_remote_object, object_id::ObjectId}, - objects::{community::Group, person::ApubPerson}, -}; -use activitystreams::collection::{CollectionExt, OrderedCollection}; -use anyhow::Context; -use lemmy_api_common::blocking; -use lemmy_db_schema::{ - source::community::{Community, CommunityModerator, CommunityModeratorForm}, - traits::Joinable, -}; -use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView; -use lemmy_utils::{location_info, LemmyError}; -use lemmy_websocket::LemmyContext; -use url::Url; - -pub(crate) async fn update_community_mods( - group: &Group, - community: &Community, - context: &LemmyContext, - request_counter: &mut i32, -) -> Result<(), LemmyError> { - let new_moderators = fetch_community_mods(context, group, request_counter).await?; - let community_id = community.id; - let current_moderators = blocking(context.pool(), move |conn| { - CommunityModeratorView::for_community(conn, community_id) - }) - .await??; - // Remove old mods from database which arent in the moderators collection anymore - for mod_user in ¤t_moderators { - if !new_moderators.contains(&mod_user.moderator.actor_id.clone().into()) { - let community_moderator_form = CommunityModeratorForm { - community_id: mod_user.community.id, - person_id: mod_user.moderator.id, - }; - blocking(context.pool(), move |conn| { - CommunityModerator::leave(conn, &community_moderator_form) - }) - .await??; - } - } - - // Add new mods to database which have been added to moderators collection - for mod_id in new_moderators { - let mod_id = ObjectId::new(mod_id); - let mod_user: ApubPerson = mod_id.dereference(context, request_counter).await?; - - if !current_moderators - .clone() - .iter() - .map(|c| c.moderator.actor_id.clone()) - .any(|x| x == mod_user.actor_id) - { - let community_moderator_form = CommunityModeratorForm { - community_id: community.id, - person_id: mod_user.id, - }; - blocking(context.pool(), move |conn| { - CommunityModerator::join(conn, &community_moderator_form) - }) - .await??; - } - } - - Ok(()) -} - -async fn fetch_community_mods( - context: &LemmyContext, - group: &Group, - recursion_counter: &mut i32, -) -> Result, LemmyError> { - if let Some(mods_url) = &group.moderators { - let mods = fetch_remote_object::( - context.client(), - &context.settings(), - mods_url, - recursion_counter, - ) - .await?; - let mods = mods - .items() - .map(|i| i.as_many()) - .flatten() - .context(location_info!())? - .iter() - .filter_map(|i| i.as_xsd_any_uri()) - .map(|u| u.to_owned()) - .collect(); - Ok(mods) - } else { - Ok(vec![]) - } -} diff --git a/crates/apub/src/fetcher/fetch.rs b/crates/apub/src/fetcher/fetch.rs deleted file mode 100644 index 95b2f55fc..000000000 --- a/crates/apub/src/fetcher/fetch.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::check_is_apub_id_valid; -use anyhow::anyhow; -use lemmy_apub_lib::APUB_JSON_CONTENT_TYPE; -use lemmy_utils::{request::retry, settings::structs::Settings, LemmyError}; -use log::info; -use reqwest::Client; -use serde::Deserialize; -use std::time::Duration; -use url::Url; - -/// Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object -/// fetch through the search). -/// -/// A community fetch will load the outbox with up to 20 items, and fetch the creator for each item. -/// So we are looking at a maximum of 22 requests (rounded up just to be safe). -static MAX_REQUEST_NUMBER: i32 = 25; - -/// Fetch any type of ActivityPub object, handling things like HTTP headers, deserialisation, -/// timeouts etc. -pub(in crate::fetcher) async fn fetch_remote_object( - client: &Client, - settings: &Settings, - url: &Url, - recursion_counter: &mut i32, -) -> Result -where - Response: for<'de> Deserialize<'de> + std::fmt::Debug, -{ - *recursion_counter += 1; - if *recursion_counter > MAX_REQUEST_NUMBER { - return Err(anyhow!("Maximum recursion depth reached").into()); - } - check_is_apub_id_valid(url, false, settings)?; - - let timeout = Duration::from_secs(60); - - let res = retry(|| { - client - .get(url.as_str()) - .header("Accept", APUB_JSON_CONTENT_TYPE) - .timeout(timeout) - .send() - }) - .await?; - - let object = res.json().await?; - info!("Fetched remote object {}", url); - Ok(object) -} diff --git a/crates/apub/src/fetcher/mod.rs b/crates/apub/src/fetcher/mod.rs index 28488ecb8..db39d2282 100644 --- a/crates/apub/src/fetcher/mod.rs +++ b/crates/apub/src/fetcher/mod.rs @@ -1,5 +1,3 @@ -pub mod community; -mod fetch; pub mod object_id; pub mod post_or_comment; pub mod search; diff --git a/crates/apub/src/fetcher/object_id.rs b/crates/apub/src/fetcher/object_id.rs index f29bbf6c5..66466362e 100644 --- a/crates/apub/src/fetcher/object_id.rs +++ b/crates/apub/src/fetcher/object_id.rs @@ -8,6 +8,7 @@ use lemmy_utils::{ settings::structs::Settings, LemmyError, }; +use log::info; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; use std::{ @@ -115,6 +116,7 @@ where ) -> Result { // dont fetch local objects this way debug_assert!(self.0.domain() != Some(&Settings::get().hostname)); + info!("Fetching remote object {}", self.to_string()); *request_counter += 1; if *request_counter > REQUEST_LIMIT { diff --git a/crates/apub/src/http/community.rs b/crates/apub/src/http/community.rs index 14870c888..1e10e3844 100644 --- a/crates/apub/src/http/community.rs +++ b/crates/apub/src/http/community.rs @@ -5,10 +5,14 @@ use crate::{ following::{follow::FollowCommunity, undo::UndoFollowCommunity}, report::Report, }, - collections::community_outbox::{ApubCommunityOutbox, OutboxData}, + collections::{ + community_moderators::ApubCommunityModerators, + community_outbox::ApubCommunityOutbox, + CommunityContext, + }, context::lemmy_context, fetcher::object_id::ObjectId, - generate_moderators_url, + generate_outbox_url, http::{ create_apub_response, create_apub_tombstone_response, @@ -19,17 +23,13 @@ use crate::{ }; use activitystreams::{ base::BaseExt, - collection::{CollectionExt, OrderedCollection, UnorderedCollection}, - url::Url, + collection::{CollectionExt, UnorderedCollection}, }; use actix_web::{body::Body, web, web::Payload, HttpRequest, HttpResponse}; use lemmy_api_common::blocking; use lemmy_apub_lib::traits::{ActivityFields, ActivityHandler, ApubObject}; use lemmy_db_schema::source::community::Community; -use lemmy_db_views_actor::{ - community_follower_view::CommunityFollowerView, - community_moderator_view::CommunityModeratorView, -}; +use lemmy_db_views_actor::community_follower_view::CommunityFollowerView; use lemmy_utils::LemmyError; use lemmy_websocket::LemmyContext; use log::trace; @@ -128,36 +128,18 @@ pub(crate) async fn get_apub_community_followers( /// activites like votes or comments). pub(crate) async fn get_apub_community_outbox( info: web::Path, - req: HttpRequest, context: web::Data, ) -> Result, LemmyError> { let community = blocking(context.pool(), move |conn| { Community::read_from_name(conn, &info.community_name) }) .await??; - let outbox_data = OutboxData(community.into(), context.get_ref().clone()); - let url = Url::parse(&req.head().uri.to_string())?; - let id = ObjectId::::new(url); - let outbox = id.dereference(&outbox_data, &mut 0).await?; + let id = ObjectId::new(generate_outbox_url(&community.actor_id)?.into_inner()); + let outbox_data = CommunityContext(community.into(), context.get_ref().clone()); + let outbox: ApubCommunityOutbox = id.dereference(&outbox_data, &mut 0).await?; Ok(create_apub_response(&outbox.to_apub(&outbox_data).await?)) } -pub(crate) async fn get_apub_community_inbox( - info: web::Path, - context: web::Data, -) -> Result, LemmyError> { - let community = blocking(context.pool(), move |conn| { - Community::read_from_name(conn, &info.community_name) - }) - .await??; - - let mut collection = OrderedCollection::new(); - collection - .set_id(community.inbox_url.into()) - .set_many_contexts(lemmy_context()); - Ok(create_apub_response(&collection)) -} - pub(crate) async fn get_apub_community_moderators( info: web::Path, context: web::Data, @@ -167,26 +149,10 @@ pub(crate) async fn get_apub_community_moderators( }) .await?? .into(); - - // The attributed to, is an ordered vector with the creator actor_ids first, - // then the rest of the moderators - // TODO Technically the instance admins can mod the community, but lets - // ignore that for now - let cid = community.id; - let moderators = blocking(context.pool(), move |conn| { - CommunityModeratorView::for_community(conn, cid) - }) - .await??; - - let moderators: Vec = moderators - .into_iter() - .map(|m| m.moderator.actor_id.into()) - .collect(); - let mut collection = OrderedCollection::new(); - collection - .set_id(generate_moderators_url(&community.actor_id)?.into()) - .set_total_items(moderators.len() as u64) - .set_many_items(moderators) - .set_many_contexts(lemmy_context()); - Ok(create_apub_response(&collection)) + let id = ObjectId::new(generate_outbox_url(&community.actor_id)?.into_inner()); + let outbox_data = CommunityContext(community, context.get_ref().clone()); + let moderators: ApubCommunityModerators = id.dereference(&outbox_data, &mut 0).await?; + Ok(create_apub_response( + &moderators.to_apub(&outbox_data).await?, + )) } diff --git a/crates/apub/src/http/person.rs b/crates/apub/src/http/person.rs index 470ff4749..a7e94020e 100644 --- a/crates/apub/src/http/person.rs +++ b/crates/apub/src/http/person.rs @@ -109,19 +109,3 @@ pub(crate) async fn get_apub_person_outbox( .set_total_items(0_u64); Ok(create_apub_response(&collection)) } - -pub(crate) async fn get_apub_person_inbox( - info: web::Path, - context: web::Data, -) -> Result, LemmyError> { - let person = blocking(context.pool(), move |conn| { - Person::find_by_name(conn, &info.user_name) - }) - .await??; - - let mut collection = OrderedCollection::new(); - collection - .set_id(person.inbox_url.into()) - .set_many_contexts(lemmy_context()); - Ok(create_apub_response(&collection)) -} diff --git a/crates/apub/src/http/routes.rs b/crates/apub/src/http/routes.rs index 7adb0ca9f..5dfbc2382 100644 --- a/crates/apub/src/http/routes.rs +++ b/crates/apub/src/http/routes.rs @@ -4,12 +4,11 @@ use crate::http::{ community_inbox, get_apub_community_followers, get_apub_community_http, - get_apub_community_inbox, get_apub_community_moderators, get_apub_community_outbox, }, get_activity, - person::{get_apub_person_http, get_apub_person_inbox, get_apub_person_outbox, person_inbox}, + person::{get_apub_person_http, get_apub_person_outbox, person_inbox}, post::get_apub_post, shared_inbox, }; @@ -49,10 +48,6 @@ pub fn config(cfg: &mut web::ServiceConfig, settings: &Settings) { "/c/{community_name}/outbox", web::get().to(get_apub_community_outbox), ) - .route( - "/c/{community_name}/inbox", - web::get().to(get_apub_community_inbox), - ) .route( "/c/{community_name}/moderators", web::get().to(get_apub_community_moderators), @@ -62,7 +57,6 @@ pub fn config(cfg: &mut web::ServiceConfig, settings: &Settings) { "/u/{user_name}/outbox", web::get().to(get_apub_person_outbox), ) - .route("/u/{user_name}/inbox", web::get().to(get_apub_person_inbox)) .route("/post/{post_id}", web::get().to(get_apub_post)) .route("/comment/{comment_id}", web::get().to(get_apub_comment)) .route("/activities/{type_}/{id}", web::get().to(get_activity)), diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index ee1f6e80c..bcd2f1adb 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -1,8 +1,12 @@ use crate::{ check_is_apub_id_valid, - collections::community_outbox::{ApubCommunityOutbox, OutboxData}, + collections::{ + community_moderators::ApubCommunityModerators, + community_outbox::ApubCommunityOutbox, + CommunityContext, + }, context::lemmy_context, - fetcher::{community::update_community_mods, object_id::ObjectId}, + fetcher::object_id::ObjectId, generate_moderators_url, generate_outbox_url, objects::{create_tombstone, get_summary_from_string_or_source, ImageObject, Source}, @@ -64,7 +68,7 @@ pub struct Group { // lemmy extension sensitive: Option, // lemmy extension - pub(crate) moderators: Option, + pub(crate) moderators: Option>, inbox: Url, pub(crate) outbox: ObjectId, followers: Url, @@ -192,7 +196,9 @@ impl ApubObject for ApubCommunity { icon, image, sensitive: Some(self.nsfw), - moderators: Some(generate_moderators_url(&self.actor_id)?.into()), + moderators: Some(ObjectId::::new( + generate_moderators_url(&self.actor_id)?.into_inner(), + )), inbox: self.inbox_url.clone().into(), outbox: ObjectId::new(generate_outbox_url(&self.actor_id)?), followers: self.followers_url.clone().into(), @@ -232,12 +238,8 @@ impl ApubObject for ApubCommunity { blocking(context.pool(), move |conn| Community::upsert(conn, &form)) .await?? .into(); - update_community_mods(group, &community, context, request_counter) - .await - .map_err(|e| debug!("{}", e)) - .ok(); + let outbox_data = CommunityContext(community.clone(), context.clone()); - let outbox_data = OutboxData(community.clone(), context.clone()); group .outbox .dereference(&outbox_data, request_counter) @@ -245,6 +247,14 @@ impl ApubObject for ApubCommunity { .map_err(|e| debug!("{}", e)) .ok(); + if let Some(moderators) = &group.moderators { + moderators + .dereference(&outbox_data, request_counter) + .await + .map_err(|e| debug!("{}", e)) + .ok(); + } + Ok(community) } } @@ -322,8 +332,9 @@ mod tests { let mut json: Group = file_to_json_object("assets/lemmy-community.json"); let json_orig = json.clone(); // change these links so they dont fetch over the network - json.moderators = - Some(Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_moderators").unwrap()); + json.moderators = Some(ObjectId::new( + Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_moderators").unwrap(), + )); json.outbox = ObjectId::new(Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_outbox").unwrap()); diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index fd98f87ad..f4ba225b8 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -79,7 +79,7 @@ impl Person { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct ApubPerson(DbPerson); impl Deref for ApubPerson {