Rewrite community moderators collection

rewrite-collections
Felix Ableitner 2021-10-27 12:56:07 +02:00
parent a75b6cb5c9
commit b439cb36aa
12 changed files with 197 additions and 242 deletions

View File

@ -0,0 +1,141 @@
use crate::{
collections::CommunityContext,
context::lemmy_context,
fetcher::object_id::ObjectId,
generate_moderators_url,
objects::person::ApubPerson,
};
use activitystreams::{
base::AnyBase,
chrono::NaiveDateTime,
collection::kind::OrderedCollectionType,
primitives::OneOrMany,
url::Url,
};
use lemmy_api_common::blocking;
use lemmy_apub_lib::{traits::ApubObject, verify::verify_domains_match};
use lemmy_db_schema::{
source::community::{CommunityModerator, CommunityModeratorForm},
traits::Joinable,
};
use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView;
use lemmy_utils::LemmyError;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[skip_serializing_none]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GroupModerators {
#[serde(rename = "@context")]
context: OneOrMany<AnyBase>,
r#type: OrderedCollectionType,
id: Url,
ordered_items: Vec<ObjectId<ApubPerson>>,
}
#[derive(Clone, Debug)]
pub(crate) struct ApubCommunityModerators(pub(crate) Vec<CommunityModeratorView>);
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubCommunityModerators {
type DataType = CommunityContext;
type TombstoneType = ();
type ApubType = GroupModerators;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
}
async fn read_from_apub_id(
_object_id: Url,
data: &Self::DataType,
) -> Result<Option<Self>, LemmyError> {
// Only read from database if its a local community, otherwise fetch over http
if data.0.local {
let cid = data.0.id;
let moderators = blocking(data.1.pool(), move |conn| {
CommunityModeratorView::for_community(conn, cid)
})
.await??;
Ok(Some(ApubCommunityModerators { 0: moderators }))
} else {
Ok(None)
}
}
async fn delete(self, _data: &Self::DataType) -> Result<(), LemmyError> {
unimplemented!()
}
async fn to_apub(&self, data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
let ordered_items = self
.0
.iter()
.map(|m| ObjectId::<ApubPerson>::new(m.moderator.actor_id.clone().into_inner()))
.collect();
Ok(GroupModerators {
context: lemmy_context(),
r#type: OrderedCollectionType::OrderedCollection,
id: generate_moderators_url(&data.0.actor_id)?.into(),
ordered_items,
})
}
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError> {
unimplemented!()
}
async fn from_apub(
apub: &Self::ApubType,
data: &Self::DataType,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<Self, LemmyError> {
verify_domains_match(expected_domain, &apub.id)?;
let community_id = data.0.id;
let current_moderators = blocking(data.1.pool(), move |conn| {
CommunityModeratorView::for_community(conn, community_id)
})
.await??;
// Remove old mods from database which arent in the moderators collection anymore
for mod_user in &current_moderators {
let mod_id = ObjectId::new(mod_user.moderator.actor_id.clone().into_inner());
if !apub.ordered_items.contains(&mod_id) {
let community_moderator_form = CommunityModeratorForm {
community_id: mod_user.community.id,
person_id: mod_user.moderator.id,
};
blocking(data.1.pool(), move |conn| {
CommunityModerator::leave(conn, &community_moderator_form)
})
.await??;
}
}
// Add new mods to database which have been added to moderators collection
for mod_id in &apub.ordered_items {
let mod_id = ObjectId::new(mod_id.clone());
let mod_user: ApubPerson = mod_id.dereference(&data.1, request_counter).await?;
if !current_moderators
.clone()
.iter()
.map(|c| c.moderator.actor_id.clone())
.any(|x| x == mod_user.actor_id)
{
let community_moderator_form = CommunityModeratorForm {
community_id: data.0.id,
person_id: mod_user.id,
};
blocking(data.1.pool(), move |conn| {
CommunityModerator::join(conn, &community_moderator_form)
})
.await??;
}
}
// This return value is unused, so just set an empty vec
Ok(ApubCommunityModerators { 0: vec![] })
}
}

View File

@ -1,14 +1,14 @@
use crate::{
activities::{post::create_or_update::CreateOrUpdatePost, CreateOrUpdateType},
collections::CommunityContext,
context::lemmy_context,
generate_outbox_url,
objects::{community::ApubCommunity, person::ApubPerson, post::ApubPost},
objects::{person::ApubPerson, post::ApubPost},
};
use activitystreams::{
base::AnyBase,
chrono::NaiveDateTime,
collection::kind::OrderedCollectionType,
object::Tombstone,
primitives::OneOrMany,
url::Url,
};
@ -23,14 +23,13 @@ use lemmy_db_schema::{
traits::Crud,
};
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[skip_serializing_none]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CommunityOutbox {
pub struct GroupOutbox {
#[serde(rename = "@context")]
context: OneOrMany<AnyBase>,
r#type: OrderedCollectionType,
@ -41,14 +40,11 @@ pub struct CommunityOutbox {
#[derive(Clone, Debug)]
pub(crate) struct ApubCommunityOutbox(Vec<ApubPost>);
/// Put community in the data, so we dont have to read it again from the database.
pub(crate) struct OutboxData(pub ApubCommunity, pub LemmyContext);
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubCommunityOutbox {
type DataType = OutboxData;
type TombstoneType = Tombstone;
type ApubType = CommunityOutbox;
type DataType = CommunityContext;
type TombstoneType = ();
type ApubType = GroupOutbox;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
@ -91,7 +87,7 @@ impl ApubObject for ApubCommunityOutbox {
ordered_items.push(a);
}
Ok(CommunityOutbox {
Ok(GroupOutbox {
context: lemmy_context(),
r#type: OrderedCollectionType::OrderedCollection,
id: generate_outbox_url(&data.0.actor_id)?.into(),

View File

@ -1 +1,7 @@
use crate::objects::community::ApubCommunity;
use lemmy_websocket::LemmyContext;
pub(crate) mod community_moderators;
pub(crate) mod community_outbox;
/// Put community in the data, so we dont have to read it again from the database.
pub(crate) struct CommunityContext(pub ApubCommunity, pub LemmyContext);

View File

@ -1,94 +0,0 @@
use crate::{
fetcher::{fetch::fetch_remote_object, object_id::ObjectId},
objects::{community::Group, person::ApubPerson},
};
use activitystreams::collection::{CollectionExt, OrderedCollection};
use anyhow::Context;
use lemmy_api_common::blocking;
use lemmy_db_schema::{
source::community::{Community, CommunityModerator, CommunityModeratorForm},
traits::Joinable,
};
use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView;
use lemmy_utils::{location_info, LemmyError};
use lemmy_websocket::LemmyContext;
use url::Url;
pub(crate) async fn update_community_mods(
group: &Group,
community: &Community,
context: &LemmyContext,
request_counter: &mut i32,
) -> Result<(), LemmyError> {
let new_moderators = fetch_community_mods(context, group, request_counter).await?;
let community_id = community.id;
let current_moderators = blocking(context.pool(), move |conn| {
CommunityModeratorView::for_community(conn, community_id)
})
.await??;
// Remove old mods from database which arent in the moderators collection anymore
for mod_user in &current_moderators {
if !new_moderators.contains(&mod_user.moderator.actor_id.clone().into()) {
let community_moderator_form = CommunityModeratorForm {
community_id: mod_user.community.id,
person_id: mod_user.moderator.id,
};
blocking(context.pool(), move |conn| {
CommunityModerator::leave(conn, &community_moderator_form)
})
.await??;
}
}
// Add new mods to database which have been added to moderators collection
for mod_id in new_moderators {
let mod_id = ObjectId::new(mod_id);
let mod_user: ApubPerson = mod_id.dereference(context, request_counter).await?;
if !current_moderators
.clone()
.iter()
.map(|c| c.moderator.actor_id.clone())
.any(|x| x == mod_user.actor_id)
{
let community_moderator_form = CommunityModeratorForm {
community_id: community.id,
person_id: mod_user.id,
};
blocking(context.pool(), move |conn| {
CommunityModerator::join(conn, &community_moderator_form)
})
.await??;
}
}
Ok(())
}
async fn fetch_community_mods(
context: &LemmyContext,
group: &Group,
recursion_counter: &mut i32,
) -> Result<Vec<Url>, LemmyError> {
if let Some(mods_url) = &group.moderators {
let mods = fetch_remote_object::<OrderedCollection>(
context.client(),
&context.settings(),
mods_url,
recursion_counter,
)
.await?;
let mods = mods
.items()
.map(|i| i.as_many())
.flatten()
.context(location_info!())?
.iter()
.filter_map(|i| i.as_xsd_any_uri())
.map(|u| u.to_owned())
.collect();
Ok(mods)
} else {
Ok(vec![])
}
}

View File

@ -1,49 +0,0 @@
use crate::check_is_apub_id_valid;
use anyhow::anyhow;
use lemmy_apub_lib::APUB_JSON_CONTENT_TYPE;
use lemmy_utils::{request::retry, settings::structs::Settings, LemmyError};
use log::info;
use reqwest::Client;
use serde::Deserialize;
use std::time::Duration;
use url::Url;
/// Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object
/// fetch through the search).
///
/// A community fetch will load the outbox with up to 20 items, and fetch the creator for each item.
/// So we are looking at a maximum of 22 requests (rounded up just to be safe).
static MAX_REQUEST_NUMBER: i32 = 25;
/// Fetch any type of ActivityPub object, handling things like HTTP headers, deserialisation,
/// timeouts etc.
pub(in crate::fetcher) async fn fetch_remote_object<Response>(
client: &Client,
settings: &Settings,
url: &Url,
recursion_counter: &mut i32,
) -> Result<Response, LemmyError>
where
Response: for<'de> Deserialize<'de> + std::fmt::Debug,
{
*recursion_counter += 1;
if *recursion_counter > MAX_REQUEST_NUMBER {
return Err(anyhow!("Maximum recursion depth reached").into());
}
check_is_apub_id_valid(url, false, settings)?;
let timeout = Duration::from_secs(60);
let res = retry(|| {
client
.get(url.as_str())
.header("Accept", APUB_JSON_CONTENT_TYPE)
.timeout(timeout)
.send()
})
.await?;
let object = res.json().await?;
info!("Fetched remote object {}", url);
Ok(object)
}

View File

@ -1,5 +1,3 @@
pub mod community;
mod fetch;
pub mod object_id;
pub mod post_or_comment;
pub mod search;

View File

@ -8,6 +8,7 @@ use lemmy_utils::{
settings::structs::Settings,
LemmyError,
};
use log::info;
use reqwest::{Client, StatusCode};
use serde::{Deserialize, Serialize};
use std::{
@ -115,6 +116,7 @@ where
) -> Result<Kind, LemmyError> {
// dont fetch local objects this way
debug_assert!(self.0.domain() != Some(&Settings::get().hostname));
info!("Fetching remote object {}", self.to_string());
*request_counter += 1;
if *request_counter > REQUEST_LIMIT {

View File

@ -5,10 +5,14 @@ use crate::{
following::{follow::FollowCommunity, undo::UndoFollowCommunity},
report::Report,
},
collections::community_outbox::{ApubCommunityOutbox, OutboxData},
collections::{
community_moderators::ApubCommunityModerators,
community_outbox::ApubCommunityOutbox,
CommunityContext,
},
context::lemmy_context,
fetcher::object_id::ObjectId,
generate_moderators_url,
generate_outbox_url,
http::{
create_apub_response,
create_apub_tombstone_response,
@ -19,17 +23,13 @@ use crate::{
};
use activitystreams::{
base::BaseExt,
collection::{CollectionExt, OrderedCollection, UnorderedCollection},
url::Url,
collection::{CollectionExt, UnorderedCollection},
};
use actix_web::{body::Body, web, web::Payload, HttpRequest, HttpResponse};
use lemmy_api_common::blocking;
use lemmy_apub_lib::traits::{ActivityFields, ActivityHandler, ApubObject};
use lemmy_db_schema::source::community::Community;
use lemmy_db_views_actor::{
community_follower_view::CommunityFollowerView,
community_moderator_view::CommunityModeratorView,
};
use lemmy_db_views_actor::community_follower_view::CommunityFollowerView;
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
use log::trace;
@ -128,36 +128,18 @@ pub(crate) async fn get_apub_community_followers(
/// activites like votes or comments).
pub(crate) async fn get_apub_community_outbox(
info: web::Path<CommunityQuery>,
req: HttpRequest,
context: web::Data<LemmyContext>,
) -> Result<HttpResponse<Body>, LemmyError> {
let community = blocking(context.pool(), move |conn| {
Community::read_from_name(conn, &info.community_name)
})
.await??;
let outbox_data = OutboxData(community.into(), context.get_ref().clone());
let url = Url::parse(&req.head().uri.to_string())?;
let id = ObjectId::<ApubCommunityOutbox>::new(url);
let outbox = id.dereference(&outbox_data, &mut 0).await?;
let id = ObjectId::new(generate_outbox_url(&community.actor_id)?.into_inner());
let outbox_data = CommunityContext(community.into(), context.get_ref().clone());
let outbox: ApubCommunityOutbox = id.dereference(&outbox_data, &mut 0).await?;
Ok(create_apub_response(&outbox.to_apub(&outbox_data).await?))
}
pub(crate) async fn get_apub_community_inbox(
info: web::Path<CommunityQuery>,
context: web::Data<LemmyContext>,
) -> Result<HttpResponse<Body>, LemmyError> {
let community = blocking(context.pool(), move |conn| {
Community::read_from_name(conn, &info.community_name)
})
.await??;
let mut collection = OrderedCollection::new();
collection
.set_id(community.inbox_url.into())
.set_many_contexts(lemmy_context());
Ok(create_apub_response(&collection))
}
pub(crate) async fn get_apub_community_moderators(
info: web::Path<CommunityQuery>,
context: web::Data<LemmyContext>,
@ -167,26 +149,10 @@ pub(crate) async fn get_apub_community_moderators(
})
.await??
.into();
// The attributed to, is an ordered vector with the creator actor_ids first,
// then the rest of the moderators
// TODO Technically the instance admins can mod the community, but lets
// ignore that for now
let cid = community.id;
let moderators = blocking(context.pool(), move |conn| {
CommunityModeratorView::for_community(conn, cid)
})
.await??;
let moderators: Vec<Url> = moderators
.into_iter()
.map(|m| m.moderator.actor_id.into())
.collect();
let mut collection = OrderedCollection::new();
collection
.set_id(generate_moderators_url(&community.actor_id)?.into())
.set_total_items(moderators.len() as u64)
.set_many_items(moderators)
.set_many_contexts(lemmy_context());
Ok(create_apub_response(&collection))
let id = ObjectId::new(generate_outbox_url(&community.actor_id)?.into_inner());
let outbox_data = CommunityContext(community, context.get_ref().clone());
let moderators: ApubCommunityModerators = id.dereference(&outbox_data, &mut 0).await?;
Ok(create_apub_response(
&moderators.to_apub(&outbox_data).await?,
))
}

View File

@ -109,19 +109,3 @@ pub(crate) async fn get_apub_person_outbox(
.set_total_items(0_u64);
Ok(create_apub_response(&collection))
}
pub(crate) async fn get_apub_person_inbox(
info: web::Path<PersonQuery>,
context: web::Data<LemmyContext>,
) -> Result<HttpResponse<Body>, LemmyError> {
let person = blocking(context.pool(), move |conn| {
Person::find_by_name(conn, &info.user_name)
})
.await??;
let mut collection = OrderedCollection::new();
collection
.set_id(person.inbox_url.into())
.set_many_contexts(lemmy_context());
Ok(create_apub_response(&collection))
}

View File

@ -4,12 +4,11 @@ use crate::http::{
community_inbox,
get_apub_community_followers,
get_apub_community_http,
get_apub_community_inbox,
get_apub_community_moderators,
get_apub_community_outbox,
},
get_activity,
person::{get_apub_person_http, get_apub_person_inbox, get_apub_person_outbox, person_inbox},
person::{get_apub_person_http, get_apub_person_outbox, person_inbox},
post::get_apub_post,
shared_inbox,
};
@ -49,10 +48,6 @@ pub fn config(cfg: &mut web::ServiceConfig, settings: &Settings) {
"/c/{community_name}/outbox",
web::get().to(get_apub_community_outbox),
)
.route(
"/c/{community_name}/inbox",
web::get().to(get_apub_community_inbox),
)
.route(
"/c/{community_name}/moderators",
web::get().to(get_apub_community_moderators),
@ -62,7 +57,6 @@ pub fn config(cfg: &mut web::ServiceConfig, settings: &Settings) {
"/u/{user_name}/outbox",
web::get().to(get_apub_person_outbox),
)
.route("/u/{user_name}/inbox", web::get().to(get_apub_person_inbox))
.route("/post/{post_id}", web::get().to(get_apub_post))
.route("/comment/{comment_id}", web::get().to(get_apub_comment))
.route("/activities/{type_}/{id}", web::get().to(get_activity)),

View File

@ -1,8 +1,12 @@
use crate::{
check_is_apub_id_valid,
collections::community_outbox::{ApubCommunityOutbox, OutboxData},
collections::{
community_moderators::ApubCommunityModerators,
community_outbox::ApubCommunityOutbox,
CommunityContext,
},
context::lemmy_context,
fetcher::{community::update_community_mods, object_id::ObjectId},
fetcher::object_id::ObjectId,
generate_moderators_url,
generate_outbox_url,
objects::{create_tombstone, get_summary_from_string_or_source, ImageObject, Source},
@ -64,7 +68,7 @@ pub struct Group {
// lemmy extension
sensitive: Option<bool>,
// lemmy extension
pub(crate) moderators: Option<Url>,
pub(crate) moderators: Option<ObjectId<ApubCommunityModerators>>,
inbox: Url,
pub(crate) outbox: ObjectId<ApubCommunityOutbox>,
followers: Url,
@ -192,7 +196,9 @@ impl ApubObject for ApubCommunity {
icon,
image,
sensitive: Some(self.nsfw),
moderators: Some(generate_moderators_url(&self.actor_id)?.into()),
moderators: Some(ObjectId::<ApubCommunityModerators>::new(
generate_moderators_url(&self.actor_id)?.into_inner(),
)),
inbox: self.inbox_url.clone().into(),
outbox: ObjectId::new(generate_outbox_url(&self.actor_id)?),
followers: self.followers_url.clone().into(),
@ -232,12 +238,8 @@ impl ApubObject for ApubCommunity {
blocking(context.pool(), move |conn| Community::upsert(conn, &form))
.await??
.into();
update_community_mods(group, &community, context, request_counter)
.await
.map_err(|e| debug!("{}", e))
.ok();
let outbox_data = CommunityContext(community.clone(), context.clone());
let outbox_data = OutboxData(community.clone(), context.clone());
group
.outbox
.dereference(&outbox_data, request_counter)
@ -245,6 +247,14 @@ impl ApubObject for ApubCommunity {
.map_err(|e| debug!("{}", e))
.ok();
if let Some(moderators) = &group.moderators {
moderators
.dereference(&outbox_data, request_counter)
.await
.map_err(|e| debug!("{}", e))
.ok();
}
Ok(community)
}
}
@ -322,8 +332,9 @@ mod tests {
let mut json: Group = file_to_json_object("assets/lemmy-community.json");
let json_orig = json.clone();
// change these links so they dont fetch over the network
json.moderators =
Some(Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_moderators").unwrap());
json.moderators = Some(ObjectId::new(
Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_moderators").unwrap(),
));
json.outbox =
ObjectId::new(Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_outbox").unwrap());

View File

@ -79,7 +79,7 @@ impl Person {
}
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, PartialEq)]
pub struct ApubPerson(DbPerson);
impl Deref for ApubPerson {