Feat: Batching for lists (#47)

* feat: Added batch censures

* Batching endorsements and hesitations
pull/53/head
Divided by Zer0 2023-10-08 01:19:23 +02:00 committed by GitHub
parent ffd569895b
commit d9dcb6902d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 424 additions and 11 deletions

View File

@ -1,5 +1,10 @@
# Changelog
# 0.20.0
* Added batching for adding/removing/modifying censures
* Added soft limit for censures/endorsements/hesitations to 2000 entries
# 0.19.1
* Fixed Deleting tags

View File

@ -148,4 +148,25 @@ class Models:
'question': fields.String(description="The entry in question form", example="What is an FAQ?"),
'stub': fields.String(description="The entry in a short form", example="faq"),
'document': fields.String(description="The answer provided by this FAQ entry", example="An FAQ stands for Frequently Asked Questions."),
})
})
self.input_batch_entry = api.inherit('BatchEntry', self.input_censures_modify, {
'domain': fields.String(required=True, description="The domain for which this entry applies to", example="lemmy.example.com"),
})
self.input_batch_censures = api.model('BatchCensures', {
'delete': fields.Boolean(required=False, default=False, description="Set to true, to delete all censures which are not in the censures list."),
'overwrite': fields.Boolean(required=False, default=False, description="Set to true, to modify all existing entries with new data."),
'censures': fields.List(fields.Nested(self.input_batch_entry)),
})
self.input_batch_endorsements_entry = api.inherit('BatchEndorsementEntry', self.input_endorsements_modify, {
'domain': fields.String(required=True, description="The domain for which this entry applies to", example="lemmy.example.com"),
})
self.input_batch_endorsements = api.model('BatchEndorsements', {
'delete': fields.Boolean(required=False, default=False, description="Set to true, to delete all endorsements which are not in the endorsements list."),
'overwrite': fields.Boolean(required=False, default=False, description="Set to true, to modify all existing entries with new data."),
'endorsements': fields.List(fields.Nested(self.input_batch_endorsements_entry)),
})
self.input_batch_hesitations = api.model('BatchHesitations', {
'delete': fields.Boolean(required=False, default=False, description="Set to true, to delete all hesitations which are not in the hesitations list."),
'overwrite': fields.Boolean(required=False, default=False, description="Set to true, to modify all existing entries with new data."),
'hesitations': fields.List(fields.Nested(self.input_batch_entry)),
})

View File

@ -23,10 +23,13 @@ api.add_resource(solicitations.Solicitations, "/solicitations")
api.add_resource(whitelist.WhitelistDomain, "/whitelist/<string:domain>")
api.add_resource(endorsements.Endorsements, "/endorsements/<string:domain>")
api.add_resource(endorsements.Approvals, "/approvals/<string:domains_csv>")
api.add_resource(endorsements.BatchEndorsements, "/batch/endorsements")
api.add_resource(censures.Censures, "/censures/<string:domain>")
api.add_resource(censures.CensuresGiven, "/censures_given/<string:domains_csv>")
api.add_resource(censures.BatchCensures, "/batch/censures")
api.add_resource(hesitations.Hesitations, "/hesitations/<string:domain>")
api.add_resource(hesitations.HesitationsGiven, "/hesitations_given/<string:domains_csv>")
api.add_resource(hesitations.BatchHesitations, "/batch/hesitations")
api.add_resource(guarantees.Guarantors, "/guarantors/<string:domain>")
api.add_resource(guarantees.Guarantees, "/guarantees/<string:domain>")
api.add_resource(badges.GuaranteeBadge, "/badges/guarantees/<string:domain>.svg")

View File

@ -206,13 +206,12 @@ class Censures(Resource):
target_instance, instance_info = ensure_instance_registered(domain, allow_unreachable=True)
if not target_instance:
raise e.NotFound(f"Something went wrong trying to register this instance.")
if not target_instance:
raise e.BadRequest("Instance to censure not found")
if database.get_endorsement(target_instance.id,instance.id):
raise e.BadRequest("You can't censure an instance you've endorsed! Please withdraw the endorsement first.")
if database.get_censure(target_instance.id,instance.id):
return {"message":'OK'}, 200
if database.count_all_censured_instances_by_censuring_id(instance.id) >= instance.max_list_size:
raise e.Forbidden("You're reached the maximum amount of instances you can add to your censures. Please contact the admins of fediseer to increase this limit is needed.")
reason = self.args.reason
if reason is not None:
reason = sanitize_string(reason)
@ -341,3 +340,125 @@ class Censures(Resource):
db.session.commit()
logger.info(f"{instance.domain} Withdrew censure from {domain}")
return {"message":'Changed'}, 200
class BatchCensures(Resource):
decorators = [limiter.limit("2/minute", key_func = get_request_path)]
post_parser = reqparse.RequestParser()
post_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers')
post_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers")
post_parser.add_argument("delete", required=False, default=False, type=bool, help="Set to true, to delete all censures which are not in the censures list", location="json")
post_parser.add_argument("overwrite", required=False, default=False, type=bool, help="Set to true, to modify all existing entries with new data", location="json")
post_parser.add_argument("censures", default=None, type=list, required=True, location="json")
@api.expect(post_parser,models.input_batch_censures, validate=True)
@api.marshal_with(models.response_model_simple_response, code=200, description='Batch Censure Instances')
@api.response(400, 'Bad Request', models.response_model_error)
@api.response(401, 'Invalid API Key', models.response_model_error)
@api.response(403, 'Access Denied', models.response_model_error)
@api.response(404, 'Instance not registered', models.response_model_error)
def post(self):
'''Batch Censure instances
'''
self.args = self.post_parser.parse_args()
if not self.args.apikey:
raise e.Unauthorized("You must provide the API key that was PM'd to your admin account")
instance = database.find_instance_by_api_key(self.args.apikey)
if not instance:
raise e.NotFound(f"No Instance found matching provided API key and domain. Have you remembered to register it?")
if len(instance.guarantors) == 0:
raise e.Forbidden("Only guaranteed instances can censure others.")
if database.instance_has_flag(instance.id,enums.InstanceFlags.RESTRICTED):
raise e.Forbidden("You cannot take this action as your instance is restricted")
if database.has_too_many_actions_per_min(instance.domain):
raise e.TooManyRequests("Your instance is doing more than 20 actions per minute. Please slow down.")
unbroken_chain, chainbreaker = database.has_unbroken_chain(instance.id)
if not unbroken_chain:
raise e.Forbidden(f"Guarantee chain for this instance has been broken. Chain ends at {chainbreaker.domain}!")
if self.args.delete is True:
if len(self.args.censures) >= instance.max_list_size:
raise e.Forbidden("You're specified more than maximum amount of instances you can add to your censures. Please contact the admins of fediseer to increase this limit is needed.")
else:
if database.count_all_censured_instances_by_censuring_id([instance.id]) + len(self.args.censures) >= instance.max_list_size:
raise e.Forbidden("You're reached the maximum amount of instances you can add to your censures. Please contact the admins of fediseer to increase this limit is needed.")
if len(self.args.censures) == 0:
raise e.BadRequest("You have not provided any entries to append to your censures.")
added_entries = 0
deleted_entries = 0
modified_entries = 0
seen_domains = set()
if self.args.delete:
existing_censures = database.get_all_censured_instances_by_censuring_id([instance.id], limit=None)
new_censures = set([c["domain"] for c in self.args.censures])
for target_instance in existing_censures:
if target_instance.domain not in new_censures:
old_censure = database.get_censure(target_instance.id,instance.id)
db.session.delete(old_censure)
deleted_entries += 1
for entry in self.args.censures:
if entry["domain"] in seen_domains:
logger.info(f"Batch censure operation by {instance.domain} had duplicate entries for {entry['domain']}")
continue
seen_domains.add(entry["domain"])
if instance.domain == entry["domain"]:
continue
target_instance, instance_info = ensure_instance_registered(entry["domain"], allow_unreachable=True)
reason = entry.get("reason")
if reason is not None:
reason = sanitize_string(reason)
evidence = entry.get("evidence")
if evidence is not None:
evidence = sanitize_string(evidence)
if not target_instance:
continue
if database.get_endorsement(target_instance.id,instance.id):
continue
censure = database.get_censure(target_instance.id,instance.id)
if censure:
if self.args.overwrite is False:
continue
if censure.reason == reason and censure.evidence == evidence:
continue
censure.reason = reason
censure.evidence = evidence
modified_entries += 1
else:
new_censure = Censure(
censuring_id=instance.id,
censured_id=target_instance.id,
reason=reason,
evidence=evidence,
)
db.session.add(new_censure)
added_entries += 1
if added_entries + deleted_entries + modified_entries == 0:
return {"message":'OK'}, 200
if added_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.CENSURE,
report_activity=enums.ReportActivity.ADDED,
)
db.session.add(new_report)
if modified_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.CENSURE,
report_activity=enums.ReportActivity.MODIFIED,
)
db.session.add(new_report)
if deleted_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.CENSURE,
report_activity=enums.ReportActivity.DELETED,
)
db.session.add(new_report)
db.session.commit()
logger.info(f"{instance.domain} Batched Censures for {added_entries + modified_entries + deleted_entries} domains.")
return {"message":'Changed'}, 200

View File

@ -58,7 +58,7 @@ class Approvals(Resource):
if len(instances) == 0:
raise e.Forbidden(f"You do not have access to see these endorsements")
if self.args.min_endorsements > len(instances):
raise e.BadRequest(f"You cannot request more censures than the amount of reference domains")
raise e.BadRequest(f"You cannot request more endorsements than the amount of reference domains")
instance_details = []
for e_instance in database.get_all_endorsed_instances_by_approving_id(
approving_ids=[instance.id for instance in instances],
@ -332,3 +332,122 @@ class Endorsements(Resource):
pass
logger.info(f"{instance.domain} Withdrew endorsement from {domain}")
return {"message":'Changed'}, 200
class BatchEndorsements(Resource):
decorators = [limiter.limit("2/minute", key_func = get_request_path)]
post_parser = reqparse.RequestParser()
post_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers')
post_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers")
post_parser.add_argument("delete", required=False, default=False, type=bool, location="json")
post_parser.add_argument("overwrite", required=False, default=False, type=bool, location="json")
post_parser.add_argument("endorsements", default=None, type=list, required=True, location="json")
@api.expect(post_parser,models.input_batch_endorsements, validate=True)
@api.marshal_with(models.response_model_simple_response, code=200, description='Batch Endorse Instances')
@api.response(400, 'Bad Request', models.response_model_error)
@api.response(401, 'Invalid API Key', models.response_model_error)
@api.response(403, 'Access Denied', models.response_model_error)
@api.response(404, 'Instance not registered', models.response_model_error)
def post(self):
'''Batch Endorse instances
'''
self.args = self.post_parser.parse_args()
if not self.args.apikey:
raise e.Unauthorized("You must provide the API key that was PM'd to your admin account")
instance = database.find_instance_by_api_key(self.args.apikey)
if not instance:
raise e.NotFound(f"No Instance found matching provided API key and domain. Have you remembered to register it?")
if len(instance.guarantors) == 0:
raise e.Forbidden("Only guaranteed instances can endorse others.")
if database.instance_has_flag(instance.id,enums.InstanceFlags.RESTRICTED):
raise e.Forbidden("You cannot take this action as your instance is restricted")
if database.has_too_many_actions_per_min(instance.domain):
raise e.TooManyRequests("Your instance is doing more than 20 actions per minute. Please slow down.")
unbroken_chain, chainbreaker = database.has_unbroken_chain(instance.id)
if not unbroken_chain:
raise e.Forbidden(f"Guarantee chain for this instance has been broken. Chain ends at {chainbreaker.domain}!")
if self.args.delete is True:
if len(self.args.endorsements) >= instance.max_list_size:
raise e.Forbidden("You're specified more than maximum amount of instances you can add to your endorsements. Please contact the admins of fediseer to increase this limit is needed.")
else:
if database.count_all_endorsed_instances_by_approving_id([instance.id]) + len(self.args.endorsements) >= instance.max_list_size:
raise e.Forbidden("You're reached the maximum amount of instances you can add to your endorsements. Please contact the admins of fediseer to increase this limit is needed.")
if len(self.args.endorsements) == 0:
raise e.BadRequest("You have not provided any entries to append to your endorsements.")
added_entries = 0
deleted_entries = 0
modified_entries = 0
seen_domains = set()
if self.args.delete:
existing_endorsements = database.get_all_endorsed_instances_by_approving_id([instance.id], limit=None)
new_endorsements = set([c["domain"] for c in self.args.endorsements])
for target_instance in existing_endorsements:
if target_instance.domain not in new_endorsements:
old_endorsement = database.get_endorsement(target_instance.id,instance.id)
db.session.delete(old_endorsement)
deleted_entries += 1
for entry in self.args.endorsements:
if entry["domain"] in seen_domains:
logger.info(f"Batch endorsement operation by {instance.domain} had duplicate entries for {entry['domain']}")
continue
seen_domains.add(entry["domain"])
if instance.domain == entry["domain"]:
continue
target_instance, instance_info = ensure_instance_registered(entry["domain"], allow_unreachable=True)
reason = entry.get("reason")
if reason is not None:
reason = sanitize_string(reason)
if not target_instance:
continue
if database.get_censure(target_instance.id,instance.id):
continue
if database.get_hesitation(target_instance.id,instance.id):
continue
endorsement = database.get_endorsement(target_instance.id,instance.id)
if endorsement:
if self.args.overwrite is False:
continue
if endorsement.reason == reason:
continue
endorsement.reason = reason
modified_entries += 1
else:
new_endorsement = Endorsement(
approving_id=instance.id,
endorsed_id=target_instance.id,
reason=reason,
)
db.session.add(new_endorsement)
added_entries += 1
if added_entries + deleted_entries + modified_entries == 0:
return {"message":'OK'}, 200
if added_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.ENDORSEMENT,
report_activity=enums.ReportActivity.ADDED,
)
db.session.add(new_report)
if modified_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.ENDORSEMENT,
report_activity=enums.ReportActivity.MODIFIED,
)
db.session.add(new_report)
if deleted_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.ENDORSEMENT,
report_activity=enums.ReportActivity.DELETED,
)
db.session.add(new_report)
db.session.commit()
logger.info(f"{instance.domain} Batched endorsements for {added_entries + modified_entries + deleted_entries} domains.")
return {"message":'Changed'}, 200

View File

@ -327,3 +327,124 @@ class Hesitations(Resource):
db.session.commit()
logger.info(f"{instance.domain} Withdrew hesitation from {domain}")
return {"message":'Changed'}, 200
class BatchHesitations(Resource):
decorators = [limiter.limit("2/minute", key_func = get_request_path)]
post_parser = reqparse.RequestParser()
post_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers')
post_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers")
post_parser.add_argument("delete", required=False, default=False, type=bool, help="Set to true, to delete all hesitations which are not in the hesitation list", location="json")
post_parser.add_argument("overwrite", required=False, default=False, type=bool, help="Set to true, to modify all existing entries with new data", location="json")
post_parser.add_argument("hesitations", default=None, type=list, required=True, location="json")
@api.expect(post_parser,models.input_batch_hesitations, validate=True)
@api.marshal_with(models.response_model_simple_response, code=200, description='Batch Doubt Instances')
@api.response(400, 'Bad Request', models.response_model_error)
@api.response(401, 'Invalid API Key', models.response_model_error)
@api.response(403, 'Access Denied', models.response_model_error)
@api.response(404, 'Instance not registered', models.response_model_error)
def post(self):
'''Batch Doubt instances
'''
self.args = self.post_parser.parse_args()
if not self.args.apikey:
raise e.Unauthorized("You must provide the API key that was PM'd to your admin account")
instance = database.find_instance_by_api_key(self.args.apikey)
if not instance:
raise e.NotFound(f"No Instance found matching provided API key and domain. Have you remembered to register it?")
if len(instance.guarantors) == 0:
raise e.Forbidden("Only guaranteed instances can doubt others.")
if database.instance_has_flag(instance.id,enums.InstanceFlags.RESTRICTED):
raise e.Forbidden("You cannot take this action as your instance is restricted")
if database.has_too_many_actions_per_min(instance.domain):
raise e.TooManyRequests("Your instance is doing more than 20 actions per minute. Please slow down.")
unbroken_chain, chainbreaker = database.has_unbroken_chain(instance.id)
if not unbroken_chain:
raise e.Forbidden(f"Guarantee chain for this instance has been broken. Chain ends at {chainbreaker.domain}!")
if self.args.delete is True:
if len(self.args.hesitations) >= instance.max_list_size:
raise e.Forbidden("You're specified more than maximum amount of instances you can add to your hesitations. Please contact the admins of fediseer to increase this limit is needed.")
else:
if database.count_all_dubious_instances_by_hesitant_id([instance.id]) + len(self.args.hesitations) >= instance.max_list_size:
raise e.Forbidden("You're reached the maximum amount of instances you can add to your hesitations. Please contact the admins of fediseer to increase this limit is needed.")
if len(self.args.hesitations) == 0:
raise e.BadRequest("You have not provided any entries to append to your hesitations.")
added_entries = 0
deleted_entries = 0
modified_entries = 0
seen_domains = set()
if self.args.delete:
existing_hesitations = database.get_all_dubious_instances_by_hesitant_id([instance.id], limit=None)
new_hesitations = set([c["domain"] for c in self.args.hesitations])
for target_instance in existing_hesitations:
if target_instance.domain not in new_hesitations:
old_hesitation = database.get_hesitation(target_instance.id,instance.id)
db.session.delete(old_hesitation)
deleted_entries += 1
for entry in self.args.hesitations:
if entry["domain"] in seen_domains:
logger.info(f"Batch hesitation operation by {instance.domain} had duplicate entries for {entry['domain']}")
continue
seen_domains.add(entry["domain"])
if instance.domain == entry["domain"]:
continue
target_instance, instance_info = ensure_instance_registered(entry["domain"], allow_unreachable=True)
reason = entry.get("reason")
if reason is not None:
reason = sanitize_string(reason)
evidence = entry.get("evidence")
if evidence is not None:
evidence = sanitize_string(evidence)
if not target_instance:
continue
if database.get_endorsement(target_instance.id,instance.id):
continue
hesitation = database.get_hesitation(target_instance.id,instance.id)
if hesitation:
if self.args.overwrite is False:
continue
if hesitation.reason == reason and hesitation.evidence == evidence:
continue
hesitation.reason = reason
hesitation.evidence = evidence
modified_entries += 1
else:
new_hesitation = Hesitation(
hesitant_id=instance.id,
dubious_id=target_instance.id,
reason=reason,
evidence=evidence,
)
db.session.add(new_hesitation)
added_entries += 1
if added_entries + deleted_entries + modified_entries == 0:
return {"message":'OK'}, 200
if added_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.HESITATION,
report_activity=enums.ReportActivity.ADDED,
)
db.session.add(new_report)
if modified_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.HESITATION,
report_activity=enums.ReportActivity.MODIFIED,
)
db.session.add(new_report)
if deleted_entries > 0:
new_report = Report(
source_domain=instance.domain,
target_domain='[MULTIPLE]',
report_type=enums.ReportType.HESITATION,
report_activity=enums.ReportActivity.DELETED,
)
db.session.add(new_report)
db.session.commit()
logger.info(f"{instance.domain} Batched Hesitations for {added_entries + modified_entries + deleted_entries} domains.")
return {"message":'Changed'}, 200

View File

@ -121,6 +121,7 @@ class Instance(db.Model):
software = db.Column(db.String(50), unique=False, nullable=False, index=True)
sysadmins = db.Column(db.Integer, unique=False, nullable=True)
moderators = db.Column(db.Integer, unique=False, nullable=True)
max_list_size = db.Column(db.Integer, unique=False, nullable=False, default=2000)
pm_proxy = db.Column(Enum(enums.PMProxy), default=enums.PMProxy.NONE, nullable=False)
poll_failures = db.Column(db.Integer, default=0, nullable=True)
visibility_endorsements = db.Column(Enum(enums.ListVisibility), default=enums.ListVisibility.OPEN, nullable=False)

View File

@ -46,8 +46,8 @@ def get_all_instances(
page = 0
return query.order_by(Instance.created.desc()).offset(limit * page).limit(limit).all()
def get_all_endorsed_instances_by_approving_id(approving_ids,page=1,limit=100):
query = db.session.query(
def query_all_endorsed_instances_by_approving_id(approving_ids):
return db.session.query(
Instance
).outerjoin(
Instance.endorsements,
@ -58,6 +58,13 @@ def get_all_endorsed_instances_by_approving_id(approving_ids,page=1,limit=100):
).group_by(
Instance.id
)
def count_all_endorsed_instances_by_approving_id(approving_ids):
query = query_all_endorsed_instances_by_approving_id(approving_ids)
return query.count()
def get_all_endorsed_instances_by_approving_id(approving_ids,page=1,limit=100):
query = query_all_endorsed_instances_by_approving_id(approving_ids)
if limit is not None:
page -= 1
if page < 0:
@ -92,8 +99,8 @@ def get_all_endorsement_reasons_for_endorsed_id(endorsed_id, approving_ids):
return query.all()
def get_all_censured_instances_by_censuring_id(censuring_ids,page=1,limit=100):
query = db.session.query(
def query_all_censured_instances_by_censuring_id(censuring_ids):
return db.session.query(
Instance
).outerjoin(
Instance.censures_received,
@ -104,6 +111,13 @@ def get_all_censured_instances_by_censuring_id(censuring_ids,page=1,limit=100):
).group_by(
Instance.id
)
def count_all_censured_instances_by_censuring_id(censuring_ids):
query = query_all_censured_instances_by_censuring_id(censuring_ids)
return query.count()
def get_all_censured_instances_by_censuring_id(censuring_ids,page=1,limit=100):
query = query_all_censured_instances_by_censuring_id(censuring_ids)
if limit is not None:
page -= 1
if page < 0:
@ -139,8 +153,8 @@ def get_all_censure_reasons_for_censured_id(censured_id, censuring_ids):
return query.all()
def get_all_dubious_instances_by_hesitant_id(hesitant_ids,page=1,limit=100):
query = db.session.query(
def query_all_dubious_instances_by_hesitant_id(hesitant_ids):
return db.session.query(
Instance
).outerjoin(
Instance.hesitations_received,
@ -151,6 +165,13 @@ def get_all_dubious_instances_by_hesitant_id(hesitant_ids,page=1,limit=100):
).group_by(
Instance.id
)
def count_all_dubious_instances_by_hesitant_id(hesitant_ids):
query = query_all_dubious_instances_by_hesitant_id(hesitant_ids)
return query.count()
def get_all_dubious_instances_by_hesitant_id(hesitant_ids,page=1,limit=100):
query = query_all_dubious_instances_by_hesitant_id(hesitant_ids)
if limit is not None:
page -= 1
if page < 0:

View File

@ -0,0 +1 @@
ALTER TABLE instances ADD COLUMN max_list_size INTEGER default 2000;