From ffd569895bb33251b9732466becfa9f6e17d09ee Mon Sep 17 00:00:00 2001 From: db0 Date: Sat, 7 Oct 2023 00:56:59 +0200 Subject: [PATCH] feat: paginating unfiltered results --- fediseer/apis/v1/censures.py | 17 ++++++++++++++++- fediseer/apis/v1/endorsements.py | 18 ++++++++++++++---- fediseer/apis/v1/hesitations.py | 17 ++++++++++++++++- fediseer/apis/v1/whitelist.py | 4 ++-- fediseer/database/functions.py | 30 ++++++++++++++++++++++++------ 5 files changed, 72 insertions(+), 14 deletions(-) diff --git a/fediseer/apis/v1/censures.py b/fediseer/apis/v1/censures.py index 3a59c5b..ddf0199 100644 --- a/fediseer/apis/v1/censures.py +++ b/fediseer/apis/v1/censures.py @@ -13,6 +13,8 @@ class CensuresGiven(Resource): get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args") get_parser.add_argument("min_censures", required=False, default=1, type=int, help="Limit to this amount of censures of more", location="args") get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve censures where their reasons include any of the text in this csv", location="args") + get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve. Only unfiltered results will be paginated.", location="args") + get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which amount of results to retrieve. Only unfiltered results will be limited.", location="args") decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)] @api.expect(get_parser) @@ -27,6 +29,10 @@ class CensuresGiven(Resource): and the results will be a set of all their censures together. ''' self.args = self.get_parser.parse_args() + # if self.args.limit > 100: # Once limit is in effect + # raise e.BadRequest("limit cannot be more than 100") + if self.args.limit < 10: + raise e.BadRequest("Limit cannot be less than 10") get_instance = None if self.args.apikey: get_instance = database.find_instance_by_api_key(self.args.apikey) @@ -54,7 +60,16 @@ class CensuresGiven(Resource): if self.args.min_censures > len(instances): raise e.BadRequest(f"You cannot request more censures than the amount of reference domains") instance_details = [] - for c_instance in database.get_all_censured_instances_by_censuring_id([instance.id for instance in instances]): + limit = self.args.limit + if self.args.reasons_csv: + limit = None + if self.args.min_censures and self.args.min_censures != 1: + limit = None + for c_instance in database.get_all_censured_instances_by_censuring_id( + censuring_ids = [instance.id for instance in instances], + page=self.args.page, + limit=limit, + ): censures = database.get_all_censure_reasons_for_censured_id(c_instance.id, [instance.id for instance in instances]) censure_count = len(censures) censures = [c for c in censures if c.reason is not None] diff --git a/fediseer/apis/v1/endorsements.py b/fediseer/apis/v1/endorsements.py index 35e4814..c5ac418 100644 --- a/fediseer/apis/v1/endorsements.py +++ b/fediseer/apis/v1/endorsements.py @@ -13,8 +13,8 @@ class Approvals(Resource): get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args") get_parser.add_argument("min_endorsements", required=False, default=1, type=int, help="Limit to this amount of endorsements of more", location="args") get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve endorsements where their reasons include any of the text in this csv", location="args") - get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve", location="args") - get_parser.add_argument("limit", required=False, type=int, default=10, help="Which page of results to retrieve", location="args") + get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve.", location="args") + get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which amount of results to retrieve.", location="args") decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)] @api.expect(get_parser) @@ -28,13 +28,17 @@ class Approvals(Resource): You can pass a comma-separated list of domain names and the results will be a set of all their endorsements together. ''' - domains_list = domains_csv.split(',') self.args = self.get_parser.parse_args() + # if self.args.limit > 100: # Once limit is in effect + # raise e.BadRequest("limit cannot be more than 100") + if self.args.limit < 10: + raise e.BadRequest("Limit cannot be less than 10") get_instance = None if self.args.apikey: get_instance = database.find_instance_by_api_key(self.args.apikey) if not get_instance: raise e.Unauthorized(f"No Instance found matching provided API key. Please ensure you've typed it correctly") + domains_list = domains_csv.split(',') precheck_instances = database.find_multiple_instance_by_domains(domains_list) if not precheck_instances: raise e.NotFound(f"No Instances found matching any of the provided domains. Have you remembered to register them?") @@ -53,8 +57,14 @@ class Approvals(Resource): instances.append(p_instance) if len(instances) == 0: raise e.Forbidden(f"You do not have access to see these endorsements") + if self.args.min_endorsements > len(instances): + raise e.BadRequest(f"You cannot request more censures than the amount of reference domains") instance_details = [] - for e_instance in database.get_all_endorsed_instances_by_approving_id([instance.id for instance in instances]): + for e_instance in database.get_all_endorsed_instances_by_approving_id( + approving_ids=[instance.id for instance in instances], + page=self.args.page, + limit=self.args.limit, + ): endorsements = database.get_all_endorsement_reasons_for_endorsed_id(e_instance.id, [instance.id for instance in instances]) endorsement_count = len(endorsements) endorsements = [e for e in endorsements if e.reason is not None] diff --git a/fediseer/apis/v1/hesitations.py b/fediseer/apis/v1/hesitations.py index 9aa5cc4..e832590 100644 --- a/fediseer/apis/v1/hesitations.py +++ b/fediseer/apis/v1/hesitations.py @@ -13,6 +13,8 @@ class HesitationsGiven(Resource): get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args") get_parser.add_argument("min_hesitations", required=False, default=1, type=int, help="Limit to this amount of hesitations of more", location="args") get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve hesitations where their reasons include any of the text in this csv", location="args") + get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve. Only unfiltered results will be paginated.", location="args") + get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which amount of results to retrieve. Only unfiltered results will be limited.", location="args") decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)] @api.expect(get_parser) @@ -27,6 +29,10 @@ class HesitationsGiven(Resource): and the results will be a set of all their hesitations together. ''' self.args = self.get_parser.parse_args() + # if self.args.limit > 100: # Once limit is in effect + # raise e.BadRequest("limit cannot be more than 100") + if self.args.limit < 10: + raise e.BadRequest("Limit cannot be less than 10") get_instance = None if self.args.apikey: get_instance = database.find_instance_by_api_key(self.args.apikey) @@ -54,7 +60,16 @@ class HesitationsGiven(Resource): if self.args.min_hesitations > len(instances): raise e.BadRequest(f"You cannot request more hesitations than the amount of reference domains") instance_details = [] - for c_instance in database.get_all_dubious_instances_by_hesitant_id([instance.id for instance in instances]): + limit = self.args.limit + if self.args.reasons_csv: + limit = None + if self.args.min_hesitations and self.args.min_hesitations != 1: + limit = None + for c_instance in database.get_all_dubious_instances_by_hesitant_id( + hesitant_ids=[instance.id for instance in instances], + page=self.args.page, + limit=limit, + ): hesitations = database.get_all_hesitation_reasons_for_dubious_id(c_instance.id, [instance.id for instance in instances]) hesitation_count = len(hesitations) hesitations = [c for c in hesitations if c.reason is not None] diff --git a/fediseer/apis/v1/whitelist.py b/fediseer/apis/v1/whitelist.py index 0ea7bd6..b9d6b1a 100644 --- a/fediseer/apis/v1/whitelist.py +++ b/fediseer/apis/v1/whitelist.py @@ -24,8 +24,8 @@ class Whitelist(Resource): '''A List with the details of all instances and their endorsements ''' self.args = self.get_parser.parse_args() - # if self.args.limit > 100: # Once limit is in effect - # raise e.BadRequest("limit cannot be more than 100") + if self.args.limit > 100: # Once limit is in effect + raise e.BadRequest("limit cannot be more than 100") if self.args.limit < 10: raise e.BadRequest("Limit cannot be less than 10") tags = None diff --git a/fediseer/database/functions.py b/fediseer/database/functions.py index fd03232..abab9cb 100644 --- a/fediseer/database/functions.py +++ b/fediseer/database/functions.py @@ -46,7 +46,7 @@ def get_all_instances( page = 0 return query.order_by(Instance.created.desc()).offset(limit * page).limit(limit).all() -def get_all_endorsed_instances_by_approving_id(approving_ids): +def get_all_endorsed_instances_by_approving_id(approving_ids,page=1,limit=100): query = db.session.query( Instance ).outerjoin( @@ -58,7 +58,13 @@ def get_all_endorsed_instances_by_approving_id(approving_ids): ).group_by( Instance.id ) - return query.all() + if limit is not None: + page -= 1 + if page < 0: + page = 0 + return query.offset(limit * page).limit(limit).all() + else: + return query.all() def get_all_approving_instances_by_endorsed_id(endorsed_id): query = db.session.query( @@ -86,7 +92,7 @@ def get_all_endorsement_reasons_for_endorsed_id(endorsed_id, approving_ids): return query.all() -def get_all_censured_instances_by_censuring_id(censuring_ids): +def get_all_censured_instances_by_censuring_id(censuring_ids,page=1,limit=100): query = db.session.query( Instance ).outerjoin( @@ -98,7 +104,13 @@ def get_all_censured_instances_by_censuring_id(censuring_ids): ).group_by( Instance.id ) - return query.all() + if limit is not None: + page -= 1 + if page < 0: + page = 0 + return query.offset(limit * page).limit(limit).all() + else: + return query.all() def get_all_censuring_instances_by_censured_id(censured_id): query = db.session.query( @@ -127,7 +139,7 @@ def get_all_censure_reasons_for_censured_id(censured_id, censuring_ids): return query.all() -def get_all_dubious_instances_by_hesitant_id(hesitant_ids): +def get_all_dubious_instances_by_hesitant_id(hesitant_ids,page=1,limit=100): query = db.session.query( Instance ).outerjoin( @@ -139,7 +151,13 @@ def get_all_dubious_instances_by_hesitant_id(hesitant_ids): ).group_by( Instance.id ) - return query.all() + if limit is not None: + page -= 1 + if page < 0: + page = 0 + return query.offset(limit * page).limit(limit).all() + else: + return query.all() def get_all_hesitant_instances_by_dubious_id(dubious_id): query = db.session.query(