feat: paginating unfiltered results

fediseer/apis/v1/censures.py

@@ -13,6 +13,8 @@ class CensuresGiven(Resource):
     get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args")
     get_parser.add_argument("min_censures", required=False, default=1, type=int, help="Limit to this amount of censures of more", location="args")
     get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve censures where their reasons include any of the text in this csv", location="args")
+    get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve. Only unfiltered results will be paginated.", location="args")
+    get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which amount of results to retrieve. Only unfiltered results will be limited.", location="args")
 
     decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)]
     @api.expect(get_parser)
@@ -27,6 +29,10 @@ class CensuresGiven(Resource):
         and the results will be a set of all their censures together.
         '''
         self.args = self.get_parser.parse_args()
+        # if self.args.limit > 100: # Once limit is in effect
+        #     raise e.BadRequest("limit cannot be more than 100")
+        if self.args.limit < 10:
+            raise e.BadRequest("Limit cannot be less than 10")
         get_instance = None
         if self.args.apikey:
             get_instance = database.find_instance_by_api_key(self.args.apikey)
@@ -54,7 +60,16 @@ class CensuresGiven(Resource):
         if self.args.min_censures > len(instances):
             raise e.BadRequest(f"You cannot request more censures than the amount of reference domains")
         instance_details = []
-        for c_instance in database.get_all_censured_instances_by_censuring_id([instance.id for instance in instances]):
+        limit = self.args.limit
+        if self.args.reasons_csv:
+            limit = None
+        if self.args.min_censures and self.args.min_censures != 1:
+            limit = None
+        for c_instance in database.get_all_censured_instances_by_censuring_id(
+            censuring_ids = [instance.id for instance in instances],
+            page=self.args.page,
+            limit=limit,
+        ):
             censures = database.get_all_censure_reasons_for_censured_id(c_instance.id, [instance.id for instance in instances])
             censure_count = len(censures)
             censures = [c for c in censures if c.reason is not None]

fediseer/apis/v1/endorsements.py

@@ -13,8 +13,8 @@ class Approvals(Resource):
     get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args")
     get_parser.add_argument("min_endorsements", required=False, default=1, type=int, help="Limit to this amount of endorsements of more", location="args")
     get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve endorsements where their reasons include any of the text in this csv", location="args")
-    get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve", location="args")
+    get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve.", location="args")
-    get_parser.add_argument("limit", required=False, type=int, default=10, help="Which page of results to retrieve", location="args")
+    get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which amount of results to retrieve.", location="args")
 
     decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)]
     @api.expect(get_parser)
@@ -28,13 +28,17 @@ class Approvals(Resource):
         You can pass a comma-separated list of domain names and the results will be a set of all their
         endorsements together.
         '''
-        domains_list = domains_csv.split(',')
         self.args = self.get_parser.parse_args()
+        # if self.args.limit > 100: # Once limit is in effect
+        #     raise e.BadRequest("limit cannot be more than 100")
+        if self.args.limit < 10:
+            raise e.BadRequest("Limit cannot be less than 10")
         get_instance = None
         if self.args.apikey:
             get_instance = database.find_instance_by_api_key(self.args.apikey)
             if not get_instance:
                 raise e.Unauthorized(f"No Instance found matching provided API key. Please ensure you've typed it correctly")
+        domains_list = domains_csv.split(',')
         precheck_instances = database.find_multiple_instance_by_domains(domains_list)
         if not precheck_instances:
             raise e.NotFound(f"No Instances found matching any of the provided domains. Have you remembered to register them?")
@@ -53,8 +57,14 @@ class Approvals(Resource):
             instances.append(p_instance)
         if len(instances) == 0:
             raise e.Forbidden(f"You do not have access to see these endorsements")
+        if self.args.min_endorsements > len(instances):
+            raise e.BadRequest(f"You cannot request more censures than the amount of reference domains")
         instance_details = []
-        for e_instance in database.get_all_endorsed_instances_by_approving_id([instance.id for instance in instances]):
+        for e_instance in database.get_all_endorsed_instances_by_approving_id(
+            approving_ids=[instance.id for instance in instances],
+            page=self.args.page,
+            limit=self.args.limit,
+        ):
             endorsements = database.get_all_endorsement_reasons_for_endorsed_id(e_instance.id, [instance.id for instance in instances])
             endorsement_count = len(endorsements)
             endorsements = [e for e in endorsements if e.reason is not None]

fediseer/apis/v1/hesitations.py

@@ -13,6 +13,8 @@ class HesitationsGiven(Resource):
     get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args")
     get_parser.add_argument("min_hesitations", required=False, default=1, type=int, help="Limit to this amount of hesitations of more", location="args")
     get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve hesitations where their reasons include any of the text in this csv", location="args")
+    get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve. Only unfiltered results will be paginated.", location="args")
+    get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which amount of results to retrieve. Only unfiltered results will be limited.", location="args")
 
     decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)]
     @api.expect(get_parser)
@@ -27,6 +29,10 @@ class HesitationsGiven(Resource):
         and the results will be a set of all their hesitations together.
         '''
         self.args = self.get_parser.parse_args()
+        # if self.args.limit > 100: # Once limit is in effect
+        #     raise e.BadRequest("limit cannot be more than 100")
+        if self.args.limit < 10:
+            raise e.BadRequest("Limit cannot be less than 10")
         get_instance = None
         if self.args.apikey:
             get_instance = database.find_instance_by_api_key(self.args.apikey)
@@ -54,7 +60,16 @@ class HesitationsGiven(Resource):
         if self.args.min_hesitations > len(instances):
             raise e.BadRequest(f"You cannot request more hesitations than the amount of reference domains")
         instance_details = []
-        for c_instance in database.get_all_dubious_instances_by_hesitant_id([instance.id for instance in instances]):
+        limit = self.args.limit
+        if self.args.reasons_csv:
+            limit = None
+        if self.args.min_hesitations and self.args.min_hesitations != 1:
+            limit = None
+        for c_instance in database.get_all_dubious_instances_by_hesitant_id(
+            hesitant_ids=[instance.id for instance in instances],
+            page=self.args.page,
+            limit=limit,
+        ):
             hesitations = database.get_all_hesitation_reasons_for_dubious_id(c_instance.id, [instance.id for instance in instances])
             hesitation_count = len(hesitations)
             hesitations = [c for c in hesitations if c.reason is not None]

fediseer/apis/v1/whitelist.py

@@ -24,8 +24,8 @@ class Whitelist(Resource):
         '''A List with the details of all instances and their endorsements
         '''
         self.args = self.get_parser.parse_args()
-        # if self.args.limit > 100: # Once limit is in effect
+        if self.args.limit > 100: # Once limit is in effect
-        #     raise e.BadRequest("limit cannot be more than 100")
+            raise e.BadRequest("limit cannot be more than 100")
         if self.args.limit < 10:
             raise e.BadRequest("Limit cannot be less than 10")
         tags = None

fediseer/database/functions.py

@@ -46,7 +46,7 @@ def get_all_instances(
         page = 0
     return query.order_by(Instance.created.desc()).offset(limit * page).limit(limit).all()
 
-def get_all_endorsed_instances_by_approving_id(approving_ids):
+def get_all_endorsed_instances_by_approving_id(approving_ids,page=1,limit=100):
     query = db.session.query(
         Instance
     ).outerjoin(
@@ -58,7 +58,13 @@ def get_all_endorsed_instances_by_approving_id(approving_ids):
     ).group_by(
         Instance.id
     )
-    return query.all()
+    if limit is not None:
+        page -= 1
+        if page < 0:
+            page = 0
+        return query.offset(limit * page).limit(limit).all()
+    else:
+        return query.all()
 
 def get_all_approving_instances_by_endorsed_id(endorsed_id):
     query = db.session.query(
@@ -86,7 +92,7 @@ def get_all_endorsement_reasons_for_endorsed_id(endorsed_id, approving_ids):
     return query.all()
 
 
-def get_all_censured_instances_by_censuring_id(censuring_ids):
+def get_all_censured_instances_by_censuring_id(censuring_ids,page=1,limit=100):
     query = db.session.query(
         Instance
     ).outerjoin(
@@ -98,7 +104,13 @@ def get_all_censured_instances_by_censuring_id(censuring_ids):
     ).group_by(
         Instance.id
     )
-    return query.all()
+    if limit is not None:
+        page -= 1
+        if page < 0:
+            page = 0
+        return query.offset(limit * page).limit(limit).all()
+    else:
+        return query.all()
 
 def get_all_censuring_instances_by_censured_id(censured_id):
     query = db.session.query(
@@ -127,7 +139,7 @@ def get_all_censure_reasons_for_censured_id(censured_id, censuring_ids):
     return query.all()
 
 
-def get_all_dubious_instances_by_hesitant_id(hesitant_ids):
+def get_all_dubious_instances_by_hesitant_id(hesitant_ids,page=1,limit=100):
     query = db.session.query(
         Instance
     ).outerjoin(
@@ -139,7 +151,13 @@ def get_all_dubious_instances_by_hesitant_id(hesitant_ids):
     ).group_by(
         Instance.id
     )
-    return query.all()
+    if limit is not None:
+        page -= 1
+        if page < 0:
+            page = 0
+        return query.offset(limit * page).limit(limit).all()
+    else:
+        return query.all()
 
 def get_all_hesitant_instances_by_dubious_id(dubious_id):
     query = db.session.query(