feat: Add total amount returned in API responses

closes #57

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+# 0.23.0
+
+* Results will return the total count in the DB to help with pagination
+* Fixed paginated results not always returning all results
+* fixed DECOMMISSIONED typo
+
 # 0.21.1
 
 Allow to display endorsement badge without text

fediseer/__init__.py

@@ -25,3 +25,7 @@ def after_request(response):
         etag = "Runtime Error"
     response.headers["ETag"] = etag
     return response
+
+if args.test:
+    import fediseer.sandbox
+

fediseer/apis/models/v1.py

@@ -58,6 +58,7 @@ class Models:
             'instances': fields.List(fields.Nested(self.response_model_instances_visibility)),
             'domains': fields.List(fields.String(description="The instance domains as a list.")),
             'csv': fields.String(description="The instance domains as a csv."),
+            'total': fields.Integer(description="The total amount of results in the database for this query. Use this to know the amount of pages"),
         })
         self.response_model_instances_censured = api.inherit('CensuredInstanceDetails', self.response_model_instances, {
             'censure_reasons': fields.List(fields.String(description="The reasons instances have given for censuring this instance")),
@@ -69,6 +70,7 @@ class Models:
             'instances': fields.List(fields.Nested(self.response_model_instances_censured)),
             'domains': fields.List(fields.String(description="The instance domains as a list.")),
             'csv': fields.String(description="The instance domains as a csv."),
+            'total': fields.Integer(description="The total amount of results in the database for this query. Use this to know the amount of pages"),
         })
         self.response_model_instances_endorsed = api.inherit('EndorsedInstanceDetails', self.response_model_instances, {
             'endorsement_reasons': fields.List(fields.String(description="The reasons instances have given for endorsing this instance")),
@@ -77,6 +79,7 @@ class Models:
             'instances': fields.List(fields.Nested(self.response_model_instances_endorsed)),
             'domains': fields.List(fields.String(description="The instance domains as a list.")),
             'csv': fields.String(description="The instance domains as a csv."),
+            'total': fields.Integer(description="The total amount of results in the database for this query. Use this to know the amount of pages"),
         })
         self.response_model_dubious_instances = api.inherit('DubiousInstanceDetails', self.response_model_instances, {
             'hesitation_reasons': fields.List(fields.String(description="The reasons instances have given for hesitating against this instance")),
@@ -88,6 +91,7 @@ class Models:
             'instances': fields.List(fields.Nested(self.response_model_dubious_instances)),
             'domains': fields.List(fields.String(description="The instance domains as a list.")),
             'csv': fields.String(description="The instance domains as a csv."),
+            'total': fields.Integer(description="The total amount of results in the database for this query. Use this to know the amount of pages"),
         })
         self.input_endorsements_modify = api.model('ModifyEndorsements', {
             'reason': fields.String(required=False, description="The reason for this endorsement. No profanity or hate speech allowed!", example="I just think they're neat."),

fediseer/apis/v1/censures.py

@@ -115,11 +115,20 @@ class CensuresGiven(Resource):
             c_instance_details["censure_count"] = censure_count
             instance_details.append(c_instance_details)
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": len(censures)
+            },200
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": len(censures)
+            },200
+        return {
+            "instances": instance_details,
+            "total": len(censures)
+        },200
+            
 
 class Censures(Resource):
     get_parser = reqparse.RequestParser()
@@ -172,10 +181,19 @@ class Censures(Resource):
                     c_instance_details["rebuttal"] = rebuttals
             instance_details.append(c_instance_details)
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": len(censures)
+            },200               
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": len(censures)
+            },200
+        return {
+            "instances": instance_details,
+            "total": len(censures)
+        },200
 
     decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)]
     put_parser = reqparse.RequestParser()

fediseer/apis/v1/endorsements.py

@@ -92,10 +92,19 @@ class Approvals(Resource):
             e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in r_endorsements]
             instance_details.append(e_instance_details)
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": len(endorsements)
+            },200
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": len(endorsements)
+            },200
+        return {
+            "instances": instance_details,
+            "total": len(endorsements)
+        },200
 
 class Endorsements(Resource):
     get_parser = reqparse.RequestParser()
@@ -144,10 +153,20 @@ class Endorsements(Resource):
                 e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in endorsements]
             instance_details.append(e_instance_details)
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": len(instances)
+            },200
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": len(instances)
+            },200
+        return {
+            "instances": instance_details,
+            "total": len(instances)
+        },200
+
 
     decorators = [limiter.limit("20/minute", key_func = get_request_path)]
     put_parser = reqparse.RequestParser()

fediseer/apis/v1/hesitations.py

@@ -99,11 +99,19 @@ class HesitationsGiven(Resource):
             c_instance_details["hesitation_count"] = hesitation_count
             instance_details.append(c_instance_details)
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": len(hesitations)
+            },200
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": len(hesitations)
+            },200       
+        return {
+            "instances": instance_details,
+            "total": len(hesitations)
+        },200
 
 class Hesitations(Resource):
     get_parser = reqparse.RequestParser()
@@ -155,10 +163,19 @@ class Hesitations(Resource):
                 c_instance_details["rebuttal"] = [r.rebuttal for r in rebuttals if r.target_id == c_instance.id]
             instance_details.append(c_instance_details)
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": len(hesitations)
+            },200
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": len(hesitations)
+            },200
+        return {
+            "instances": instance_details,
+            "total": len(hesitations)
+        },200
 
     decorators = [limiter.limit("20/minute", key_func = get_request_path)]
     put_parser = reqparse.RequestParser()

fediseer/apis/v1/whitelist.py

@@ -36,20 +36,30 @@ class Whitelist(Resource):
         if self.args.software_csv is not None:
             software = [s.strip() for s in self.args.software_csv.split(',')]
         instance_details = []
-        for instance in database.get_all_instances(
+        all_instances = database.get_all_instances(
             min_endorsements=self.args.endorsements,
             min_guarantors=self.args.guarantors,
             tags=tags,
             software=software,
             page=self.args.page,
             limit=self.args.limit,
-        ):
+        )
+        for instance in all_instances:
             instance_details.append(instance.get_details(show_visibilities=True))
         if self.args.csv:
-            return {"csv": ",".join([instance["domain"] for instance in instance_details])},200
+            return {
+                "csv": ",".join([instance["domain"] for instance in instance_details]),
+                "total": database.count_all_instances()
+            },200
         if self.args.domains:
-            return {"domains": [instance["domain"] for instance in instance_details]},200
-        return {"instances": instance_details},200
+            return {
+                "domains": [instance["domain"] for instance in instance_details],
+                "total": database.count_all_instances()
+            },200
+        return {
+            "instances": instance_details,
+            "total": database.count_all_instances()
+        },200
 
 class WhitelistDomain(Resource):
     get_parser = reqparse.RequestParser()

fediseer/classes/instance.py

@@ -241,7 +241,7 @@ class Instance(db.Model):
             return enums.InstanceState.UNREACHABLE
         if self.poll_failures <= 30*POLLS_PER_DAY:
             return enums.InstanceState.OFFLINE
-        return enums.InstanceState.DECIMMISSIONED
+        return enums.InstanceState.DECOMMISSIONED
         
     def has_flag(self, flag_enum):
         for flag in self.flags:

fediseer/consts.py

@@ -1,4 +1,4 @@
-FEDISEER_VERSION = "0.22.1"
+FEDISEER_VERSION = "0.23.0"
 SUPPORTED_SOFTWARE = {
     "lemmy",
     "mastodon",

fediseer/database/functions.py

@@ -9,14 +9,12 @@ from fediseer.classes.user import Claim, User
 from fediseer.classes.reports import Report
 from fediseer import enums
 
-def get_all_instances(
+def get_all_instance_query(
         min_endorsements = 0, 
         min_guarantors = 1, 
         tags = None,
         software = None,
         include_decommissioned = True,
-        page=1,
-        limit=10,
     ):
     query = db.session.query(
         Instance
@@ -48,11 +46,45 @@ def get_all_instances(
     if software:
         lower_sw = [sw.lower() for sw in software]
         query = query.filter(Instance.software.in_(lower_sw))
+    return query
+
+def get_all_instances(
+        min_endorsements = 0, 
+        min_guarantors = 1, 
+        tags = None,
+        software = None,
+        include_decommissioned = True,
+        page=1,
+        limit=10,
+    ):
+    query = get_all_instance_query(
+        min_endorsements = min_endorsements, 
+        min_guarantors = min_guarantors, 
+        tags = tags,
+        software = software,
+        include_decommissioned = include_decommissioned,
+    )    
     page -= 1
     if page < 0:
         page = 0
     return query.order_by(Instance.created.desc()).offset(limit * page).limit(limit).all()
 
+def count_all_instances(
+        min_endorsements = 0, 
+        min_guarantors = 1, 
+        tags = None,
+        software = None,
+        include_decommissioned = True,
+    ):
+    query = get_all_instance_query(
+        min_endorsements = min_endorsements, 
+        min_guarantors = min_guarantors, 
+        tags = tags,
+        software = software,
+        include_decommissioned = include_decommissioned,
+    )    
+    return query.count()
+
 def query_all_endorsed_instances_by_approving_id(approving_ids):
     return db.session.query(
         Instance
@@ -76,7 +108,7 @@ def get_all_endorsed_instances_by_approving_id(approving_ids,page=1,limit=100):
         page -= 1
         if page < 0:
             page = 0
-        return query.offset(limit * page).limit(limit).all()
+        return query.order_by(Instance.domain).offset(limit * page).limit(limit).all()
     else:
         return query.all()
 
@@ -141,7 +173,7 @@ def get_all_censured_instances_by_censuring_id(censuring_ids,page=1,limit=100):
         page -= 1
         if page < 0:
             page = 0
-        return query.offset(limit * page).limit(limit).all()
+        return query.order_by(Instance.domain).offset(limit * page).limit(limit).all()
     else:
         return query.all()
 
@@ -204,7 +236,7 @@ def get_all_dubious_instances_by_hesitant_id(hesitant_ids,page=1,limit=100):
         page -= 1
         if page < 0:
             page = 0
-        return query.offset(limit * page).limit(limit).all()
+        return query.order_by(Instance.domain).offset(limit * page).limit(limit).all()
     else:
         return query.all()
 

fediseer/enums.py

@@ -28,7 +28,7 @@ class InstanceState(enum.Enum):
     UP = 0
     UNREACHABLE = 1
     OFFLINE = 2
-    DECIMMISSIONED = 3
+    DECOMMISSIONED = 3
 
 class InstanceFlags(enum.Enum):
     RESTRICTED = 0

fediseer/sandbox.py

@@ -0,0 +1,12 @@
+from fediseer.flask import OVERSEER
+from fediseer.database import functions as database
+from fediseer.logger import logger
+import sys
+
+# Limits test
+# with OVERSEER.app_context():
+#     domains_list = ["lemmy.dbzer0.com","lemmy.ca","lemmy.basedcount.com","sh.itjust.works","aussie.zone","literature.cafe","lemmy.blahaj.zone","feddit.de","lemmy.myserv.one","programming.dev","lemmy.world","lemy.lol","lemm.ee","lemmy.team","infosec.pub","discuss.tchncs.de","lemmy.zip","phpc.social","discuss.online","lemmy.magnor.ovh","lemmynsfw.com","lemmyf.uk","lemmy.cwagner.me","startrek.website","lemdro.id"]
+#     instances = database.find_multiple_instance_by_domains(domains_list)
+#     censures = database.get_all_censured_instances_by_censuring_id(
+#         censuring_ids = [instance.id for instance in instances], limit=1000)
+sys.exit()