fix: massive speed up retrieval of _given

fediseer/apis/v1/censures.py

@@ -65,37 +65,42 @@ class CensuresGiven(Resource):
             limit = None
         if self.args.min_censures and self.args.min_censures != 1:
             limit = None
+        censures = database.get_all_censures_from_censuring_id([instance.id for instance in instances])
+        if self.args.reasons_csv:
+            reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
+            reasons_filter = set(reasons_filter)
+            if "__all_pedos__" in reasons_filter:
+                reasons_filter.add("csam")
+                reasons_filter.add("loli")
+                reasons_filter.add("shota")
+                reasons_filter.add("pedophil")
+            if "__all_bigots__" in reasons_filter:
+                reasons_filter.add("racism")
+                reasons_filter.add("sexism")
+                reasons_filter.add("transphobia")
+                reasons_filter.add("homophobia")
+                reasons_filter.add("islamophobia")
+                reasons_filter.add("nazi")
+                reasons_filter.add("fascist")
+                reasons_filter.add("hate speech")
+                reasons_filter.add("bigotry")
         for c_instance in database.get_all_censured_instances_by_censuring_id(
             censuring_ids = [instance.id for instance in instances],
             page=self.args.page,
             limit=limit,
         ):
-            censures = database.get_all_censure_reasons_for_censured_id(c_instance.id, [instance.id for instance in instances])
+            c_censures = [c for c in censures if c.censured_id == c_instance.id]
-            censure_count = len(censures)
+            censure_count = len(c_censures)
-            censures = [c for c in censures if c.reason is not None]
+            r_censures = [c for c in c_censures if c.reason is not None]
-            c_instance_details = c_instance.get_details()
+            if self.args.csv or self.args.domains:
+                c_instance_details = {"domain": c_instance.domain}
+            else:
+                c_instance_details = c_instance.get_details()
             skip_instance = True
             if self.args.reasons_csv:
-                reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
-                reasons_filter = set(reasons_filter)
-                if "__all_pedos__" in reasons_filter:
-                    reasons_filter.add("csam")
-                    reasons_filter.add("loli")
-                    reasons_filter.add("shota")
-                    reasons_filter.add("pedophil")
-                if "__all_bigots__" in reasons_filter:
-                    reasons_filter.add("racism")
-                    reasons_filter.add("sexism")
-                    reasons_filter.add("transphobia")
-                    reasons_filter.add("homophobia")
-                    reasons_filter.add("islamophobia")
-                    reasons_filter.add("nazi")
-                    reasons_filter.add("fascist")
-                    reasons_filter.add("hate speech")
-                    reasons_filter.add("bigotry")
                 for r in reasons_filter:
                     reason_filter_counter = 0
-                    for censure in censures:
+                    for censure in r_censures:
                         if r in censure.reason.lower():
                             reason_filter_counter += 1
                     if reason_filter_counter >= self.args.min_censures:
@@ -105,8 +110,8 @@ class CensuresGiven(Resource):
                 skip_instance = False
             if skip_instance:
                 continue
-            c_instance_details["censure_reasons"] = [censure.reason for censure in censures]
+            c_instance_details["censure_reasons"] = [censure.reason for censure in r_censures]
-            c_instance_details["censure_evidence"] = [censure.evidence for censure in censures if censure.evidence is not None]
+            c_instance_details["censure_evidence"] = [censure.evidence for censure in r_censures if censure.evidence is not None]
             c_instance_details["censure_count"] = censure_count
             instance_details.append(c_instance_details)
         if self.args.csv:

fediseer/apis/v1/endorsements.py

@@ -60,22 +60,26 @@ class Approvals(Resource):
         if self.args.min_endorsements > len(instances):
             raise e.BadRequest(f"You cannot request more endorsements than the amount of reference domains")
         instance_details = []
+        endorsements = database.get_all_endorsements_from_approving_id([instance.id for instance in instances])
         for e_instance in database.get_all_endorsed_instances_by_approving_id(
             approving_ids=[instance.id for instance in instances],
             page=self.args.page,
             limit=self.args.limit,
         ):
-            endorsements = database.get_all_endorsement_reasons_for_endorsed_id(e_instance.id, [instance.id for instance in instances])
+            e_endorsements = [e for e in endorsements if e.endorsed_id == e_instance.id]
-            endorsement_count = len(endorsements)
+            endorsement_count = len(e_endorsements)
-            endorsements = [e for e in endorsements if e.reason is not None]
+            r_endorsements = [e for e in e_endorsements if e.reason is not None]
-            e_instance_details = e_instance.get_details()
+            if self.args.csv or self.args.domains:
+                e_instance_details = {"domain": e_instance.domain}
+            else:
+                e_instance_details = e_instance.get_details()
             skip_instance = True
             if self.args.reasons_csv:
                 reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
                 reasons_filter = set(reasons_filter)
                 for r in reasons_filter:
                     reason_filter_counter = 0
-                    for endorsement in endorsements:
+                    for endorsement in r_endorsements:
                         if r in endorsement.reason.lower():
                             reason_filter_counter += 1
                     if reason_filter_counter >= self.args.min_endorsements:
@@ -85,7 +89,7 @@ class Approvals(Resource):
                 skip_instance = False
             if skip_instance:
                 continue
-            e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in endorsements]
+            e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in r_endorsements]
             instance_details.append(e_instance_details)
         if self.args.csv:
             return {"csv": ",".join([instance["domain"] for instance in instance_details])},200

fediseer/apis/v1/hesitations.py

@@ -65,22 +65,26 @@ class HesitationsGiven(Resource):
             limit = None
         if self.args.min_hesitations and self.args.min_hesitations != 1:
             limit = None
+        hesitations = database.get_all_hesitations_from_hesitant_id([instance.id for instance in instances])
         for c_instance in database.get_all_dubious_instances_by_hesitant_id(
             hesitant_ids=[instance.id for instance in instances],
             page=self.args.page,
             limit=limit,
         ):
-            hesitations = database.get_all_hesitation_reasons_for_dubious_id(c_instance.id, [instance.id for instance in instances])
+            h_hesitations = [c for c in hesitations if c.dubious_id == c_instance.id]
-            hesitation_count = len(hesitations)
+            hesitation_count = len(h_hesitations)
-            hesitations = [c for c in hesitations if c.reason is not None]
+            r_hesitations = [c for c in h_hesitations if c.reason is not None]
-            c_instance_details = c_instance.get_details()
+            if self.args.csv or self.args.domains:
+                c_instance_details = {"domain": c_instance.domain}
+            else:
+                c_instance_details = c_instance.get_details()
             skip_instance = True
             if self.args.reasons_csv:
                 reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
                 reasons_filter = set(reasons_filter)
                 for r in reasons_filter:
                     reason_filter_counter = 0
-                    for hesitation in hesitations:
+                    for hesitation in r_hesitations:
                         if r in hesitation.reason.lower():
                             reason_filter_counter += 1
                     if reason_filter_counter >= self.args.min_hesitations:
@@ -90,8 +94,8 @@ class HesitationsGiven(Resource):
                 skip_instance = False
             if skip_instance:
                 continue
-            c_instance_details["hesitation_reasons"] = [hesitation.reason for hesitation in hesitations]
+            c_instance_details["hesitation_reasons"] = [hesitation.reason for hesitation in r_hesitations]
-            c_instance_details["hesitation_evidence"] = [hesitation.evidence for hesitation in hesitations if hesitation.evidence is not None]
+            c_instance_details["hesitation_evidence"] = [hesitation.evidence for hesitation in r_hesitations if hesitation.evidence is not None]
             c_instance_details["hesitation_count"] = hesitation_count
             instance_details.append(c_instance_details)
         if self.args.csv:

fediseer/classes/instance.py

@@ -209,7 +209,6 @@ class Instance(db.Model):
         if not guarantee:
             return None
         return guarantee.guarantor_instance
-        return Instance.query.filter_by(id=guarantee.guarantor_id).first()
 
     def get_guarantor_domain(self):
         guarantor = self.get_guarantor()

fediseer/database/functions.py

@@ -26,6 +26,9 @@ def get_all_instances(
     ).options(
         joinedload(Instance.guarantors),
         joinedload(Instance.endorsements),
+        joinedload(Instance.admins),
+        joinedload(Instance.guarantors),
+        joinedload(Instance.tags)
     ).group_by(
         Instance.id
     ).filter(
@@ -84,6 +87,9 @@ def get_all_approving_instances_by_endorsed_id(endorsed_id):
         Instance.approvals,
     ).options(
         joinedload(Instance.approvals),
+        joinedload(Instance.endorsements),
+        joinedload(Instance.admins),
+        joinedload(Instance.guarantors)
     ).filter(
         Endorsement.endorsed_id == endorsed_id
     ).group_by(
@@ -100,6 +106,13 @@ def get_all_endorsement_reasons_for_endorsed_id(endorsed_id, approving_ids):
     )
     return query.all()
 
+def get_all_endorsements_from_approving_id(approving_ids):
+    query = Endorsement.query.filter(
+        Endorsement.approving_ids.in_(approving_ids)
+    )
+    return query.all()
+
+
 
 def query_all_censured_instances_by_censuring_id(censuring_ids):
     return db.session.query(
@@ -108,6 +121,10 @@ def query_all_censured_instances_by_censuring_id(censuring_ids):
         Instance.censures_received,
     ).options(
         joinedload(Instance.censures_received),
+        joinedload(Instance.endorsements),
+        joinedload(Instance.approvals),
+        joinedload(Instance.admins),
+        joinedload(Instance.guarantors)
     ).filter(
         Censure.censuring_id.in_(censuring_ids)
     ).group_by(
@@ -153,6 +170,12 @@ def get_all_censure_reasons_for_censured_id(censured_id, censuring_ids):
     )
     return query.all()
 
+def get_all_censures_from_censuring_id(censuring_ids):
+    query = Censure.query.filter(
+        Censure.censuring_id.in_(censuring_ids)
+    )
+    return query.all()
+
 
 def query_all_dubious_instances_by_hesitant_id(hesitant_ids):
     return db.session.query(
@@ -161,6 +184,10 @@ def query_all_dubious_instances_by_hesitant_id(hesitant_ids):
         Instance.hesitations_received,
     ).options(
         joinedload(Instance.hesitations_received),
+        joinedload(Instance.endorsements),
+        joinedload(Instance.approvals),
+        joinedload(Instance.admins),
+        joinedload(Instance.guarantors)
     ).filter(
         Hesitation.hesitant_id.in_(hesitant_ids)
     ).group_by(
@@ -206,6 +233,12 @@ def get_all_hesitation_reasons_for_dubious_id(dubious_id, hesitant_ids):
     )
     return query.all()
 
+def get_all_hesitations_from_hesitant_id(hesitant_ids):
+    query = Hesitation.query.filter(
+        Hesitation.hesitant_id.in_(hesitant_ids)
+    )
+    return query.all()
+
 def get_rebuttal(target_instance_id, source_instance_id):
     query = Rebuttal.query.filter_by(
         target_id=target_instance_id,