From cd6375cd2c1af1b240bb074d2df419b78c60623f Mon Sep 17 00:00:00 2001 From: db0 Date: Sun, 15 Oct 2023 20:25:37 +0200 Subject: [PATCH] fix: massive speed up retrieval of _given --- fediseer/apis/v1/censures.py | 53 +++++++++++++++++--------------- fediseer/apis/v1/endorsements.py | 16 ++++++---- fediseer/apis/v1/hesitations.py | 18 ++++++----- fediseer/classes/instance.py | 1 - fediseer/database/functions.py | 33 ++++++++++++++++++++ 5 files changed, 83 insertions(+), 38 deletions(-) diff --git a/fediseer/apis/v1/censures.py b/fediseer/apis/v1/censures.py index 83726b0..def1e29 100644 --- a/fediseer/apis/v1/censures.py +++ b/fediseer/apis/v1/censures.py @@ -65,37 +65,42 @@ class CensuresGiven(Resource): limit = None if self.args.min_censures and self.args.min_censures != 1: limit = None + censures = database.get_all_censures_from_censuring_id([instance.id for instance in instances]) + if self.args.reasons_csv: + reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')] + reasons_filter = set(reasons_filter) + if "__all_pedos__" in reasons_filter: + reasons_filter.add("csam") + reasons_filter.add("loli") + reasons_filter.add("shota") + reasons_filter.add("pedophil") + if "__all_bigots__" in reasons_filter: + reasons_filter.add("racism") + reasons_filter.add("sexism") + reasons_filter.add("transphobia") + reasons_filter.add("homophobia") + reasons_filter.add("islamophobia") + reasons_filter.add("nazi") + reasons_filter.add("fascist") + reasons_filter.add("hate speech") + reasons_filter.add("bigotry") for c_instance in database.get_all_censured_instances_by_censuring_id( censuring_ids = [instance.id for instance in instances], page=self.args.page, limit=limit, ): - censures = database.get_all_censure_reasons_for_censured_id(c_instance.id, [instance.id for instance in instances]) - censure_count = len(censures) - censures = [c for c in censures if c.reason is not None] - c_instance_details = c_instance.get_details() + c_censures = [c for c in censures if c.censured_id == c_instance.id] + censure_count = len(c_censures) + r_censures = [c for c in c_censures if c.reason is not None] + if self.args.csv or self.args.domains: + c_instance_details = {"domain": c_instance.domain} + else: + c_instance_details = c_instance.get_details() skip_instance = True if self.args.reasons_csv: - reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')] - reasons_filter = set(reasons_filter) - if "__all_pedos__" in reasons_filter: - reasons_filter.add("csam") - reasons_filter.add("loli") - reasons_filter.add("shota") - reasons_filter.add("pedophil") - if "__all_bigots__" in reasons_filter: - reasons_filter.add("racism") - reasons_filter.add("sexism") - reasons_filter.add("transphobia") - reasons_filter.add("homophobia") - reasons_filter.add("islamophobia") - reasons_filter.add("nazi") - reasons_filter.add("fascist") - reasons_filter.add("hate speech") - reasons_filter.add("bigotry") for r in reasons_filter: reason_filter_counter = 0 - for censure in censures: + for censure in r_censures: if r in censure.reason.lower(): reason_filter_counter += 1 if reason_filter_counter >= self.args.min_censures: @@ -105,8 +110,8 @@ class CensuresGiven(Resource): skip_instance = False if skip_instance: continue - c_instance_details["censure_reasons"] = [censure.reason for censure in censures] - c_instance_details["censure_evidence"] = [censure.evidence for censure in censures if censure.evidence is not None] + c_instance_details["censure_reasons"] = [censure.reason for censure in r_censures] + c_instance_details["censure_evidence"] = [censure.evidence for censure in r_censures if censure.evidence is not None] c_instance_details["censure_count"] = censure_count instance_details.append(c_instance_details) if self.args.csv: diff --git a/fediseer/apis/v1/endorsements.py b/fediseer/apis/v1/endorsements.py index d70162a..d86efea 100644 --- a/fediseer/apis/v1/endorsements.py +++ b/fediseer/apis/v1/endorsements.py @@ -60,22 +60,26 @@ class Approvals(Resource): if self.args.min_endorsements > len(instances): raise e.BadRequest(f"You cannot request more endorsements than the amount of reference domains") instance_details = [] + endorsements = database.get_all_endorsements_from_approving_id([instance.id for instance in instances]) for e_instance in database.get_all_endorsed_instances_by_approving_id( approving_ids=[instance.id for instance in instances], page=self.args.page, limit=self.args.limit, ): - endorsements = database.get_all_endorsement_reasons_for_endorsed_id(e_instance.id, [instance.id for instance in instances]) - endorsement_count = len(endorsements) - endorsements = [e for e in endorsements if e.reason is not None] - e_instance_details = e_instance.get_details() + e_endorsements = [e for e in endorsements if e.endorsed_id == e_instance.id] + endorsement_count = len(e_endorsements) + r_endorsements = [e for e in e_endorsements if e.reason is not None] + if self.args.csv or self.args.domains: + e_instance_details = {"domain": e_instance.domain} + else: + e_instance_details = e_instance.get_details() skip_instance = True if self.args.reasons_csv: reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')] reasons_filter = set(reasons_filter) for r in reasons_filter: reason_filter_counter = 0 - for endorsement in endorsements: + for endorsement in r_endorsements: if r in endorsement.reason.lower(): reason_filter_counter += 1 if reason_filter_counter >= self.args.min_endorsements: @@ -85,7 +89,7 @@ class Approvals(Resource): skip_instance = False if skip_instance: continue - e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in endorsements] + e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in r_endorsements] instance_details.append(e_instance_details) if self.args.csv: return {"csv": ",".join([instance["domain"] for instance in instance_details])},200 diff --git a/fediseer/apis/v1/hesitations.py b/fediseer/apis/v1/hesitations.py index 827f550..6d3b1bf 100644 --- a/fediseer/apis/v1/hesitations.py +++ b/fediseer/apis/v1/hesitations.py @@ -65,22 +65,26 @@ class HesitationsGiven(Resource): limit = None if self.args.min_hesitations and self.args.min_hesitations != 1: limit = None + hesitations = database.get_all_hesitations_from_hesitant_id([instance.id for instance in instances]) for c_instance in database.get_all_dubious_instances_by_hesitant_id( hesitant_ids=[instance.id for instance in instances], page=self.args.page, limit=limit, ): - hesitations = database.get_all_hesitation_reasons_for_dubious_id(c_instance.id, [instance.id for instance in instances]) - hesitation_count = len(hesitations) - hesitations = [c for c in hesitations if c.reason is not None] - c_instance_details = c_instance.get_details() + h_hesitations = [c for c in hesitations if c.dubious_id == c_instance.id] + hesitation_count = len(h_hesitations) + r_hesitations = [c for c in h_hesitations if c.reason is not None] + if self.args.csv or self.args.domains: + c_instance_details = {"domain": c_instance.domain} + else: + c_instance_details = c_instance.get_details() skip_instance = True if self.args.reasons_csv: reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')] reasons_filter = set(reasons_filter) for r in reasons_filter: reason_filter_counter = 0 - for hesitation in hesitations: + for hesitation in r_hesitations: if r in hesitation.reason.lower(): reason_filter_counter += 1 if reason_filter_counter >= self.args.min_hesitations: @@ -90,8 +94,8 @@ class HesitationsGiven(Resource): skip_instance = False if skip_instance: continue - c_instance_details["hesitation_reasons"] = [hesitation.reason for hesitation in hesitations] - c_instance_details["hesitation_evidence"] = [hesitation.evidence for hesitation in hesitations if hesitation.evidence is not None] + c_instance_details["hesitation_reasons"] = [hesitation.reason for hesitation in r_hesitations] + c_instance_details["hesitation_evidence"] = [hesitation.evidence for hesitation in r_hesitations if hesitation.evidence is not None] c_instance_details["hesitation_count"] = hesitation_count instance_details.append(c_instance_details) if self.args.csv: diff --git a/fediseer/classes/instance.py b/fediseer/classes/instance.py index 39b26b7..f79b3a8 100644 --- a/fediseer/classes/instance.py +++ b/fediseer/classes/instance.py @@ -209,7 +209,6 @@ class Instance(db.Model): if not guarantee: return None return guarantee.guarantor_instance - return Instance.query.filter_by(id=guarantee.guarantor_id).first() def get_guarantor_domain(self): guarantor = self.get_guarantor() diff --git a/fediseer/database/functions.py b/fediseer/database/functions.py index 784c106..6ad39a9 100644 --- a/fediseer/database/functions.py +++ b/fediseer/database/functions.py @@ -26,6 +26,9 @@ def get_all_instances( ).options( joinedload(Instance.guarantors), joinedload(Instance.endorsements), + joinedload(Instance.admins), + joinedload(Instance.guarantors), + joinedload(Instance.tags) ).group_by( Instance.id ).filter( @@ -84,6 +87,9 @@ def get_all_approving_instances_by_endorsed_id(endorsed_id): Instance.approvals, ).options( joinedload(Instance.approvals), + joinedload(Instance.endorsements), + joinedload(Instance.admins), + joinedload(Instance.guarantors) ).filter( Endorsement.endorsed_id == endorsed_id ).group_by( @@ -100,6 +106,13 @@ def get_all_endorsement_reasons_for_endorsed_id(endorsed_id, approving_ids): ) return query.all() +def get_all_endorsements_from_approving_id(approving_ids): + query = Endorsement.query.filter( + Endorsement.approving_ids.in_(approving_ids) + ) + return query.all() + + def query_all_censured_instances_by_censuring_id(censuring_ids): return db.session.query( @@ -108,6 +121,10 @@ def query_all_censured_instances_by_censuring_id(censuring_ids): Instance.censures_received, ).options( joinedload(Instance.censures_received), + joinedload(Instance.endorsements), + joinedload(Instance.approvals), + joinedload(Instance.admins), + joinedload(Instance.guarantors) ).filter( Censure.censuring_id.in_(censuring_ids) ).group_by( @@ -153,6 +170,12 @@ def get_all_censure_reasons_for_censured_id(censured_id, censuring_ids): ) return query.all() +def get_all_censures_from_censuring_id(censuring_ids): + query = Censure.query.filter( + Censure.censuring_id.in_(censuring_ids) + ) + return query.all() + def query_all_dubious_instances_by_hesitant_id(hesitant_ids): return db.session.query( @@ -161,6 +184,10 @@ def query_all_dubious_instances_by_hesitant_id(hesitant_ids): Instance.hesitations_received, ).options( joinedload(Instance.hesitations_received), + joinedload(Instance.endorsements), + joinedload(Instance.approvals), + joinedload(Instance.admins), + joinedload(Instance.guarantors) ).filter( Hesitation.hesitant_id.in_(hesitant_ids) ).group_by( @@ -206,6 +233,12 @@ def get_all_hesitation_reasons_for_dubious_id(dubious_id, hesitant_ids): ) return query.all() +def get_all_hesitations_from_hesitant_id(hesitant_ids): + query = Hesitation.query.filter( + Hesitation.hesitant_id.in_(hesitant_ids) + ) + return query.all() + def get_rebuttal(target_instance_id, source_instance_id): query = Rebuttal.query.filter_by( target_id=target_instance_id,