gsathecat
/
fediseer
mirror of https://github.com/Fediseer/fediseer.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: massive speed up retrieval of _given

pull/56/head
db0 2023-10-15 20:25:37 +02:00
parent f0b53af3d2
commit cd6375cd2c
5 changed files with 83 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
fediseer/apis/v1/censures.py
View File

@ -65,37 +65,42 @@ class CensuresGiven(Resource):
limit = None
if self.args.min_censures and self.args.min_censures != 1:
limit = None
censures = database.get_all_censures_from_censuring_id([instance.id for instance in instances])
if self.args.reasons_csv:
reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
reasons_filter = set(reasons_filter)
if "__all_pedos__" in reasons_filter:
reasons_filter.add("csam")
reasons_filter.add("loli")
reasons_filter.add("shota")
reasons_filter.add("pedophil")
if "__all_bigots__" in reasons_filter:
reasons_filter.add("racism")
reasons_filter.add("sexism")
reasons_filter.add("transphobia")
reasons_filter.add("homophobia")
reasons_filter.add("islamophobia")
reasons_filter.add("nazi")
reasons_filter.add("fascist")
reasons_filter.add("hate speech")
reasons_filter.add("bigotry")
for c_instance in database.get_all_censured_instances_by_censuring_id(
censuring_ids = [instance.id for instance in instances],
page=self.args.page,
limit=limit,
):
censures = database.get_all_censure_reasons_for_censured_id(c_instance.id, [instance.id for instance in instances])
censure_count = len(censures)
censures = [c for c in censures if c.reason is not None]
c_instance_details = c_instance.get_details()
c_censures = [c for c in censures if c.censured_id == c_instance.id]
censure_count = len(c_censures)
r_censures = [c for c in c_censures if c.reason is not None]
if self.args.csv or self.args.domains:
c_instance_details = {"domain": c_instance.domain}
else:
c_instance_details = c_instance.get_details()
skip_instance = True
if self.args.reasons_csv:
reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
reasons_filter = set(reasons_filter)
if "__all_pedos__" in reasons_filter:
reasons_filter.add("csam")
reasons_filter.add("loli")
reasons_filter.add("shota")
reasons_filter.add("pedophil")
if "__all_bigots__" in reasons_filter:
reasons_filter.add("racism")
reasons_filter.add("sexism")
reasons_filter.add("transphobia")
reasons_filter.add("homophobia")
reasons_filter.add("islamophobia")
reasons_filter.add("nazi")
reasons_filter.add("fascist")
reasons_filter.add("hate speech")
reasons_filter.add("bigotry")
for r in reasons_filter:
reason_filter_counter = 0
for censure in censures:
for censure in r_censures:
if r in censure.reason.lower():
reason_filter_counter += 1
if reason_filter_counter >= self.args.min_censures:
@ -105,8 +110,8 @@ class CensuresGiven(Resource):
skip_instance = False
if skip_instance:
continue
c_instance_details["censure_reasons"] = [censure.reason for censure in censures]
c_instance_details["censure_evidence"] = [censure.evidence for censure in censures if censure.evidence is not None]
c_instance_details["censure_reasons"] = [censure.reason for censure in r_censures]
c_instance_details["censure_evidence"] = [censure.evidence for censure in r_censures if censure.evidence is not None]
c_instance_details["censure_count"] = censure_count
instance_details.append(c_instance_details)
if self.args.csv:

16
fediseer/apis/v1/endorsements.py
View File

@ -60,22 +60,26 @@ class Approvals(Resource):
if self.args.min_endorsements > len(instances):
raise e.BadRequest(f"You cannot request more endorsements than the amount of reference domains")
instance_details = []
endorsements = database.get_all_endorsements_from_approving_id([instance.id for instance in instances])
for e_instance in database.get_all_endorsed_instances_by_approving_id(
approving_ids=[instance.id for instance in instances],
page=self.args.page,
limit=self.args.limit,
):
endorsements = database.get_all_endorsement_reasons_for_endorsed_id(e_instance.id, [instance.id for instance in instances])
endorsement_count = len(endorsements)
endorsements = [e for e in endorsements if e.reason is not None]
e_instance_details = e_instance.get_details()
e_endorsements = [e for e in endorsements if e.endorsed_id == e_instance.id]
endorsement_count = len(e_endorsements)
r_endorsements = [e for e in e_endorsements if e.reason is not None]
if self.args.csv or self.args.domains:
e_instance_details = {"domain": e_instance.domain}
else:
e_instance_details = e_instance.get_details()
skip_instance = True
if self.args.reasons_csv:
reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
reasons_filter = set(reasons_filter)
for r in reasons_filter:
reason_filter_counter = 0
for endorsement in endorsements:
for endorsement in r_endorsements:
if r in endorsement.reason.lower():
reason_filter_counter += 1
if reason_filter_counter >= self.args.min_endorsements:
@ -85,7 +89,7 @@ class Approvals(Resource):
skip_instance = False
if skip_instance:
continue
e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in endorsements]
e_instance_details["endorsement_reasons"] = [endorsement.reason for endorsement in r_endorsements]
instance_details.append(e_instance_details)
if self.args.csv:
return {"csv": ",".join([instance["domain"] for instance in instance_details])},200

18
fediseer/apis/v1/hesitations.py
View File

@ -65,22 +65,26 @@ class HesitationsGiven(Resource):
limit = None
if self.args.min_hesitations and self.args.min_hesitations != 1:
limit = None
hesitations = database.get_all_hesitations_from_hesitant_id([instance.id for instance in instances])
for c_instance in database.get_all_dubious_instances_by_hesitant_id(
hesitant_ids=[instance.id for instance in instances],
page=self.args.page,
limit=limit,
):
hesitations = database.get_all_hesitation_reasons_for_dubious_id(c_instance.id, [instance.id for instance in instances])
hesitation_count = len(hesitations)
hesitations = [c for c in hesitations if c.reason is not None]
c_instance_details = c_instance.get_details()
h_hesitations = [c for c in hesitations if c.dubious_id == c_instance.id]
hesitation_count = len(h_hesitations)
r_hesitations = [c for c in h_hesitations if c.reason is not None]
if self.args.csv or self.args.domains:
c_instance_details = {"domain": c_instance.domain}
else:
c_instance_details = c_instance.get_details()
skip_instance = True
if self.args.reasons_csv:
reasons_filter = [r.strip().lower() for r in self.args.reasons_csv.split(',')]
reasons_filter = set(reasons_filter)
for r in reasons_filter:
reason_filter_counter = 0
for hesitation in hesitations:
for hesitation in r_hesitations:
if r in hesitation.reason.lower():
reason_filter_counter += 1
if reason_filter_counter >= self.args.min_hesitations:
@ -90,8 +94,8 @@ class HesitationsGiven(Resource):
skip_instance = False
if skip_instance:
continue
c_instance_details["hesitation_reasons"] = [hesitation.reason for hesitation in hesitations]
c_instance_details["hesitation_evidence"] = [hesitation.evidence for hesitation in hesitations if hesitation.evidence is not None]
c_instance_details["hesitation_reasons"] = [hesitation.reason for hesitation in r_hesitations]
c_instance_details["hesitation_evidence"] = [hesitation.evidence for hesitation in r_hesitations if hesitation.evidence is not None]
c_instance_details["hesitation_count"] = hesitation_count
instance_details.append(c_instance_details)
if self.args.csv:

1
fediseer/classes/instance.py
View File

@ -209,7 +209,6 @@ class Instance(db.Model):
if not guarantee:
return None
return guarantee.guarantor_instance
return Instance.query.filter_by(id=guarantee.guarantor_id).first()
def get_guarantor_domain(self):
guarantor = self.get_guarantor()

33
fediseer/database/functions.py
View File

@ -26,6 +26,9 @@ def get_all_instances(
).options(
joinedload(Instance.guarantors),
joinedload(Instance.endorsements),
joinedload(Instance.admins),
joinedload(Instance.guarantors),
joinedload(Instance.tags)
).group_by(
Instance.id
).filter(
@ -84,6 +87,9 @@ def get_all_approving_instances_by_endorsed_id(endorsed_id):
Instance.approvals,
).options(
joinedload(Instance.approvals),
joinedload(Instance.endorsements),
joinedload(Instance.admins),
joinedload(Instance.guarantors)
).filter(
Endorsement.endorsed_id == endorsed_id
).group_by(
@ -100,6 +106,13 @@ def get_all_endorsement_reasons_for_endorsed_id(endorsed_id, approving_ids):
)
return query.all()
def get_all_endorsements_from_approving_id(approving_ids):
query = Endorsement.query.filter(
Endorsement.approving_ids.in_(approving_ids)
)
return query.all()
def query_all_censured_instances_by_censuring_id(censuring_ids):
return db.session.query(
@ -108,6 +121,10 @@ def query_all_censured_instances_by_censuring_id(censuring_ids):
Instance.censures_received,
).options(
joinedload(Instance.censures_received),
joinedload(Instance.endorsements),
joinedload(Instance.approvals),
joinedload(Instance.admins),
joinedload(Instance.guarantors)
).filter(
Censure.censuring_id.in_(censuring_ids)
).group_by(
@ -153,6 +170,12 @@ def get_all_censure_reasons_for_censured_id(censured_id, censuring_ids):
)
return query.all()
def get_all_censures_from_censuring_id(censuring_ids):
query = Censure.query.filter(
Censure.censuring_id.in_(censuring_ids)
)
return query.all()
def query_all_dubious_instances_by_hesitant_id(hesitant_ids):
return db.session.query(
@ -161,6 +184,10 @@ def query_all_dubious_instances_by_hesitant_id(hesitant_ids):
Instance.hesitations_received,
).options(
joinedload(Instance.hesitations_received),
joinedload(Instance.endorsements),
joinedload(Instance.approvals),
joinedload(Instance.admins),
joinedload(Instance.guarantors)
).filter(
Hesitation.hesitant_id.in_(hesitant_ids)
).group_by(
@ -206,6 +233,12 @@ def get_all_hesitation_reasons_for_dubious_id(dubious_id, hesitant_ids):
)
return query.all()
def get_all_hesitations_from_hesitant_id(hesitant_ids):
query = Hesitation.query.filter(
Hesitation.hesitant_id.in_(hesitant_ids)
)
return query.all()
def get_rebuttal(target_instance_id, source_instance_id):
query = Rebuttal.query.filter_by(
target_id=target_instance_id,