From 0159c507426a25213c0f5b8fa39fcf003a25600d Mon Sep 17 00:00:00 2001 From: Divided by Zer0 Date: Wed, 27 Sep 2023 23:24:39 +0200 Subject: [PATCH] feat: Tags (#39) --- CHANGELOG.md | 9 ++++ fediseer/apis/models/v1.py | 5 ++ fediseer/apis/v1/__init__.py | 2 + fediseer/apis/v1/admin.py | 1 - fediseer/apis/v1/endorsements.py | 2 + fediseer/apis/v1/hesitations.py | 4 +- fediseer/apis/v1/tags.py | 84 ++++++++++++++++++++++++++++++++ fediseer/apis/v1/whitelist.py | 18 ++++++- fediseer/classes/instance.py | 17 ++++++- fediseer/consts.py | 2 +- fediseer/database/functions.py | 50 ++++++++++++++----- 11 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 fediseer/apis/v1/tags.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 50bcb9c..e44b3d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +# 0.19.0 + +* Added instance Tags. Instance owners can add and remove them +* Limited retrieval of whitelist to 10 instances by default +* Added paging and limit to whitelist get +* Instances will now display their tags, unless muted. +* Can now retrieve instances in the whitelist filtered by tags +* Fix hesitations received appearing when visibility was limited. + # 0.18.0 * Added instance flags diff --git a/fediseer/apis/models/v1.py b/fediseer/apis/models/v1.py index c0e2c06..2decc01 100644 --- a/fediseer/apis/models/v1.py +++ b/fediseer/apis/models/v1.py @@ -41,6 +41,8 @@ class Models: 'sysadmins': fields.Integer(required=False, default=None, description="The count of system administrators in this instance as reported by its admins."), 'moderators': fields.Integer(required=False, default=None, description="The count of community moderators in this instance as reported by its admins."), 'state': fields.String(required=True, enum=[e.name for e in enums.InstanceState], description="The state of the instance as seen from the fediseer."), + 'tags': fields.List(fields.String(min_length=2, required=False, description="Domain tags (if any)")), + }) self.response_model_flag_details = api.model('FlagDetails', { 'flag': fields.String(required=True, enum=[e.name for e in enums.InstanceFlags], description="The type of flag"), @@ -135,3 +137,6 @@ class Models: 'flag': fields.String(required=True, enum=[e.name for e in enums.InstanceFlags], description="The type of flag to apply"), 'comment': fields.String(max_length=255, required=False, description="A comment explaining this flag", example="reasons"), }) + self.input_tags = api.model('Tags', { + 'tags_csv': fields.String(min_length=2, required=True, description="A comma-separated list of tags"), + }) diff --git a/fediseer/apis/v1/__init__.py b/fediseer/apis/v1/__init__.py index 49d42dd..f4a7e34 100644 --- a/fediseer/apis/v1/__init__.py +++ b/fediseer/apis/v1/__init__.py @@ -10,6 +10,7 @@ import fediseer.apis.v1.badges as badges import fediseer.apis.v1.find as find import fediseer.apis.v1.report as report import fediseer.apis.v1.admin as admin +import fediseer.apis.v1.tags as tags from fediseer.apis.v1.base import api api.add_resource(base.Suspicions, "/instances") @@ -30,4 +31,5 @@ api.add_resource(guarantees.Guarantees, "/guarantees/") api.add_resource(badges.GuaranteeBadge, "/badges/guarantees/.svg") api.add_resource(badges.EndorsementBadge, "/badges/endorsements/.svg") api.add_resource(admin.Flag, "/admin/flags/") +api.add_resource(tags.Tags, "/admin/tags") api.add_resource(report.Report, "/reports") diff --git a/fediseer/apis/v1/admin.py b/fediseer/apis/v1/admin.py index 2455449..41654a7 100644 --- a/fediseer/apis/v1/admin.py +++ b/fediseer/apis/v1/admin.py @@ -1,5 +1,4 @@ from fediseer.apis.v1.base import * -from fediseer.messaging import activitypub_pm from fediseer import enums from fediseer.classes.reports import Report from fediseer.register import ensure_instance_registered diff --git a/fediseer/apis/v1/endorsements.py b/fediseer/apis/v1/endorsements.py index c513322..35e4814 100644 --- a/fediseer/apis/v1/endorsements.py +++ b/fediseer/apis/v1/endorsements.py @@ -13,6 +13,8 @@ class Approvals(Resource): get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args") get_parser.add_argument("min_endorsements", required=False, default=1, type=int, help="Limit to this amount of endorsements of more", location="args") get_parser.add_argument("reasons_csv", required=False, type=str, help="Only retrieve endorsements where their reasons include any of the text in this csv", location="args") + get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve", location="args") + get_parser.add_argument("limit", required=False, type=int, default=10, help="Which page of results to retrieve", location="args") decorators = [limiter.limit("45/minute"), limiter.limit("30/minute", key_func = get_request_path)] @api.expect(get_parser) diff --git a/fediseer/apis/v1/hesitations.py b/fediseer/apis/v1/hesitations.py index 4d0d131..9aa5cc4 100644 --- a/fediseer/apis/v1/hesitations.py +++ b/fediseer/apis/v1/hesitations.py @@ -113,12 +113,12 @@ class Hesitations(Resource): precheck_instances = database.get_all_hesitant_instances_by_dubious_id(instance.id) instances = [] for p_instance in precheck_instances: - if p_instance.visibility_endorsements == enums.ListVisibility.ENDORSED: + if p_instance.visibility_hesitations == enums.ListVisibility.ENDORSED: if get_instance is None: continue if not p_instance.is_endorsing(get_instance): continue - if p_instance.visibility_endorsements == enums.ListVisibility.PRIVATE: + if p_instance.visibility_hesitations == enums.ListVisibility.PRIVATE: if get_instance is None: continue if p_instance != get_instance: diff --git a/fediseer/apis/v1/tags.py b/fediseer/apis/v1/tags.py new file mode 100644 index 0000000..6c76a1d --- /dev/null +++ b/fediseer/apis/v1/tags.py @@ -0,0 +1,84 @@ +from fediseer.apis.v1.base import * +from fediseer import enums +from fediseer.classes.reports import Report +from fediseer.classes.instance import InstanceTag + +class Tags(Resource): + + put_parser = reqparse.RequestParser() + put_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers') + put_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers") + put_parser.add_argument("tags_csv", required=True, type=str, help="The tags to apply", location="json") + + + @api.expect(put_parser,models.input_tags, validate=True) + @api.marshal_with(models.response_model_simple_response, code=200, description='Action Result') + @api.response(400, 'Bad Request', models.response_model_error) + @api.response(401, 'Invalid API Key', models.response_model_error) + @api.response(403, 'Access Denied', models.response_model_error) + + def put(self): + '''Tag your instance. + No hate speech allowed! + ''' + self.args = self.put_parser.parse_args() + if not self.args.apikey: + raise e.Unauthorized("You must provide the API key that was PM'd to the admin account of this instance") + user = database.find_user_by_api_key(self.args.apikey) + if not user: + raise e.Forbidden("Instance not found. Have you remembed to claim it?") + instance = database.find_instance_by_user(user) + changed = False + tags = [t.strip() for t in self.args.tags_csv.split(',')] + if database.instance_has_flag(instance.id,enums.InstanceFlags.RESTRICTED): + raise e.Forbidden("You cannot take this action as your instance is restricted") + if database.count_instance_tags(instance.id) + len(tags) >= 100: + raise e.BadRequest("You can't have more than 100 tags") + if len(tags) != len(set([t.lower() for t in tags])): + raise e.BadRequest("You cannot specify the same tag with different case.") + for tag in tags: + if database.instance_has_tag(instance.id,tag): + continue + new_tag = InstanceTag( + instance_id = instance.id, + tag = tag, + ) + db.session.add(new_tag) + changed = True + if changed: + db.session.commit() + return {"message": "Changed"},200 + return {"message": "OK"},200 + + delete_parser = reqparse.RequestParser() + delete_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers') + delete_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers") + delete_parser.add_argument("tags_csv", required=True, type=str, help="The tags to delete", location="json") + + + @api.expect(delete_parser,models.input_tags) + @api.marshal_with(models.response_model_simple_response, code=200, description='Instances', skip_none=True) + @api.response(400, 'Bad Request', models.response_model_error) + @api.response(401, 'Invalid API Key', models.response_model_error) + @api.response(403, 'Forbidden', models.response_model_error) + def delete(self): + '''Delete an instance's tag + ''' + self.args = self.patch_parser.parse_args() + if not self.args.apikey: + raise e.Unauthorized("You must provide the API key that was PM'd to the admin account") + user = database.find_user_by_api_key(self.args.apikey) + if not user: + raise e.Forbidden("Instance not found. Have you remembed to claim it?") + instance = database.find_instance_by_user(user) + changed = False + tags = [t.strip() for t in self.args.tags_csv.split(',')] + for tag in tags: + existing_tag = database.get_instance_tag(instance.id,tag) + if not existing_tag: + existing_tag + db.session.delete(existing_tag) + if changed: + db.session.commit() + return {"message": "Changed"},200 + return {"message": "OK"},200 \ No newline at end of file diff --git a/fediseer/apis/v1/whitelist.py b/fediseer/apis/v1/whitelist.py index 66e04fb..de591f0 100644 --- a/fediseer/apis/v1/whitelist.py +++ b/fediseer/apis/v1/whitelist.py @@ -11,6 +11,9 @@ class Whitelist(Resource): get_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers") get_parser.add_argument("endorsements", required=False, default=0, type=int, help="Limit to this amount of endorsements of more", location="args") get_parser.add_argument("guarantors", required=False, default=1, type=int, help="Limit to this amount of guarantors of more", location="args") + get_parser.add_argument("tags_csv", required=False, type=str, help="A list of tags to filter.", location="args") + get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve", location="args") + get_parser.add_argument("limit", required=False, type=int, default=1000, help="Which page of results to retrieve", location="args") get_parser.add_argument("csv", required=False, type=bool, help="Set to true to return just the domains as a csv. Mutually exclusive with domains", location="args") get_parser.add_argument("domains", required=False, type=bool, help="Set to true to return just the domains as a list. Mutually exclusive with csv", location="args") @@ -21,8 +24,21 @@ class Whitelist(Resource): '''A List with the details of all instances and their endorsements ''' self.args = self.get_parser.parse_args() + # if self.args.limit > 100: # Once limit is in effect + # raise e.BadRequest("limit cannot be more than 100") + if self.args.limit < 10: + raise e.BadRequest("Limit cannot be less than 10") + tags = None + if self.args.tags_csv is not None: + tags = [t.strip() for t in self.args.tags_csv.split(',')] instance_details = [] - for instance in database.get_all_instances(self.args.endorsements,self.args.guarantors): + for instance in database.get_all_instances( + min_endorsements=self.args.endorsements, + min_guarantors=self.args.guarantors, + tags=tags, + page=self.args.page, + limit=self.args.limit, + ): instance_details.append(instance.get_details(show_visibilities=True)) if self.args.csv: return {"csv": ",".join([instance["domain"] for instance in instance_details])},200 diff --git a/fediseer/classes/instance.py b/fediseer/classes/instance.py index 1ba0ef6..0a9134b 100644 --- a/fediseer/classes/instance.py +++ b/fediseer/classes/instance.py @@ -1,6 +1,7 @@ from datetime import datetime from sqlalchemy import Enum, UniqueConstraint from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import func, Index from loguru import logger from fediseer.flask import db, SQLITE_MODE @@ -92,6 +93,18 @@ class InstanceFlag(db.Model): instance = db.relationship("Instance", back_populates="flags") created = db.Column(db.DateTime, default=datetime.utcnow, nullable=False, index=True) +class InstanceTag(db.Model): + __tablename__ = "instance_tags" + __table_args__ = ( + UniqueConstraint('instance_id', 'tag', name='instance_tags_instance_id_tag'), + Index("ix_tags_lower", func.lower('tag')), + ) + id = db.Column(db.Integer, primary_key=True) + tag = db.Column(db.String(100), unique=False, nullable=False) + instance_id = db.Column(db.Integer, db.ForeignKey("instances.id", ondelete="CASCADE"), nullable=False, index=True) + instance = db.relationship("Instance", back_populates="tags") + created = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + class Instance(db.Model): __tablename__ = "instances" @@ -127,7 +140,8 @@ class Instance(db.Model): rejections = db.relationship("RejectionRecord", back_populates="rejector_instance", cascade="all, delete-orphan", foreign_keys=[RejectionRecord.rejector_id]) rejectors = db.relationship("RejectionRecord", back_populates="rejected_instance", cascade="all, delete-orphan", foreign_keys=[RejectionRecord.rejected_id]) admins = db.relationship("Claim", back_populates="instance", cascade="all, delete-orphan") - flags = db.relationship("InstanceFlag", back_populates="instance", cascade="all, delete-orphan") + flags = db.relationship("InstanceFlag", back_populates="instance", cascade="all, delete-orphan", lazy='joined') + tags = db.relationship("InstanceTag", back_populates="instance", cascade="all, delete-orphan", lazy='joined') def create(self): db.session.add(self) @@ -153,6 +167,7 @@ class Instance(db.Model): "sysadmins": self.sysadmins, "moderators": self.moderators, "state": self.get_state().name, + "tags": [t.tag for t in self.tags] if not self.has_flag(enums.InstanceFlags.MUTED) else [], } if show_visibilities: ret_dict["visibility_endorsements"] = self.visibility_endorsements.name diff --git a/fediseer/consts.py b/fediseer/consts.py index bb0ae58..0fbd340 100644 --- a/fediseer/consts.py +++ b/fediseer/consts.py @@ -1,4 +1,4 @@ -FEDISEER_VERSION = "0.18.0" +FEDISEER_VERSION = "0.19.0" SUPPORTED_SOFTWARE = { "lemmy", "mastodon", diff --git a/fediseer/database/functions.py b/fediseer/database/functions.py index 545a8ef..6657fac 100644 --- a/fediseer/database/functions.py +++ b/fediseer/database/functions.py @@ -1,20 +1,22 @@ -import time -import uuid -import json from loguru import logger from datetime import datetime, timedelta from sqlalchemy import func, or_, and_, not_, Boolean -from sqlalchemy.orm import noload -from fediseer.flask import db, SQLITE_MODE +from fediseer.flask import db from fediseer.utils import hash_api_key from sqlalchemy.orm import joinedload -from fediseer.classes.instance import Instance, Endorsement, Guarantee, RejectionRecord, Censure, Hesitation, Solicitation, InstanceFlag +from fediseer.classes.instance import Instance, Endorsement, Guarantee, RejectionRecord, Censure, Hesitation, Solicitation, InstanceFlag, InstanceTag from fediseer.classes.user import Claim, User from fediseer.classes.reports import Report from fediseer import enums -from fediseer.consts import POLLS_PER_DAY -def get_all_instances(min_endorsements = 0, min_guarantors = 1, include_decommissioned = True): +def get_all_instances( + min_endorsements = 0, + min_guarantors = 1, + tags = None, + include_decommissioned = True, + page=1, + limit=10, + ): query = db.session.query( Instance ).outerjoin( @@ -35,8 +37,14 @@ def get_all_instances(min_endorsements = 0, min_guarantors = 1, include_decommis ).having( db.func.count(Instance.guarantors) >= min_guarantors, ) - return query.all() - + if tags: + # Convert tags to lowercase and filter instances that have any of the tags + lower_tags = [tag.lower() for tag in tags] + query = query.filter(Instance.tags.any(func.lower(InstanceTag.tag).in_(lower_tags))) + page -= 1 + if page < 0: + page = 0 + return query.order_by(Instance.created.desc()).offset(limit * page).limit(limit).all() def get_all_endorsed_instances_by_approving_id(approving_ids): query = db.session.query( @@ -473,4 +481,24 @@ def instance_has_flag(instance_id, flag_enum): InstanceFlag.instance_id == instance_id, InstanceFlag.flag == flag_enum, ) - return query.count() == 1 \ No newline at end of file + return query.count() == 1 + +def get_instance_tag(instance_id, tag: str): + query = InstanceTag.query.filter( + InstanceTag.instance_id == instance_id, + InstanceTag.flag == tag, + ) + return query.first() + +def instance_has_tag(instance_id, tag: str): + query = InstanceTag.query.filter( + InstanceTag.instance_id == instance_id, + InstanceTag.tag == tag.lower(), + ) + return query.count() == 1 + +def count_instance_tags(instance_id): + query = InstanceTag.query.filter( + InstanceTag.instance_id == instance_id, + ) + return query.count() \ No newline at end of file