From fff0415906cf207fc4df3a0803f714a363daa00a Mon Sep 17 00:00:00 2001 From: Divided by Zer0 Date: Sat, 24 Jun 2023 02:23:53 +0200 Subject: [PATCH] Fediseer refactoring (#6) Can add instances without having to register them in a lemmy instance first Can guarantee instances even if nobody has claimed them yet Can handle multiple fediverse projects Can send PMs directly from fediseer Can have multiple API keys per instance --- .env_template | 8 +- .gitignore | 12 +- README.md | 8 +- Untitled-1.json | 44 +++++ examples/update_blacklist.py | 2 +- {overseer => fediseer}/__init__.py | 12 +- {overseer => fediseer}/apis/__init__.py | 0 {overseer => fediseer}/apis/apiv1.py | 2 +- .../apis/models/__init__.py | 0 {overseer => fediseer}/apis/models/v1.py | 0 {overseer => fediseer}/apis/v1/__init__.py | 12 +- {overseer => fediseer}/apis/v1/activitypub.py | 24 +-- {overseer => fediseer}/apis/v1/base.py | 16 +- .../apis/v1/endorsements.py | 19 ++- {overseer => fediseer}/apis/v1/guarantees.py | 57 +++++-- {overseer => fediseer}/apis/v1/whitelist.py | 157 ++++++++++++------ {overseer => fediseer}/argparser.py | 0 fediseer/classes/__init__.py | 49 ++++++ {overseer => fediseer}/classes/instance.py | 24 ++- {overseer => fediseer}/classes/news.py | 0 fediseer/classes/user.py | 35 ++++ fediseer/consts.py | 5 + {overseer => fediseer}/database/__init__.py | 0 {overseer => fediseer}/database/functions.py | 103 +++++++++++- {overseer => fediseer}/exceptions.py | 0 fediseer/fediverse.py | 37 +++++ {overseer => fediseer}/flask.py | 5 +- {overseer => fediseer}/lemmy.py | 12 +- {overseer => fediseer}/limiter.py | 2 +- {overseer => fediseer}/logger.py | 4 +- fediseer/messaging.py | 137 +++++++++++++++ {overseer => fediseer}/observer.py | 0 {overseer => fediseer}/routes.py | 13 +- {overseer => fediseer}/templates/index.md | 4 +- {overseer => fediseer}/utils.py | 12 +- overseer/classes/__init__.py | 32 ---- overseer/consts.py | 1 - quickrun.py | 2 +- server.py | 4 +- test.py | 41 +++++ 40 files changed, 700 insertions(+), 195 deletions(-) create mode 100644 Untitled-1.json rename {overseer => fediseer}/__init__.py (72%) rename {overseer => fediseer}/apis/__init__.py (100%) rename {overseer => fediseer}/apis/apiv1.py (88%) rename {overseer => fediseer}/apis/models/__init__.py (100%) rename {overseer => fediseer}/apis/models/v1.py (100%) rename {overseer => fediseer}/apis/v1/__init__.py (67%) rename {overseer => fediseer}/apis/v1/activitypub.py (64%) rename {overseer => fediseer}/apis/v1/base.py (83%) rename {overseer => fediseer}/apis/v1/endorsements.py (90%) rename {overseer => fediseer}/apis/v1/guarantees.py (78%) rename {overseer => fediseer}/apis/v1/whitelist.py (61%) rename {overseer => fediseer}/argparser.py (100%) create mode 100644 fediseer/classes/__init__.py rename {overseer => fediseer}/classes/instance.py (71%) rename {overseer => fediseer}/classes/news.py (100%) create mode 100644 fediseer/classes/user.py create mode 100644 fediseer/consts.py rename {overseer => fediseer}/database/__init__.py (100%) rename {overseer => fediseer}/database/functions.py (60%) rename {overseer => fediseer}/exceptions.py (100%) create mode 100644 fediseer/fediverse.py rename {overseer => fediseer}/flask.py (88%) rename {overseer => fediseer}/lemmy.py (68%) rename {overseer => fediseer}/limiter.py (89%) rename {overseer => fediseer}/logger.py (94%) create mode 100644 fediseer/messaging.py rename {overseer => fediseer}/observer.py (100%) rename {overseer => fediseer}/routes.py (79%) rename {overseer => fediseer}/templates/index.md (81%) rename {overseer => fediseer}/utils.py (86%) delete mode 100644 overseer/classes/__init__.py delete mode 100644 overseer/consts.py create mode 100644 test.py diff --git a/.env_template b/.env_template index 8830186..e4d19dc 100644 --- a/.env_template +++ b/.env_template @@ -1,7 +1,7 @@ -POSTGRES_URI="postgresql://postgres:ChangeMe@postgres.example.tld/overseer" +POSTGRES_URI="postgresql://postgres:ChangeMe@postgres.example.tld/fediseer" USE_SQLITE=0 -OVERSEER_LEMMY_DOMAIN="overctrl.example.tld" -OVERSEER_LEMMY_USERNAME="overseer" -OVERSEER_LEMMY_PASSWORD="LemmyPassword" +FEDISEER_LEMMY_DOMAIN="fediseer.com" +FEDISEER_LEMMY_USERNAME="fediseer" +FEDISEER_LEMMY_PASSWORD="LemmyPassword" ADMIN_API_KEY="Password" secret_key="VerySecretKey" \ No newline at end of file diff --git a/.gitignore b/.gitignore index dfbe880..42dc2db 100644 --- a/.gitignore +++ b/.gitignore @@ -130,10 +130,8 @@ dmypy.json db/* -test_commands.txt -SQL_statements.txt -horde.log -horde*.bz2 -horde.db -/.idea -/boto3oeo.py \ No newline at end of file +fediseer.log +fediseer*.bz2 +fediseer.db +private.pem +public.pem \ No newline at end of file diff --git a/README.md b/README.md index b341484..ace0e0e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# Lemmy Overseer +# Fediseer -This service provides an REST API which can be used to retrieve various information about Lemmy instances, particularly focused on detecting and countering bad actors. +This service provides an REST API which can be used to retrieve various information about Fediverse instances, particularly focused on detecting and countering bad actors. It's reliant on the [Lemmy Fediverse Observer](https://lemmy.fediverse.observer/) -The currently running instance is on https://overseer.dbzer0.com +The currently running instance is on https://fediseer.com -See devlog: https://dbzer0.com/blog/overseer-a-fediverse-chain-of-trust/ \ No newline at end of file +See devlog: https://dbzer0.com/blog/fediseer-a-fediverse-chain-of-trust/ \ No newline at end of file diff --git a/Untitled-1.json b/Untitled-1.json new file mode 100644 index 0000000..03e44ab --- /dev/null +++ b/Untitled-1.json @@ -0,0 +1,44 @@ +{ + "id": "https://overctrl.dbzer0.com/activities/create/d50c9fe4-28a5-4de4-8421-8f806d9d0947", + "to": ["https://lemmy.dbzer0.com/u/db0"], + "type": "Create", + "actor": "https://overctrl.dbzer0.com/u/fediseer", + "object": { + "id": "https://overctrl.dbzer0.com/private_message/68", + "type": "ChatMessage", + "to": ["https://lemmy.dbzer0.com/u/db0"], + "source": { + "content": "password", + "mediaType": "text/markdown" + }, + "attributedTo": "https: //overctrl.dbzer0.com/u/fediseer", + "content": "

password

\n", + "mediaType": "text/html", + "published": "2023-06-23T15: 57: 51.950327+00: 00" + }, + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https: //w3id.org/security/v1", + { + "pt": "https://joinpeertube.org/ns#", + "sc": "http://schema.org/", + "lemmy": "https://join-lemmy.org/ns#", + "expires": "as:endTime", + "litepub": "http://litepub.social/ns#", + "language": "sc:inLanguage", + "stickied": "lemmy:stickied", + "sensitive": "as:sensitive", + "identifier": "sc:identifier", + "moderators": { + "@id": "lemmy:moderators", + "@type": "@id" + }, + "removeData": "lemmy:removeData", + "ChatMessage": "litepub:ChatMessage", + "matrixUserId": "lemmy:matrixUserId", + "distinguished": "lemmy:distinguished", + "commentsEnabled": "pt:commentsEnabled", + "postingRestrictedToMods": "lemmy:postingRestrictedToMods" + } + ] +} \ No newline at end of file diff --git a/examples/update_blacklist.py b/examples/update_blacklist.py index 3bd6ee1..69208ec 100644 --- a/examples/update_blacklist.py +++ b/examples/update_blacklist.py @@ -21,7 +21,7 @@ if lemmy.log_in(USERNAME, PASSWORD) is False: raise Exception("Could not log in to lemmy") print("Fetching suspicions") -sus = requests.get(f"https://overseer.dbzer0.com/api/v1/instances?activity_suspicion={ACTIVITY_SUSPICION}&domains=true", timeout=5).json() +sus = requests.get(f"https://fediseer.com/api/v1/instances?activity_suspicion={ACTIVITY_SUSPICION}&domains=true", timeout=5).json() defed = blacklist | set(sus["domains"]) print("Editing Defederation list") ret = lemmy.site.edit(blocked_instances=list(defed)) diff --git a/overseer/__init__.py b/fediseer/__init__.py similarity index 72% rename from overseer/__init__.py rename to fediseer/__init__.py index 17c2a12..f76b146 100644 --- a/overseer/__init__.py +++ b/fediseer/__init__.py @@ -2,12 +2,12 @@ import os import socket from uuid import uuid4 -from overseer.logger import logger -from overseer.flask import OVERSEER -from overseer.routes import * -from overseer.apis import apiv1 -from overseer.argparser import args -from overseer.consts import OVERSEER_VERSION +from fediseer.logger import logger +from fediseer.flask import OVERSEER +from fediseer.routes import * +from fediseer.apis import apiv1 +from fediseer.argparser import args +from fediseer.consts import OVERSEER_VERSION OVERSEER.register_blueprint(apiv1) diff --git a/overseer/apis/__init__.py b/fediseer/apis/__init__.py similarity index 100% rename from overseer/apis/__init__.py rename to fediseer/apis/__init__.py diff --git a/overseer/apis/apiv1.py b/fediseer/apis/apiv1.py similarity index 88% rename from overseer/apis/apiv1.py rename to fediseer/apis/apiv1.py index 6ccbcf0..0d235b1 100644 --- a/overseer/apis/apiv1.py +++ b/fediseer/apis/apiv1.py @@ -2,7 +2,7 @@ from flask import Blueprint from flask_restx import Api from importlib import import_module -from overseer.apis.v1 import api as v1 +from fediseer.apis.v1 import api as v1 blueprint = Blueprint('apiv1', __name__, url_prefix='/api') api = Api(blueprint, diff --git a/overseer/apis/models/__init__.py b/fediseer/apis/models/__init__.py similarity index 100% rename from overseer/apis/models/__init__.py rename to fediseer/apis/models/__init__.py diff --git a/overseer/apis/models/v1.py b/fediseer/apis/models/v1.py similarity index 100% rename from overseer/apis/models/v1.py rename to fediseer/apis/models/v1.py diff --git a/overseer/apis/v1/__init__.py b/fediseer/apis/v1/__init__.py similarity index 67% rename from overseer/apis/v1/__init__.py rename to fediseer/apis/v1/__init__.py index 00bb60e..c999cb4 100644 --- a/overseer/apis/v1/__init__.py +++ b/fediseer/apis/v1/__init__.py @@ -1,9 +1,9 @@ -import overseer.apis.v1.base as base -import overseer.apis.v1.whitelist as whitelist -import overseer.apis.v1.endorsements as endorsements -import overseer.apis.v1.guarantees as guarantees -import overseer.apis.v1.activitypub as activitypub -from overseer.apis.v1.base import api +import fediseer.apis.v1.base as base +import fediseer.apis.v1.whitelist as whitelist +import fediseer.apis.v1.endorsements as endorsements +import fediseer.apis.v1.guarantees as guarantees +import fediseer.apis.v1.activitypub as activitypub +from fediseer.apis.v1.base import api api.add_resource(base.Suspicions, "/instances") api.add_resource(activitypub.User, "/user/") diff --git a/overseer/apis/v1/activitypub.py b/fediseer/apis/v1/activitypub.py similarity index 64% rename from overseer/apis/v1/activitypub.py rename to fediseer/apis/v1/activitypub.py index 4523387..8053235 100644 --- a/overseer/apis/v1/activitypub.py +++ b/fediseer/apis/v1/activitypub.py @@ -1,5 +1,5 @@ -from overseer.apis.v1.base import * -from overseer.utils import get_nodeinfo +from fediseer.apis.v1.base import * +from fediseer.fediverse import get_nodeinfo class User(Resource): get_parser = reqparse.RequestParser() @@ -10,28 +10,28 @@ class User(Resource): '''User details ''' self.args = self.get_parser.parse_args() - if username != "overseer": + if username != "fediseer": raise e.NotFound("User does not exist") with open('public.pem', 'r') as file: pubkey = file.read() - overseer = { + fediseer = { "@context": [ "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" ], "type": "Person", - "id": "https://overseer.dbzer0.com/api/v1/user/overseer", + "id": "https://fediseer.com/api/v1/user/fediseer", "type": "Person", - "preferredUsername": "overseer", - "inbox": "https://overseer.dbzer0.com/api/v1/inbox/overseer", - "outbox": "https://overseer.dbzer0.com/api/v1/outbox/overseer", + "preferredUsername": "fediseer", + "inbox": "https://fediseer.com/api/v1/inbox/fediseer", + "outbox": "https://fediseer.com/api/v1/outbox/fediseer", "publicKey": { - "id": "https://overseer.dbzer0.com/api/v1/user/overseer#main-key", - "owner": "https://overseer.dbzer0.com/api/v1/user/overseer", + "id": "https://fediseer.com/api/v1/user/fediseer#main-key", + "owner": "https://fediseer.com/api/v1/user/fediseer", "publicKeyPem": pubkey } } - return overseer,200 + return fediseer,200 class Inbox(Resource): post_parser = reqparse.RequestParser() @@ -40,7 +40,7 @@ class Inbox(Resource): def post(self, username): '''User Inbox ''' - if username != "overseer": + if username != "fediseer": raise e.NotFound("User does not exist") self.args = self.post_parser.parse_args() json_payload = request.get_json() diff --git a/overseer/apis/v1/base.py b/fediseer/apis/v1/base.py similarity index 83% rename from overseer/apis/v1/base.py rename to fediseer/apis/v1/base.py index 844a3c1..d430df8 100644 --- a/overseer/apis/v1/base.py +++ b/fediseer/apis/v1/base.py @@ -1,19 +1,19 @@ import os from flask import request from flask_restx import Namespace, Resource, reqparse -from overseer.flask import cache, db -from overseer.observer import retrieve_suspicious_instances +from fediseer.flask import cache, db +from fediseer.observer import retrieve_suspicious_instances from loguru import logger -from overseer.classes.instance import Instance -from overseer.database import functions as database -from overseer import exceptions as e -from overseer.utils import hash_api_key -from overseer.lemmy import pm_new_api_key, pm_instance +from fediseer.classes.instance import Instance +from fediseer.database import functions as database +from fediseer import exceptions as e +from fediseer.utils import hash_api_key +from fediseer.messaging import activitypub_pm from pythorhead import Lemmy api = Namespace('v1', 'API Version 1' ) -from overseer.apis.models.v1 import Models +from fediseer.apis.models.v1 import Models models = Models(api) diff --git a/overseer/apis/v1/endorsements.py b/fediseer/apis/v1/endorsements.py similarity index 90% rename from overseer/apis/v1/endorsements.py rename to fediseer/apis/v1/endorsements.py index 0093ef6..a3d510c 100644 --- a/overseer/apis/v1/endorsements.py +++ b/fediseer/apis/v1/endorsements.py @@ -1,5 +1,5 @@ -from overseer.apis.v1.base import * -from overseer.classes.instance import Endorsement +from fediseer.apis.v1.base import * +from fediseer.classes.instance import Endorsement class Approvals(Resource): get_parser = reqparse.RequestParser() @@ -93,7 +93,13 @@ class Endorsements(Resource): ) db.session.add(new_endorsement) db.session.commit() - pm_instance(target_instance.domain, f"Your instance has just been endorsed by {instance.domain}") + if not database.has_recent_endorsement(target_instance.id): + activitypub_pm.pm_admins( + message=f"Your instance has just been endorsed by {instance.domain}", + domain=target_instance.domain, + software=target_instance.software, + instance=target_instance, + ) logger.info(f"{instance.domain} Endorsed {domain}") return {"message":'Changed'}, 200 @@ -124,6 +130,11 @@ class Endorsements(Resource): return {"message":'OK'}, 200 db.session.delete(endorsement) db.session.commit() - pm_instance(target_instance.domain, f"Oh now. {instance.domain} has just withdrawn the endorsement of your instance") + activitypub_pm.pm_admins( + message=f"Oh no. {instance.domain} has just withdrawn the endorsement of your instance", + domain=target_instance.domain, + software=target_instance.software, + instance=target_instance, + ) logger.info(f"{instance.domain} Withdrew endorsement from {domain}") return {"message":'Changed'}, 200 \ No newline at end of file diff --git a/overseer/apis/v1/guarantees.py b/fediseer/apis/v1/guarantees.py similarity index 78% rename from overseer/apis/v1/guarantees.py rename to fediseer/apis/v1/guarantees.py index c41b764..3cfd2fd 100644 --- a/overseer/apis/v1/guarantees.py +++ b/fediseer/apis/v1/guarantees.py @@ -1,5 +1,5 @@ -from overseer.apis.v1.base import * -from overseer.classes.instance import Guarantee, Endorsement +from fediseer.apis.v1.base import * +from fediseer.classes.instance import Guarantee, Endorsement, RejectionRecord class Guarantors(Resource): get_parser = reqparse.RequestParser() @@ -66,7 +66,7 @@ class Guarantees(Resource): @api.response(403, 'Instance Not Guaranteed or Tartget instance Guaranteed by others', models.response_model_error) @api.response(404, 'Instance not registered', models.response_model_error) def put(self, domain): - '''Endorse an instance + '''Guarantee an instance ''' self.args = self.put_parser.parse_args() if not self.args.apikey: @@ -83,12 +83,12 @@ class Guarantees(Resource): raise e.Forbidden(f"Guarantee chain for this instance has been broken. Chain ends at {chainbreaker.domain}!") target_instance = database.find_instance_by_domain(domain=domain) if not target_instance: - raise e.BadRequest("Instance to endorse not found") + raise e.BadRequest("Instance to endorse not found") if database.get_guarantee(target_instance.id,instance.id): return {"message":'OK'}, 200 gdomain = target_instance.get_guarantor_domain() if gdomain: - raise e.Forbidden("Target instance already guaranteed by {gdomain}") + raise e.Forbidden(f"Target instance already guaranteed by {gdomain}") new_guarantee = Guarantee( guaranteed_id=target_instance.id, guarantor_id=instance.id, @@ -101,10 +101,20 @@ class Guarantees(Resource): ) db.session.add(new_endorsement) db.session.commit() - pm_instance(target_instance.domain, f"Congratulations! Your instance has just been guaranteed by {instance.domain}. This also comes with your first endorsement.") + activitypub_pm.pm_admins( + message=f"Congratulations! Your instance has just been guaranteed by {instance.domain}. This also comes with your first endorsement.", + domain=target_instance.domain, + software=target_instance.software, + instance=target_instance, + ) orphan_ids = database.get_guarantee_chain(target_instance.id) for orphan in database.get_instances_by_ids(orphan_ids): - pm_instance(orphan.domain, f"Phew! You guarantor chain has been repaired as {instance.domain} has guaranteed for {domain}.") + activitypub_pm.pm_admins( + message=f"Phew! You guarantor chain has been repaired as {instance.domain} has guaranteed for {domain}.", + domain=orphan.domain, + software=orphan.software, + instance=orphan, + ) orphan.unset_as_orphan() logger.info(f"{instance.domain} Guaranteed for {domain}") return {"message":'Changed'}, 200 @@ -138,21 +148,40 @@ class Guarantees(Resource): guarantee = database.get_guarantee(target_instance.id,instance.id) if not guarantee: return {"message":'OK'}, 200 + if database.has_recent_rejection(target_instance.id,instance.id): + raise e.Forbidden("You cannot remove your guarantee from the same instance within 24 hours") # Removing a guarantee removes the endorsement - endorsement = database.get_endorsement(target_instance.id,instance.id) - if endorsement: + endorsement = database.get_endorsement(target_instance.id,instance.id) + if endorsement: db.session.delete(endorsement) db.session.delete(guarantee) + rejection_record = database.get_rejection_record(instance.id,target_instance.id) + if rejection_record: + rejection_record.refresh() + else: + rejection = RejectionRecord( + rejected_id=target_instance.id, + rejector_id=instance.id, + ) + db.session.add(rejection) db.session.commit() - pm_instance(target_instance.domain, - f"Attention! You guarantor instance {instance.domain} has withdrawn their backing.\n\n" - "IMPORTANT: You are still considered guaranteed for the next 24hours, but you cannot further endorse or guarantee others." + activitypub_pm.pm_admins( + message=f"Attention! You guarantor instance {instance.domain} has withdrawn their backing.\n\n" + "IMPORTANT: The instances you vouched for are still considered guaraneed but cannot guarantee or endorse others" "If you find a new guarantor then your guarantees will be reactivated!.\n\n" - "Note that if you do not find a guarantor within 7 days, all your endorsements will be removed." + "Note that if you do not find a guarantor within 7 days, all your guarantees and endorsements will be removed.", + domain=target_instance.domain, + software=target_instance.software, + instance=target_instance, ) orphan_ids = database.get_guarantee_chain(target_instance.id) for orphan in database.get_instances_by_ids(orphan_ids): - pm_instance(orphan.domain, f"Attention! You guarantor chain has been broken because {instance.domain} has withdrawn their backing from {domain}.\n\nIMPORTANT: All your guarantees will be deleted unless the chain is repaired or you find a new guarantor within 24hours!") + activitypub_pm.pm_admins( + message=f"Attention! You guarantor chain has been broken because {instance.domain} has withdrawn their backing from {domain}.\n\nIMPORTANT: All your guarantees will be deleted unless the chain is repaired or you find a new guarantor within 24hours!", + domain=orphan.domain, + software=orphan.software, + instance=orphan, + ) orphan.set_as_oprhan() logger.info(f"{instance.domain} Withdrew guarantee from {domain}") return {"message":'Changed'}, 200 \ No newline at end of file diff --git a/overseer/apis/v1/whitelist.py b/fediseer/apis/v1/whitelist.py similarity index 61% rename from overseer/apis/v1/whitelist.py rename to fediseer/apis/v1/whitelist.py index 8eb9741..d8e4a3b 100644 --- a/overseer/apis/v1/whitelist.py +++ b/fediseer/apis/v1/whitelist.py @@ -1,5 +1,8 @@ -from overseer.apis.v1.base import * -from overseer.utils import get_nodeinfo +from fediseer.apis.v1.base import * +from fediseer.messaging import activitypub_pm +from fediseer.fediverse import get_admin_for_software, get_nodeinfo +from fediseer.classes.user import User, Claim +from fediseer.consts import SUPPORTED_SOFTWARE class Whitelist(Resource): get_parser = reqparse.RequestParser() @@ -38,14 +41,15 @@ class WhitelistDomain(Resource): '''Display info about a specific instance ''' self.args = self.get_parser.parse_args() - instance = database.find_instance_by_domain(domain) + instance, nodeinfo, site, admin_usernames = self.ensure_instance_registered(domain) if not instance: - raise e.NotFound(f"No Instance found matching provided domain. Have you remembered to register it?") + raise e.NotFound(f"Something went wrong trying to register this instance.") return instance.get_details(),200 put_parser = reqparse.RequestParser() put_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers") + put_parser.add_argument("admin", required=False, type=str, help="The username of the admin who wants to register this domain", location="json") put_parser.add_argument("guarantor", required=False, type=str, help="(Optiona) The domain of the guaranteeing instance. They will receive a PM to validate you", location="json") @@ -53,64 +57,57 @@ class WhitelistDomain(Resource): @api.marshal_with(models.response_model_instances, code=200, description='Instances') @api.response(400, 'Bad Request', models.response_model_error) def put(self, domain): - '''Register a new instance to the overseer - An instance account has to exist in the overseer lemmylemmy instance + '''Register a new instance to the fediseer + An instance account has to exist in the fediseer lemmylemmy instance That account will recieve the new API key via PM ''' self.args = self.put_parser.parse_args() - existing_instance = database.find_instance_by_domain(domain) - if existing_instance: - return existing_instance.get_details(),200 + if '@' in self.args.admin: + raise e.BadRequest("Please send the username without any @ signs or domains") + instance, nodeinfo, site, admin_usernames = self.ensure_instance_registered(domain) guarantor_instance = None if self.args.guarantor: guarantor_instance = database.find_instance_by_domain(self.args.guarantor) if not guarantor_instance: raise e.BadRequest(f"Requested guarantor domain {self.args.guarantor} is not registered with the Overseer yet!") - if domain.endswith("test.dbzer0.com"): - requested_lemmy = Lemmy(f"https://{domain}") - requested_lemmy._requestor.nodeinfo = {"software":{"name":"lemmy"}} - open_registrations = False - email_verify = True - software = "lemmy" - else: - nodeinfo = get_nodeinfo(domain) - if not nodeinfo: - raise e.BadRequest(f"Error encountered while polling domain {domain}. Please check it's running correctly") - software = nodeinfo["software"]["name"] - if software == "lemmy": - requested_lemmy = Lemmy(f"https://{domain}") - site = requested_lemmy.site.get() - if not site: - raise e.BadRequest(f"Error encountered while polling lemmy domain {domain}. Please check it's running correctly") - open_registrations = site["site_view"]["local_site"]["registration_mode"] == "open" - email_verify = site["site_view"]["local_site"]["require_email_verification"] - software = software - else: - open_registrations = nodeinfo["openRegistrations"] - email_verify = False - api_key = pm_new_api_key(domain) + if self.args.admin not in admin_usernames: + raise e.Forbidden(f"Only admins of that {instance.software} are allowed to claim it.") + existing_claim = database.find_claim(f"@{self.args.admin}@{domain}") + if existing_claim: + raise e.Forbidden(f"You have already claimed this instance as this admin. Please use the PATCH method to reset your API key.") + api_key = activitypub_pm.pm_new_api_key(domain, self.args.admin, instance.software) if not api_key: raise e.BadRequest("Failed to generate API Key") - new_instance = Instance( - domain=domain, + new_user = User( api_key=hash_api_key(api_key), - open_registrations=open_registrations, - email_verify=email_verify, - software=software, + account=f"@{self.args.admin}@{domain}", + username=self.args.admin, ) - new_instance.create() + db.session.add(new_user) + db.session.commit() + new_claim = Claim( + user_id = new_user.id, + instance_id = instance.id, + ) + db.session.add(new_claim) + db.session.commit() if guarantor_instance: - pm_instance(guarantor_instance.domain, f"New instance {domain} was just registered with the Overseer and have asked you to guarantee for them!") - return new_instance.get_details(),200 + activitypub_pm.pm_admins( + message=f"New instance {domain} was just registered with the Overseer and have asked you to guarantee for them!", + domain=guarantor_instance.domain, + software=guarantor_instance.software, + instance=guarantor_instance, + ) + return instance.get_details(),200 patch_parser = reqparse.RequestParser() patch_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers') patch_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers") - patch_parser.add_argument("regenerate_key", required=False, type=bool, help="If True, will PM a new api key to this instance", location="json") + patch_parser.add_argument("regenerate_key", required=False, type=str, help="If a username is given, their API will be reset. This can be initiated by other instance admins or the fediseer.", location="json") @api.expect(patch_parser) - @api.marshal_with(models.response_model_instances, code=200, description='Instances', skip_none=True) + @api.marshal_with(models.response_model_simple_response, code=200, description='Instances', skip_none=True) @api.response(401, 'Invalid API Key', models.response_model_error) @api.response(403, 'Instance Not Registered', models.response_model_error) def patch(self, domain): @@ -118,19 +115,29 @@ class WhitelistDomain(Resource): ''' self.args = self.patch_parser.parse_args() if not self.args.apikey: - raise e.Unauthorized("You must provide the API key that was PM'd to your overctrl.dbzer0.com account") - instance = database.find_instance_by_api_key(self.args.apikey) - if not instance: - raise e.Forbidden(f"No Instance found matching provided API key and domain. Have you remembered to register it?") + raise e.Unauthorized("You must provide the API key that was PM'd admin account") + user = database.find_user_by_api_key(self.args.apikey) + if not user: + raise e.Forbidden("You have not yet claimed an instance. Use the POST method to do so.") + instance = database.find_instance_by_user(user) if self.args.regenerate_key: - new_key = pm_new_api_key(domain) - instance.api_key = hash_api_key(new_key) + requestor = None + if self.args.regenerate_key != user.username or user.username == "fediseer": + requestor = user.username + instance_to_reset = database.find_instance_by_account(f"@{self.args.regenerate_key}@{domain}") + if instance != instance_to_reset and user.username != "fediseer": + raise e.BadRequest("Only other admins or the fediseer can request API key reset for others.") + instance = instance_to_reset + user = database.find_user_by_account(f"@{self.args.regenerate_key}@{domain}") + new_key = activitypub_pm.pm_new_api_key(domain, self.args.regenerate_key, instance.software, requestor=requestor) + user.api_key = hash_api_key(new_key) db.session.commit() - return instance.get_details(),200 + return {"message": "Changed"},200 delete_parser = reqparse.RequestParser() delete_parser.add_argument("apikey", type=str, required=True, help="The sending instance's API key.", location='headers') delete_parser.add_argument("Client-Agent", default="unknown:0:unknown", type=str, required=False, help="The client name and version.", location="headers") + delete_parser.add_argument("username", required=False, type=str, help="(Not Implemented) Provide the username of another admin to remove their API key", location="json") @api.expect(delete_parser) @@ -139,18 +146,62 @@ class WhitelistDomain(Resource): @api.response(401, 'Invalid API Key', models.response_model_error) @api.response(403, 'Forbidden', models.response_model_error) def delete(self, domain): - '''Delete instance from overseer + '''Delete claim to instance ''' + return e.BadRequest("Not implemented") self.args = self.patch_parser.parse_args() if not self.args.apikey: - raise e.Unauthorized("You must provide the API key that was PM'd to your overctrl.dbzer0.com account") + raise e.Unauthorized("You must provide the API key that was PM'd to your account") instance = database.find_authenticated_instance(domain, self.args.apikey) if not instance: raise e.BadRequest(f"No Instance found matching provided API key and domain. Have you remembered to register it?") - if domain == os.getenv('OVERSEER_LEMMY_DOMAIN'): - raise e.Forbidden("Cannot delete overseer control instance") + if domain == os.getenv('FEDISEER_LEMMY_DOMAIN'): + raise e.Forbidden("Cannot delete fediseer control instance") db.session.delete(instance) db.session.commit() logger.warning(f"{domain} deleted") return {"message":'Changed'}, 200 + def ensure_instance_registered(self, domain): + if domain.endswith("test.dbzer0.com"): + # Fake instances for testing chain of trust + requested_lemmy = Lemmy(f"https://{domain}") + requested_lemmy._requestor.nodeinfo = {"software":{"name":"lemmy"}} + open_registrations = False + email_verify = True + software = "lemmy" + admin_usernames = ["db0"] + nodeinfo = get_nodeinfo("lemmy.dbzer0.com") + requested_lemmy = Lemmy(f"https://{domain}") + site = requested_lemmy.site.get() + else: + nodeinfo = get_nodeinfo(domain) + if not nodeinfo: + raise e.BadRequest(f"Error encountered while polling domain {domain}. Please check it's running correctly") + software = nodeinfo["software"]["name"] + if software not in SUPPORTED_SOFTWARE: + raise e.BadRequest(f"Fediverse software {software} not supported at this time") + if software == "lemmy": + requested_lemmy = Lemmy(f"https://{domain}") + site = requested_lemmy.site.get() + if not site: + raise e.BadRequest(f"Error encountered while polling lemmy domain {domain}. Please check it's running correctly") + open_registrations = site["site_view"]["local_site"]["registration_mode"] == "open" + email_verify = site["site_view"]["local_site"]["require_email_verification"] + software = software + admin_usernames = [a["person"]["name"] for a in site["admins"]] + else: + open_registrations = nodeinfo["openRegistrations"] + email_verify = False + admin_usernames = get_admin_for_software(software, domain) + instance = database.find_instance_by_domain(domain) + if instance: + return instance, nodeinfo, site, admin_usernames + new_instance = Instance( + domain=domain, + open_registrations=open_registrations, + email_verify=email_verify, + software=software, + ) + new_instance.create() + return new_instance, nodeinfo, site, admin_usernames diff --git a/overseer/argparser.py b/fediseer/argparser.py similarity index 100% rename from overseer/argparser.py rename to fediseer/argparser.py diff --git a/fediseer/classes/__init__.py b/fediseer/classes/__init__.py new file mode 100644 index 0000000..38bd68c --- /dev/null +++ b/fediseer/classes/__init__.py @@ -0,0 +1,49 @@ +import os +from loguru import logger +from fediseer.argparser import args +from importlib import import_module +from fediseer.flask import db, OVERSEER +from fediseer.utils import hash_api_key + +# Importing for DB creation +from fediseer.classes.instance import Instance, Guarantee +from fediseer.classes.user import User, Claim +import fediseer.classes.user + +with OVERSEER.app_context(): + + db.create_all() + + admin_domain = os.getenv("FEDISEER_LEMMY_DOMAIN") + admin_instance = db.session.query(Instance).filter_by(domain=admin_domain).first() + if not admin_instance: + admin_instance = Instance( + id=0, + domain=admin_domain, + open_registrations=False, + email_verify=False, + software="fediseer", + ) + admin_instance.create() + guarantee = Guarantee( + id=0, + guarantor_id = admin_instance.id, + guaranteed_id = admin_instance.id, + ) + db.session.add(guarantee) + db.session.commit() + admin_user = User( + id=0, + account = "@fediseer@fediseer.com", + username = "fediseer", + api_key=hash_api_key(os.getenv("ADMIN_API_KEY")), + ) + db.session.add(admin_user) + db.session.commit() + claim = Claim( + id=0, + user_id = admin_user.id, + instance_id = admin_instance.id + ) + db.session.add(claim) + db.session.commit() \ No newline at end of file diff --git a/overseer/classes/instance.py b/fediseer/classes/instance.py similarity index 71% rename from overseer/classes/instance.py rename to fediseer/classes/instance.py index c88b21b..abc367c 100644 --- a/overseer/classes/instance.py +++ b/fediseer/classes/instance.py @@ -7,10 +7,26 @@ from sqlalchemy import Enum, UniqueConstraint from sqlalchemy.dialects.postgresql import UUID from loguru import logger -from overseer.flask import db, SQLITE_MODE +from fediseer.flask import db, SQLITE_MODE uuid_column_type = lambda: UUID(as_uuid=True) if not SQLITE_MODE else db.String(36) - + +# This is used to know when last time an instance removed their guarantee from another to prevent trolling/spamming +# By someone adding/removing guarantees +class RejectionRecord(db.Model): + __tablename__ = "rejection_records" + __table_args__ = (UniqueConstraint('rejector_id', 'rejected_id', name='endoresements_rejector_id_rejected_id'),) + id = db.Column(db.Integer, primary_key=True) + rejector_id = db.Column(db.Integer, db.ForeignKey("instances.id", ondelete="CASCADE"), nullable=False) + rejector_instance = db.relationship("Instance", back_populates="rejections", foreign_keys=[rejector_id]) + rejected_id = db.Column(db.Integer, db.ForeignKey("instances.id", ondelete="CASCADE"), nullable=False) + rejected_instance = db.relationship("Instance", back_populates="rejectors", foreign_keys=[rejected_id]) + created = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + performed = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + def refresh(self): + self.performed = datetime.utcnow() + class Guarantee(db.Model): __tablename__ = "guarantees" @@ -38,7 +54,6 @@ class Instance(db.Model): id = db.Column(db.Integer, primary_key=True) domain = db.Column(db.String(255), unique=True, nullable=False, index=True) - api_key = db.Column(db.String(100), unique=True, nullable=False, index=True) created = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) updated = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) oprhan_since = db.Column(db.DateTime, nullable=True) @@ -51,6 +66,9 @@ class Instance(db.Model): endorsements = db.relationship("Endorsement", back_populates="endorsed_instance", cascade="all, delete-orphan", foreign_keys=[Endorsement.endorsed_id]) guarantees = db.relationship("Guarantee", back_populates="guarantor_instance", cascade="all, delete-orphan", foreign_keys=[Guarantee.guarantor_id]) guarantors = db.relationship("Guarantee", back_populates="guaranteed_instance", cascade="all, delete-orphan", foreign_keys=[Guarantee.guaranteed_id]) + rejections = db.relationship("RejectionRecord", back_populates="rejector_instance", cascade="all, delete-orphan", foreign_keys=[RejectionRecord.rejector_id]) + rejectors = db.relationship("RejectionRecord", back_populates="rejected_instance", cascade="all, delete-orphan", foreign_keys=[RejectionRecord.rejected_id]) + admins = db.relationship("Claim", back_populates="instance", cascade="all, delete-orphan") def create(self): db.session.add(self) diff --git a/overseer/classes/news.py b/fediseer/classes/news.py similarity index 100% rename from overseer/classes/news.py rename to fediseer/classes/news.py diff --git a/fediseer/classes/user.py b/fediseer/classes/user.py new file mode 100644 index 0000000..c4d16cb --- /dev/null +++ b/fediseer/classes/user.py @@ -0,0 +1,35 @@ +import uuid +import os + +import dateutil.relativedelta +from datetime import datetime +from sqlalchemy import Enum, UniqueConstraint +from sqlalchemy.dialects.postgresql import UUID + +from loguru import logger +from fediseer.flask import db, SQLITE_MODE + +uuid_column_type = lambda: UUID(as_uuid=True) if not SQLITE_MODE else db.String(36) + + +class Claim(db.Model): + __tablename__ = "claims" + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey("users.id", ondelete="CASCADE"), nullable=False) + user = db.relationship("User", back_populates="claims") + instance_id = db.Column(db.Integer, db.ForeignKey("instances.id", ondelete="CASCADE"), nullable=False) + instance = db.relationship("Instance", back_populates="admins") + created = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + +class User(db.Model): + __tablename__ = "users" + + id = db.Column(db.Integer, primary_key=True) + account = db.Column(db.String(255), unique=True, nullable=False, index=True) + username = db.Column(db.String(255), unique=False, nullable=False) + api_key = db.Column(db.String(100), unique=True, nullable=False, index=True) + created = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + updated = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + claims = db.relationship("Claim", back_populates="user", cascade="all, delete-orphan") diff --git a/fediseer/consts.py b/fediseer/consts.py new file mode 100644 index 0000000..aa4d6e5 --- /dev/null +++ b/fediseer/consts.py @@ -0,0 +1,5 @@ +OVERSEER_VERSION = "0.5.0" +SUPPORTED_SOFTWARE = [ + "lemmy", + "mastodon", +] \ No newline at end of file diff --git a/overseer/database/__init__.py b/fediseer/database/__init__.py similarity index 100% rename from overseer/database/__init__.py rename to fediseer/database/__init__.py diff --git a/overseer/database/functions.py b/fediseer/database/functions.py similarity index 60% rename from overseer/database/functions.py rename to fediseer/database/functions.py index 04fb7de..ccbedbf 100644 --- a/overseer/database/functions.py +++ b/fediseer/database/functions.py @@ -5,10 +5,11 @@ from loguru import logger from datetime import datetime, timedelta from sqlalchemy import func, or_, and_, not_, Boolean from sqlalchemy.orm import noload -from overseer.flask import db, SQLITE_MODE -from overseer.utils import hash_api_key +from fediseer.flask import db, SQLITE_MODE +from fediseer.utils import hash_api_key from sqlalchemy.orm import joinedload -from overseer.classes.instance import Instance, Endorsement, Guarantee +from fediseer.classes.instance import Instance, Endorsement, Guarantee, RejectionRecord +from fediseer.classes.user import Claim, User def get_all_instances(min_endorsements = 0, min_guarantors = 1): query = db.session.query( @@ -92,15 +93,78 @@ def get_all_guarantor_instances_by_guaranteed_id(guaranteed_id): def find_instance_by_api_key(api_key): - instance = Instance.query.filter_by(api_key=hash_api_key(api_key)).first() + instance = Instance.query.join( + Claim + ).join( + User + ).filter( + User.api_key == hash_api_key(api_key) + ).first() return instance +def find_instance_by_user(user): + instance = Instance.query.join( + Claim + ).join( + User + ).filter( + User.id == user.id + ).first() + return instance + +def find_instance_by_account(user_account): + instance = Instance.query.join( + Claim + ).join( + User + ).filter( + User.account == user_account + ).first() + return instance + +def find_admins_by_instance(instance): + users = User.query.join( + Claim + ).join( + Instance + ).filter( + Instance.id == instance.id + ).all() + return users + +def find_claim(admin_username): + claim = Claim.query.join( + User + ).filter( + User.account == admin_username + ).first() + return claim + +def find_user_by_api_key(api_key): + user = User.query.filter( + User.api_key == hash_api_key(api_key) + ).first() + return user + +def find_user_by_account(user_account): + user = User.query.filter( + User.account == user_account + ).first() + return user + def find_instance_by_domain(domain): instance = Instance.query.filter_by(domain=domain).first() return instance def find_authenticated_instance(domain,api_key): - instance = Instance.query.filter_by(domain=domain, api_key=hash_api_key(api_key)).first() + instance = Instance.query.join( + Claim + ).join( + User + ).filter( + User.api_key == hash_api_key(api_key), + Instance.domain ==domain, + ).first() return instance def get_endorsement(instance_id, endorsing_instance_id): @@ -110,6 +174,35 @@ def get_endorsement(instance_id, endorsing_instance_id): ) return query.first() +def has_recent_endorsement(instance_id): + query = Endorsement.query.filter( + Endorsement.endorsed_id == instance_id, + Endorsement.created > datetime.utcnow() - timedelta(hours=1), + ) + return query.first() + +def count_endorsements(instance_id): + query = Endorsement.query.filter_by( + endorsed_id=instance_id + ) + return query.count() + +def has_recent_rejection(instance_id, rejector_id): + query = RejectionRecord.query.filter_by( + rejected_id=instance_id, + rejector_id=rejector_id, + ).filter( + RejectionRecord.performed > datetime.utcnow() - timedelta(hours=24) + ) + return query.count() > 0 + +def get_rejection_record(rejector_id, rejected_id): + query = RejectionRecord.query.filter_by( + rejected_id=rejected_id, + rejector_id=rejector_id, + ) + return query.first() + def get_guarantee(instance_id, guarantor_id): query = Guarantee.query.filter_by( guaranteed_id=instance_id, diff --git a/overseer/exceptions.py b/fediseer/exceptions.py similarity index 100% rename from overseer/exceptions.py rename to fediseer/exceptions.py diff --git a/fediseer/fediverse.py b/fediseer/fediverse.py new file mode 100644 index 0000000..b70d166 --- /dev/null +++ b/fediseer/fediverse.py @@ -0,0 +1,37 @@ +import requests +from loguru import logger +from pythorhead import Lemmy + +def get_lemmy_admins(domain): + requested_lemmy = Lemmy(f"https://{domain}") + site = requested_lemmy.site.get() + if not site: + logger.warning(f"Error retrieving mastodon site info for {domain}") + return None + return [a["person"]["name"] for a in site["admins"]] + +def get_mastodon_admins(domain): + try: + site = requests(f"https://{domain}/api/v2/instance").json() + return [site["contact"]["account"]["username"]] + except Exception as err: + logger.warning(f"Error retrieving mastodon site info for {domain}") + return None + +def get_admin_for_software(software: str, domain: str): + software_map = { + "lemmy": get_lemmy_admins, + "mastodon": get_mastodon_admins, + } + if software not in software_map: + return None + return software_map[software](domain) + + +def get_nodeinfo(domain): + try: + wellknown = requests.get(f"https://{domain}/.well-known/nodeinfo", timeout=2).json() + nodeinfo = requests.get(wellknown['links'][0]['href'], timeout=2).json() + return nodeinfo + except Exception as err: + return None \ No newline at end of file diff --git a/overseer/flask.py b/fediseer/flask.py similarity index 88% rename from overseer/flask.py rename to fediseer/flask.py index 44dea2a..1b72f47 100644 --- a/overseer/flask.py +++ b/fediseer/flask.py @@ -1,5 +1,5 @@ import os -from flask import Flask +from flask import Flask, redirect from flask_caching import Cache from werkzeug.middleware.proxy_fix import ProxyFix from flask_sqlalchemy import SQLAlchemy @@ -13,7 +13,7 @@ SQLITE_MODE = os.getenv("USE_SQLITE", "0") == "1" if SQLITE_MODE: logger.warning("Using SQLite for database") - OVERSEER.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///overseer.db" + OVERSEER.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///fediseer.db" else: OVERSEER.config["SQLALCHEMY_DATABASE_URI"] = os.getenv('POSTGRES_URI') OVERSEER.config['SQLALCHEMY_ENGINE_OPTIONS'] = { @@ -38,3 +38,4 @@ if cache is None: cache = Cache(config=cache_config) cache.init_app(OVERSEER) logger.init_warn("Flask Cache", status="SimpleCache") + diff --git a/overseer/lemmy.py b/fediseer/lemmy.py similarity index 68% rename from overseer/lemmy.py rename to fediseer/lemmy.py index 7c9fe24..f8c0562 100644 --- a/overseer/lemmy.py +++ b/fediseer/lemmy.py @@ -2,13 +2,13 @@ from pythorhead import Lemmy from loguru import logger import os import secrets -import overseer.exceptions as e - -overctrl_lemmy = Lemmy(f"https://{os.getenv('OVERSEER_LEMMY_DOMAIN')}") -_login = overctrl_lemmy.log_in(os.getenv('OVERSEER_LEMMY_USERNAME'),os.getenv('OVERSEER_LEMMY_PASSWORD')) +import fediseer.exceptions as e + +overctrl_lemmy = Lemmy(f"https://{os.getenv('FEDISEER_LEMMY_DOMAIN')}") +_login = overctrl_lemmy.log_in(os.getenv('FEDISEER_LEMMY_USERNAME'),os.getenv('FEDISEER_LEMMY_PASSWORD')) if not _login: raise Exception("Failed to login to overctrl") -overseer_lemmy_user = overctrl_lemmy.user.get(username=os.getenv('OVERSEER_LEMMY_USERNAME')) +fediseer_lemmy_user = overctrl_lemmy.user.get(username=os.getenv('FEDISEER_LEMMY_USERNAME')) def pm_instance(domain: str, message: str): domain_username = domain.replace(".", "_") @@ -20,7 +20,7 @@ def pm_instance(domain: str, message: str): def pm_new_api_key(domain: str): api_key = secrets.token_urlsafe(16) - pm_content = f"The API Key for domain {domain} is\n\n{api_key}\n\nUse this to perform operations on the overseer." + pm_content = f"The API Key for domain {domain} is\n\n{api_key}\n\nUse this to perform operations on the fediseer." if not pm_instance(domain, pm_content): raise e.BadRequest("API Key PM failed") return api_key diff --git a/overseer/limiter.py b/fediseer/limiter.py similarity index 89% rename from overseer/limiter.py rename to fediseer/limiter.py index 6164ef4..8018188 100644 --- a/overseer/limiter.py +++ b/fediseer/limiter.py @@ -1,6 +1,6 @@ from flask_limiter import Limiter from flask_limiter.util import get_remote_address -from overseer.flask import OVERSEER +from fediseer.flask import OVERSEER from loguru import logger limiter = None diff --git a/overseer/logger.py b/fediseer/logger.py similarity index 94% rename from overseer/logger.py rename to fediseer/logger.py index 7e004c5..a34b183 100644 --- a/overseer/logger.py +++ b/fediseer/logger.py @@ -1,7 +1,7 @@ import sys from functools import partialmethod from loguru import logger -from overseer.argparser import args +from fediseer.argparser import args STDOUT_LEVELS = ["GENERATION", "PROMPT"] INIT_LEVELS = ["INIT", "INIT_OK", "INIT_WARN", "INIT_ERR"] @@ -100,7 +100,7 @@ config = { ], } logger.configure(**config) -logger.add("horde.log", retention="7 days", rotation="1d", compression="bz2", level=19) +logger.add("fediseer.log", retention="7 days", rotation="1d", compression="bz2", level=19) logger.disable("__main__") logger.warning("disabled") logger.enable("") diff --git a/fediseer/messaging.py b/fediseer/messaging.py new file mode 100644 index 0000000..63ecfa9 --- /dev/null +++ b/fediseer/messaging.py @@ -0,0 +1,137 @@ +import requests +import json +from datetime import datetime +import OpenSSL.crypto +import base64 +import hashlib +import uuid +import copy +import os +import secrets +import fediseer.exceptions as e +from pythorhead import Lemmy +from loguru import logger +from fediseer.database import functions as database +from fediseer.consts import SUPPORTED_SOFTWARE +from fediseer.fediverse import get_admin_for_software + +class ActivityPubPM: + private_key = None + def __init__(self): + with open('private.pem', 'rb') as file: + private_key_data = file.read() + self.private_key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, private_key_data) + self.document_core = { + "type": "Create", + "actor": "https://fediseer.com/api/v1/user/fediseer", + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https: //w3id.org/security/v1" + ], + "object": { + "attributedTo": "https://fediseer.com/api/v1/user/fediseer", + }, + } + + def send_pm_to_right_software(self, message, username, domain, software): + software_map = { + "lemmy": self.send_lemmy_pm, + "mastodon": self.send_mastodon_pm, + "fediseer": self.send_fediseer_pm, + } + return software_map[software](message, username, domain) + + def send_fediseer_pm(self, message, username, domain): + document = copy.deepcopy(self.document_core) + document["to"] = [f"https://lemmy.dbzer0.com/u/db0"] + document["object"]["type"] = "ChatMessage" + document["object"]["mediaType"] = "text/html" + document["object"]["to"] = [f"https://lemmy.dbzer0.com/u/db0"] + document["object"]["source"] = { + "content": message, + "mediaType": "text/markdown", + } + return self.send_pm(document, message, domain) + + def send_lemmy_pm(self, message, username, domain): + document = copy.deepcopy(self.document_core) + document["to"] = [f"https://{domain}/u/{username}"] + document["object"]["type"] = "ChatMessage" + document["object"]["mediaType"] = "text/html" + document["object"]["to"] = [f"https://{domain}/u/{username}"] + document["object"]["source"] = { + "content": message, + "mediaType": "text/markdown", + } + return self.send_pm(document, message, domain) + + def send_mastodon_pm(self, message, username, domain): + document = copy.deepcopy(self.document_core) + document["object"]["type"] = "Note" + document["object"]["to"] = [f"https://{domain}/u/{username}"] + return self.send_pm(document, message, domain) + + def send_pm(self, document, message, domain): + document["id"] = f"https://fediseer.com/{uuid.uuid4()}" + document["object"]["content"] = message + document["object"]["id"] = f"https://fediseer.com/{uuid.uuid4()}" + document["object"]["published"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + document = json.dumps(document, indent=4) + digest = hashlib.sha256(document.encode('utf-8')).digest() + encoded_digest = base64.b64encode(digest).decode('utf-8') + digest_header = "SHA-256=" + encoded_digest + date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + + signed_string = f"(request-target): post /inbox\nhost: {domain}\ndate: {date}\ndigest: {digest_header}" + signature = OpenSSL.crypto.sign(self.private_key, signed_string.encode('utf-8'), 'sha256') + encoded_signature = base64.b64encode(signature).decode('utf-8') + + header = f'keyId="https://fediseer.com/api/v1/user/fediseer",headers="(request-target) host date digest",signature="{encoded_signature}"' + headers = { + 'Host': domain, + 'Date': date, + 'Signature': header, + 'Digest': digest_header, + 'Content-Type': 'application/ld+json; profile="http://www.w3.org/ns/activitystreams"' + } + url = f"https://{domain}/inbox" + response = requests.post(url, data=document, headers=headers) + return response.ok + + def pm_new_api_key(self, domain: str, username: str, software: str, requestor = None): + api_key = secrets.token_urlsafe(16) + if requestor: + pm_content = f"user '{requestor}' has initiated an API Key reset for your domain {domain} on the [Fediseer](https://fediseer.com)\n\nThe new API key is\n\n{api_key}" + else: + pm_content = f"Your API Key for domain {domain} is\n\n{api_key}\n\nUse this to perform operations on the [Fediseer](https://fediseer.com)." + if not self.send_pm_to_right_software( + message=pm_content, + username=username, + domain=domain, + software=software + ): + raise e.BadRequest("API Key PM failed") + return api_key + + + def pm_admins(self, message: str, domain: str, software: str, instance): + if software not in SUPPORTED_SOFTWARE: + return None + admins = database.find_admins_by_instance(instance) + if not admins: + admins = get_admin_for_software(software, domain) + else: + admins = [a.username for a in admins] + if not admins: + raise e.BadRequest(f"Could not determine admins for {domain}") + for admin_username in admins: + if not self.send_pm_to_right_software( + message=message, + username=admin_username, + domain=domain, + software=software + ): + raise e.BadRequest("Admin PM Failed") + + +activitypub_pm = ActivityPubPM() diff --git a/overseer/observer.py b/fediseer/observer.py similarity index 100% rename from overseer/observer.py rename to fediseer/observer.py diff --git a/overseer/routes.py b/fediseer/routes.py similarity index 79% rename from overseer/routes.py rename to fediseer/routes.py index 6ce4c9f..9bb4b3a 100644 --- a/overseer/routes.py +++ b/fediseer/routes.py @@ -1,13 +1,14 @@ from flask import render_template, redirect, url_for, request from markdown import markdown from loguru import logger -from overseer.flask import OVERSEER +from fediseer.flask import OVERSEER +import fediseer.exceptions as e @logger.catch(reraise=True) @OVERSEER.route('/') # @cache.cached(timeout=300) def index(): - with open(f'overseer/templates/index.md') as index_file: + with open(f'fediseer/templates/index.md') as index_file: index = index_file.read() findex = index.format() @@ -31,23 +32,21 @@ def index(): """ return(head + markdown(findex)) - @logger.catch(reraise=True) @OVERSEER.route('/.well-known/webfinger') def wellknown_redirect(): query_string = request.query_string.decode() if not query_string: return {"message":"No user specified"},400 - if query_string != "resource=acct:overseer@overseer.dbzer0.com": + if query_string != "resource=acct:fediseer@fediseer.com": return {"message":"User does not exist"},404 webfinger = { - "subject": "acct:overseer@overseer.dbzer0.com", - + "subject": "acct:fediseer@fediseer.com", "links": [ { "rel": "self", "type": "application/activity+json", - "href": "https://overseer.dbzer0.com/api/v1/user/overseer" + "href": "https://fediseer.com/api/v1/user/fediseer" } ] } diff --git a/overseer/templates/index.md b/fediseer/templates/index.md similarity index 81% rename from overseer/templates/index.md rename to fediseer/templates/index.md index 722b209..8683b3e 100644 --- a/overseer/templates/index.md +++ b/fediseer/templates/index.md @@ -1,8 +1,8 @@ # Lemmy Overseer -This is a [FOSS service](https://github.com/db0/lemmy-overseer) to help Lemmy instances detect and avoid suspcicious instances +This is a [FOSS service](https://github.com/db0/lemmy-fediseer) to help Lemmy instances detect and avoid suspcicious instances -[Release Devlog](https://dbzer0.com/blog/overseer-a-fediverse-chain-of-trust/) +[Release Devlog](https://dbzer0.com/blog/fediseer-a-fediverse-chain-of-trust/) ## Scope diff --git a/overseer/utils.py b/fediseer/utils.py similarity index 86% rename from overseer/utils.py rename to fediseer/utils.py index 06c879e..e54e107 100644 --- a/overseer/utils.py +++ b/fediseer/utils.py @@ -9,9 +9,7 @@ import json from datetime import datetime import dateutil.relativedelta from loguru import logger -from overseer.flask import SQLITE_MODE -import requests - +from fediseer.flask import SQLITE_MODE random.seed(random.SystemRandom().randint(0, 2**32 - 1)) @@ -102,11 +100,3 @@ def validate_regex(regex_string): except: return False return True - -def get_nodeinfo(domain): - try: - wellknown = requests.get(f"https://{domain}/.well-known/nodeinfo", timeout=2).json() - nodeinfo = requests.get(wellknown['links'][0]['href'], timeout=2).json() - return nodeinfo - except Exception as err: - return None \ No newline at end of file diff --git a/overseer/classes/__init__.py b/overseer/classes/__init__.py deleted file mode 100644 index 82a7f18..0000000 --- a/overseer/classes/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -from loguru import logger -from overseer.argparser import args -from importlib import import_module -from overseer.flask import db, OVERSEER -from overseer.utils import hash_api_key - -# Importing for DB creation -from overseer.classes.instance import Instance, Guarantee - -with OVERSEER.app_context(): - - db.create_all() - - admin_domain = os.getenv("OVERSEER_LEMMY_DOMAIN") - admin = db.session.query(Instance).filter_by(domain=admin_domain).first() - if not admin: - admin = Instance( - id=0, - domain=admin_domain, - api_key=hash_api_key(os.getenv("ADMIN_API_KEY")), - open_registrations=False, - email_verify=False, - software="lemmy", - ) - admin.create() - guarantee = Guarantee( - guarantor_id = admin.id, - guaranteed_id = admin.id, - ) - db.session.add(guarantee) - db.session.commit() \ No newline at end of file diff --git a/overseer/consts.py b/overseer/consts.py deleted file mode 100644 index 47b796e..0000000 --- a/overseer/consts.py +++ /dev/null @@ -1 +0,0 @@ -OVERSEER_VERSION = "0.0.1" diff --git a/quickrun.py b/quickrun.py index b7769f7..9fa5bc5 100644 --- a/quickrun.py +++ b/quickrun.py @@ -1,5 +1,5 @@ import json -from overseer.observer import retrieve_suspicious_instances +from fediseer.observer import retrieve_suspicious_instances sus = retrieve_suspicious_instances(20) if sus: diff --git a/server.py b/server.py index 20723bd..bc8454a 100644 --- a/server.py +++ b/server.py @@ -4,8 +4,8 @@ import logging load_dotenv() -from overseer.argparser import args -from overseer.flask import OVERSEER +from fediseer.argparser import args +from fediseer.flask import OVERSEER from loguru import logger if __name__ == "__main__": diff --git a/test.py b/test.py new file mode 100644 index 0000000..c6cdd9c --- /dev/null +++ b/test.py @@ -0,0 +1,41 @@ +import requests +import json +from datetime import datetime +import OpenSSL.crypto +import base64 +import hashlib +import sys +import uuid + +with open('lemmy-hello-world.json', 'r') as file: + document = json.loads(file.read()) +document["id"] = f"https://fediseer.com/{uuid.uuid4()}" +document["object"]["id"] = f"https://fediseer.com/{uuid.uuid4()}" +document["object"]["published"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") +document = json.dumps(document, indent=4) +print(document) +digest = hashlib.sha256(document.encode('utf-8')).digest() +encoded_digest = base64.b64encode(digest).decode('utf-8') +digest_header = "SHA-256=" + encoded_digest +date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + +with open('private.pem', 'rb') as file: + private_key_data = file.read() + private_key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, private_key_data) + +signed_string = f"(request-target): post /inbox\nhost: overctrl.dbzer0.com\ndate: {date}\ndigest: {digest_header}" +signature = OpenSSL.crypto.sign(private_key, signed_string.encode('utf-8'), 'sha256') +encoded_signature = base64.b64encode(signature).decode('utf-8') + +header = f'keyId="https://fediseer.com/api/v1/user/fediseer",headers="(request-target) host date digest",signature="{encoded_signature}"' +headers = { + 'Host': 'overctrl.dbzer0.com', + 'Date': date, + 'Signature': header, + 'Digest': digest_header, + 'Content-Type': 'application/ld+json; profile="http://www.w3.org/ns/activitystreams"' +} +url = 'https://overctrl.dbzer0.com/inbox' +response = requests.post(url, data=document, headers=headers) +print('Response Status:', response.status_code) +print('Response Body:', response.text)