fix: handle bot blocking siteinfo

Closes #51

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+# 0.20.1
+
+* Allow filtering by software
+
 # 0.20.0
 
 * Added batching for adding/removing/modifying censures

fediseer/apis/v1/whitelist.py

@@ -12,6 +12,7 @@ class Whitelist(Resource):
     get_parser.add_argument("endorsements", required=False, default=0, type=int, help="Limit to this amount of endorsements of more", location="args")
     get_parser.add_argument("guarantors", required=False, default=1, type=int, help="Limit to this amount of guarantors of more", location="args")
     get_parser.add_argument("tags_csv", required=False, type=str, help="A list of tags to filter.", location="args")
+    get_parser.add_argument("software_csv", required=False, type=str, help="show only instances running one of this software", location="args")
     get_parser.add_argument("page", required=False, type=int, default=1, help="Which page of results to retrieve", location="args")
     get_parser.add_argument("limit", required=False, type=int, default=100, help="Which page of results to retrieve", location="args")
     get_parser.add_argument("csv", required=False, type=bool, help="Set to true to return just the domains as a csv. Mutually exclusive with domains", location="args")
@@ -31,11 +32,15 @@ class Whitelist(Resource):
         tags = None
         if self.args.tags_csv is not None:
             tags = [t.strip() for t in self.args.tags_csv.split(',')]
+        software = None
+        if self.args.software_csv is not None:
+            software = [s.strip() for s in self.args.software_csv.split(',')]
         instance_details = []
         for instance in database.get_all_instances(
             min_endorsements=self.args.endorsements,
             min_guarantors=self.args.guarantors,
             tags=tags,
+            software=software,
             page=self.args.page,
             limit=self.args.limit,
         ):

fediseer/database/functions.py

@@ -13,6 +13,7 @@ def get_all_instances(
         min_endorsements = 0, 
         min_guarantors = 1, 
         tags = None,
+        software = None,
         include_decommissioned = True,
         page=1,
         limit=10,
@@ -41,6 +42,9 @@ def get_all_instances(
         # Convert tags to lowercase and filter instances that have any of the tags
         lower_tags = [tag.lower() for tag in tags]
         query = query.filter(Instance.tags.any(func.lower(InstanceTag.tag).in_(lower_tags)))
+    if software:
+        lower_sw = [sw.lower() for sw in software]
+        query = query.filter(Instance.software.in_(lower_sw))
     page -= 1
     if page < 0:
         page = 0

fediseer/faq/eng.py

@@ -36,7 +36,7 @@ The fediseer provides a machine readable API to consume the data contained withi
         "question": "What is a guarantee?",
         "stub": "guarantee",
         "document": 
-"""Basically, any guaranteed instance is known as definitelly "not spam" (AKA "ham"). That doesn't mean any non-guaranteed instance is spam. Rather it is considered "unknown". The only reasoning to guarantee an instance is whether they are spam or not. The objective here being to prevent malicious actors from spawning an infinite amount of new instances on the fediverse to send spam.
+"""Basically, any guaranteed instance is known as definitely "not spam" (AKA "ham"). That doesn't mean any non-guaranteed instance is spam. Rather it is considered "unknown". The only reasoning to guarantee an instance is whether they are spam or not. The objective here being to prevent malicious actors from spawning an infinite amount of new instances on the fediverse to send spam.
 
 Each instance can only be guaranteed by one instance and guarantee another instance. This is called the "chain of trust"
 
@@ -96,7 +96,7 @@ One can export the list of instances censured by a subset of instances.
         "question": "What is a hesitation?",
         "stub": "hesitation",
         "document": 
-"""An hesitation is a mulder version of a censure, it signifies some sort of mistrust of one instance towards another. The reason for this can be anything and do not have to be stated.
+"""An hesitation is a milder version of a censure, it signifies some sort of mistrust of one instance towards another. The reason for this can be anything and do not have to be stated.
 
 An instance can be hesitate against number of instances and be doubted by number of instances.
 
@@ -111,7 +111,7 @@ One can export the list of instances hesitate by a subset of instances.
         "question": "What is an instance claim?",
         "stub": "claim",
         "document": 
-"""A claimed instance is an instance whose admin has requisted an API key with which to use the fediseer as their instance.
+"""A claimed instance is an instance whose admin has requested an API key with which to use the fediseer as their instance.
 
 Fediseer has no users. Instead it's driven by instance admins only. Instance admins likewise only act as their instances.
 """
@@ -141,7 +141,7 @@ Note that guarantees are always public as this is necessary for the good functio
         "question": "What is an instance flag?",
         "stub": "flag",
         "document": 
-"""An instance flag represents some marking from the fediverse admins toward an instance. There's currently the following flags
+"""An instance flag represents some marking from the fediverse admins toward an instance. There's currently the following flags:
 
 * `RESTRICTED`: The instance cannot guarantee, endorse, censure or hesitate other instances anymore. This flag is only used against egregious trolling or malicious behaviour.
 * `MUTED`: The instance's visibilities are forcefully set to `PRIVATE` and cannot be changed. This flag is meant to used against trolling or harassing behaviour.
@@ -172,7 +172,7 @@ Like always, no hate speech is allowed.
         "question": "How can I claim my instance?",
         "stub": "instance claim",
         "document": 
-"""You can either use the rest API we have provided, providing your instance domain and admin username on it. Alternativey you can use one of our frontends.
+"""You can either use the rest API we have provided, providing your instance domain and admin username on it. Alternatively you can use one of our frontends.
 
 You will then receive an API key in PMs, which you can afterwards use to represent your instance on the fediseer.
 """
@@ -227,4 +227,4 @@ Not only that, but the fediseer is free and open source software, allowing anyon
 """The fediseer is designed to be completely crowd-sourced as pertains to the chain of trust. The admin of the fediseer does not control what people guarantee, endorse or censure. The various instance admins are driving the chain of trust.
 """
     },
-]
+]

fediseer/fediverse.py

@@ -176,7 +176,12 @@ class InstanceInfo():
 
     def get_mastodon_info(self):
         site = requests.get(f"https://{self.domain}/api/v1/instance",timeout=self._req_timeout)
-        self.instance_info = site.json()
+        try:
+            self.instance_info = site.json()
+        except Exception as err:
+            if "challenge-error-text" in site.text:
+                raise Exception("Instance is preventing scripted retrieval of their site info.")
+            raise err
         self.approval_required = self.instance_info["approval_required"]
         if self.node_info is None:
             raise Exception("Error retrieving nodeinfo")
@@ -186,7 +191,12 @@ class InstanceInfo():
 
     def get_pleroma_info(self):
         site = requests.get(f"https://{self.domain}/api/v1/instance",timeout=self._req_timeout)
-        self.instance_info = site.json()
+        try:
+            self.instance_info = site.json()
+        except Exception as err:
+            if "challenge-error-text" in site.text:
+                raise Exception("Instance is preventing scripted retrieval of their site info.")
+            raise err
         self.approval_required = self.instance_info["approval_required"]
         if self.node_info is None:
             raise Exception("Error retrieving nodeinfo")
@@ -196,7 +206,12 @@ class InstanceInfo():
 
     def get_firefish_info(self):
         site = requests.get(f"https://{self.domain}/api/v1/instance",timeout=self._req_timeout)
-        self.instance_info = site.json()
+        try:
+            self.instance_info = site.json()
+        except Exception as err:
+            if "challenge-error-text" in site.text:
+                raise Exception("Instance is preventing scripted retrieval of their site info.")
+            raise err
         self.approval_required = self.instance_info["approval_required"]
         if self.node_info is None:
             raise Exception("Error retrieving nodeinfo")
@@ -213,7 +228,12 @@ class InstanceInfo():
         site = requests.get(f"https://{self.domain}/api/v1/instance",timeout=self._req_timeout,allow_redirects=False)
         if site.status_code != 200:
             raise Exception(f"Unexpected status code retrieved when discovering instance info: {site.status_code}")
-        self.instance_info = site.json()
+        try:
+            self.instance_info = site.json()
+        except Exception as err:
+            if "challenge-error-text" in site.text:
+                raise Exception("Instance is preventing scripted retrieval of their site info.")
+            raise err
         self.approval_required = self.instance_info.get("approval_required")
         if self.node_info is None:
             raise Exception("Error retrieving nodeinfo")
@@ -235,7 +255,7 @@ class InstanceInfo():
                 if "*" in self.domain:
                     self.software = "wildcard"
         else:
-            self.software = self.node_info["software"]["name"]
+            self.software = self.node_info["software"]["name"].lower()
         software_map = {
             "lemmy": self.get_lemmy_info,
             "mastodon": self.get_mastodon_info,

fediseer/register.py

@@ -26,7 +26,7 @@ def ensure_instance_registered(domain, allow_unreachable=False, record_unreachab
                     instance.poll_failures += 60
                 db.session.commit()
         if not allow_unreachable:
-            raise e.BadRequest(f"Error encountered while polling domain {domain}. Please check it's running correctly")
+            raise e.BadRequest(str(err))
     if instance:
         if (
             instance.software != instance_info.software or