64 lines
2.6 KiB
Python
64 lines
2.6 KiB
Python
from dotenv import load_dotenv
|
|
import os
|
|
import logging
|
|
|
|
load_dotenv()
|
|
|
|
from loguru import logger
|
|
from fediseer.flask import OVERSEER, db
|
|
import fediseer.database.functions as database
|
|
from fediseer.register import ensure_instance_registered
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
def refresh_info(domain):
|
|
logger.info(f"Refreshing domain '{domain}")
|
|
with OVERSEER.app_context():
|
|
try:
|
|
ensure_instance_registered(
|
|
domain,
|
|
# We don't want to set allow_unreachable = True here
|
|
# As InstanceInfo() won't raise an exception when failing
|
|
# Which will cause the poll_failures not not increment
|
|
allow_unreachable=False,
|
|
record_unreachable=True,
|
|
allowed_timeout=20
|
|
)
|
|
except Exception as err:
|
|
pass
|
|
|
|
if __name__ == "__main__":
|
|
logger.add("updater.log", retention="7 days", rotation="1d", compression="bz2", level=10)
|
|
# Only setting this for the WSGI logs
|
|
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(module)s:%(lineno)d - %(message)s', level=logging.WARNING)
|
|
logger.init("Updater", status="Starting")
|
|
logger.info(f"Skipping instances with poll fails > {int(os.getenv('FEDISEER_IGNORE_POLL_FAILS', 0))}")
|
|
# try: # Debug
|
|
# with OVERSEER.app_context():
|
|
# ensure_instance_registered(
|
|
# "firefish.social",
|
|
# allow_unreachable=False,
|
|
# record_unreachable=True,
|
|
# allowed_timeout=20
|
|
# )
|
|
# except Exception as err:
|
|
# logger.error(err)
|
|
futures = []
|
|
with ThreadPoolExecutor(max_workers=int(os.getenv('FEDISEER_UPDATE_THREADS', 25))) as executor:
|
|
with OVERSEER.app_context():
|
|
for instance in database.get_all_instances(0,0,limit=1000000):
|
|
if instance.software == 'wildcard':
|
|
continue
|
|
# -1 doesn't skip anything
|
|
if int(os.getenv('FEDISEER_IGNORE_POLL_FAILS', -1)) >= 0 and instance.poll_failures > int(os.getenv('FEDISEER_IGNORE_POLL_FAILS', 0)):
|
|
logger.debug(f"Skipped {instance.domain} due to too many poll fails.")
|
|
continue
|
|
futures.append(executor.submit(refresh_info, instance.domain))
|
|
if len(futures) >= 500:
|
|
for future in futures:
|
|
future.result()
|
|
futures = []
|
|
for future in futures:
|
|
future.result()
|
|
|
|
logger.init("Updater", status="Ended")
|
|
[] |