From c049c20376ab33c14a7d302830eee548eaf5b4f0 Mon Sep 17 00:00:00 2001 From: Aevann Date: Fri, 23 Feb 2024 22:15:21 +0200 Subject: [PATCH] notify for soyjak.party mentions --- files/classes/user.py | 2 +- files/helpers/cron.py | 6 +- files/helpers/offsite_mentions/fourchan.py | 50 ----------- files/helpers/offsite_mentions/lemmy.py | 56 ------------- files/helpers/offsite_mentions/other.py | 98 ++++++++++++++++++++++ files/routes/notifications.py | 2 +- 6 files changed, 104 insertions(+), 110 deletions(-) delete mode 100644 files/helpers/offsite_mentions/fourchan.py delete mode 100644 files/helpers/offsite_mentions/lemmy.py create mode 100644 files/helpers/offsite_mentions/other.py diff --git a/files/classes/user.py b/files/classes/user.py index aba6b3439..5964126c2 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -891,7 +891,7 @@ class User(Base): return g.db.query(Comment).filter( Comment.created_utc > self.last_viewed_offsite_notifs, Comment.is_banned == False, Comment.deleted_utc == 0, - Comment.body_html.like('

New site mention by%'), + Comment.body_html.like('

New site mention%'), Comment.parent_post == None, Comment.author_id == AUTOJANNY_ID).count() @property diff --git a/files/helpers/cron.py b/files/helpers/cron.py index 17ef420ff..1694e4140 100644 --- a/files/helpers/cron.py +++ b/files/helpers/cron.py @@ -27,8 +27,7 @@ from files.helpers.roulette import spin_roulette_wheel from files.helpers.sanitize import filter_emojis_only, sanitize from files.helpers.useractions import * from files.helpers.offsite_mentions.reddit import * -from files.helpers.offsite_mentions.lemmy import * -from files.helpers.offsite_mentions.fourchan import * +from files.helpers.offsite_mentions.other import * from files.cli import app, db_session, g @@ -70,6 +69,9 @@ def cron_fn(every_5m, every_1d, every_1mo): fourchan_mentions_task() g.db.commit() + soyjak_mentions_task() + g.db.commit() + if every_1d or (not cache.get('stats') and not IS_LOCALHOST): if IS_HOMOWEEN(): g.db.execute(text( diff --git a/files/helpers/offsite_mentions/fourchan.py b/files/helpers/offsite_mentions/fourchan.py deleted file mode 100644 index 36ca36809..000000000 --- a/files/helpers/offsite_mentions/fourchan.py +++ /dev/null @@ -1,50 +0,0 @@ -import requests -from flask import g - -from files.helpers.config.const import * -from files.classes.comment import Comment -from files.helpers.sanitize import * -from files.helpers.alerts import push_notif -from files.classes.notifications import Notification - -def fourchan_mentions_task(): - queries = OFFSITE_NOTIF_QUERIES - {'r/drama'} - for q in queries: - url = f'https://archived.moe/_/api/chan/search?text={q}' - data = requests.get(url, headers=HEADERS, timeout=5).json()['0']['posts'] - - for thing in data: - board = thing['board']['shortname'] - author_string = thing['name'] - num = thing["num"] - thread_num = thing["thread_num"] - - if num != thread_num: - text = f'

{thing["comment"]}

' - permalink = f'https://archived.moe/{board}/thread/{thread_num}/#{num}' - else: - text = f'

{thing["title"]}


{thing["comment"]}

' - permalink = f'https://archived.moe/{board}/thread/{thread_num}' - - text = f'New site mention by {author_string}\n\n{permalink}\n\n{text}' - text = sanitize(text, blackjack="fourchan mention", golden=False) - - existing_comment = g.db.query(Comment.id).filter_by( - author_id=AUTOJANNY_ID, - parent_post=None, - body_html=text).one_or_none() - if existing_comment: break - - created_utc = thing["timestamp"] - - new_comment = Comment( - author_id=AUTOJANNY_ID, - parent_post=None, - body_html=text, - distinguished=True, - created_utc=created_utc, - ) - - g.db.add(new_comment) - g.db.flush() - new_comment.top_comment_id = new_comment.id diff --git a/files/helpers/offsite_mentions/lemmy.py b/files/helpers/offsite_mentions/lemmy.py deleted file mode 100644 index 66799b841..000000000 --- a/files/helpers/offsite_mentions/lemmy.py +++ /dev/null @@ -1,56 +0,0 @@ -import requests -from flask import g - -from files.helpers.config.const import * -from files.classes.comment import Comment -from files.helpers.sanitize import * -from files.helpers.alerts import push_notif -from files.classes.notifications import Notification - -def lemmy_mentions_task(): - for q in OFFSITE_NOTIF_QUERIES: - url = f'https://lemm.ee/api/v3/search?q={q}' - data = requests.get(url, headers=HEADERS, timeout=5).json() - - for kind in ("post", "comment"): - for thing in data[f'{kind}s']: - creator = thing['creator'] - author_string = f"[/u/{creator['name']}]({creator['actor_id']})" - thing = thing[kind] - if kind == 'comment': - body = thing["content"] - text = f'

{body}

' - else: - title = thing["name"] - text = f'

{title}

' - - if thing.get("body"): - selftext = thing["body"][:5000] - text += f'

{selftext}

' - - if 'erdrama' in text: continue - - permalink = thing['ap_id'] - text = f'New site mention by {author_string}\n\n{permalink}\n\n{text}' - text = sanitize(text, blackjack="lemmy mention", golden=False) - - existing_comment = g.db.query(Comment.id).filter_by( - author_id=AUTOJANNY_ID, - parent_post=None, - body_html=text).one_or_none() - if existing_comment: break - - try: created_utc = int(time.mktime(time.strptime(thing['published'].split('.')[0], "%Y-%m-%dT%H:%M:%S"))) - except: created_utc = int(time.mktime(time.strptime(thing['published'].split('.')[0], "%Y-%m-%dT%H:%M:%SZ"))) - - new_comment = Comment( - author_id=AUTOJANNY_ID, - parent_post=None, - body_html=text, - distinguished=True, - created_utc=created_utc, - ) - - g.db.add(new_comment) - g.db.flush() - new_comment.top_comment_id = new_comment.id diff --git a/files/helpers/offsite_mentions/other.py b/files/helpers/offsite_mentions/other.py new file mode 100644 index 000000000..dab2d9202 --- /dev/null +++ b/files/helpers/offsite_mentions/other.py @@ -0,0 +1,98 @@ +import requests +from flask import g + +from files.helpers.config.const import * +from files.classes.comment import Comment +from files.helpers.sanitize import * +from files.helpers.alerts import push_notif +from files.classes.notifications import Notification + +def notify(text, created_utc): + text = sanitize(text, blackjack="offsite mention", golden=False) + + existing_comment = g.db.query(Comment.id).filter_by( + author_id=AUTOJANNY_ID, + parent_post=None, + body_html=text).one_or_none() + + if existing_comment: + return 1 + + new_comment = Comment( + author_id=AUTOJANNY_ID, + parent_post=None, + body_html=text, + distinguished=True, + created_utc=created_utc, + ) + + g.db.add(new_comment) + g.db.flush() + new_comment.top_comment_id = new_comment.id + + +def lemmy_mentions_task(): + for q in OFFSITE_NOTIF_QUERIES: + url = f'https://lemm.ee/api/v3/search?q={q}' + data = requests.get(url, headers=HEADERS, timeout=5).json() + + for kind in ("post", "comment"): + for thing in data[f'{kind}s']: + creator = thing['creator'] + author_string = f"[/u/{creator['name']}]({creator['actor_id']})" + thing = thing[kind] + if kind == 'comment': + body = thing["content"] + text = f'

{body}

' + else: + title = thing["name"] + text = f'

{title}

' + + if thing.get("body"): + selftext = thing["body"][:5000] + text += f'

{selftext}

' + + if 'erdrama' in text: continue + + permalink = thing['ap_id'] + text = f'New site mention by {author_string}\n\n{permalink}\n\n{text}' + try: created_utc = int(time.mktime(time.strptime(thing['published'].split('.')[0], "%Y-%m-%dT%H:%M:%S"))) + except: created_utc = int(time.mktime(time.strptime(thing['published'].split('.')[0], "%Y-%m-%dT%H:%M:%SZ"))) + if notify(text, created_utc) == 1: break + + +def fourchan_mentions_task(): + queries = OFFSITE_NOTIF_QUERIES - {'r/drama'} + for q in queries: + url = f'https://archived.moe/_/api/chan/search?text={q}' + data = requests.get(url, headers=HEADERS).json()['0']['posts'] + + for thing in data: + board = thing['board']['shortname'] + author_string = thing['name'] + num = thing["num"] + thread_num = thing["thread_num"] + + if num != thread_num: + text = f'

{thing["comment"]}

' + permalink = f'https://archived.moe/{board}/thread/{thread_num}/#{num}' + else: + text = f'

{thing["title"]}


{thing["comment"]}

' + permalink = f'https://archived.moe/{board}/thread/{thread_num}' + + text = f'New site mention by {author_string}\n\n{permalink}\n\n{text}' + created_utc = thing["timestamp"] + if notify(text, created_utc) == 1: break + + +def soyjak_mentions_task(): + for q in OFFSITE_NOTIF_QUERIES: + url = f'https://api.marge.moe/search?q={q}' + data = requests.get(url, headers={"User-Agent": "MarseyFromWPD"}, proxies=proxies).json()['results'] + + for thing in data: + text = f'

{thing["comment"]}

' + permalink = thing['url'] + text = f'New site mention\n\n{permalink}\n\n{text}' + created_utc = int(time.mktime(time.strptime(thing['date'].split('.')[0], "%Y-%m-%dT%H:%M:%S"))) + if notify(text, created_utc) == 1: break diff --git a/files/routes/notifications.py b/files/routes/notifications.py index 591a112e5..c03e02a7f 100644 --- a/files/routes/notifications.py +++ b/files/routes/notifications.py @@ -278,7 +278,7 @@ def notifications_offsite(v): if not v.can_view_offsite_mentions: abort(403) listing = g.db.query(Comment).filter( - Comment.body_html.like('

New site mention by%'), + Comment.body_html.like('

New site mention%'), Comment.parent_post == None, Comment.author_id == AUTOJANNY_ID )