From 995375deccbb1eaa87e9a8e907f6cc28a0de37f6 Mon Sep 17 00:00:00 2001 From: TLSM Date: Tue, 7 Jun 2022 10:42:24 -0400 Subject: [PATCH] Refactor reddit mentions, move to cron. The reddit mentions system contained much duplicated code and was grafted onto the post thumbnail pipeline to achieve semi-regular invocation. Instead, we now run it through the new cron system, and the duplicate code has been refactored out. --- files/helpers/const.py | 23 +++++--- files/helpers/cron.py | 3 + files/helpers/offsitementions.py | 74 +++++++++++++++++++++++++ files/routes/posts.py | 94 -------------------------------- 4 files changed, 92 insertions(+), 102 deletions(-) create mode 100644 files/helpers/offsitementions.py diff --git a/files/helpers/const.py b/files/helpers/const.py index f45237d89..00e55170b 100644 --- a/files/helpers/const.py +++ b/files/helpers/const.py @@ -799,14 +799,21 @@ FACTCHECK_REPLIES = ('Factcheck: This claim has been co if SITE_NAME == 'rDrama': patron = 'Paypig' else: patron = 'Patron' -REDDIT_NOTIFS = { - 'idio3': IDIO_ID, - 'aevann': AEVANN_ID, - 'carpflo': CARP_ID, - 'carpathianflorist': CARP_ID, - 'carpathian florist': CARP_ID, - 'the_homocracy': HOMO_ID -} +REDDIT_NOTIFS_SITE = [] +REDDIT_NOTIFS_USERS = {} + +if SITE_NAME == 'rDrama': + REDDIT_NOTIFS_SITE = ['rdrama', 'marsey',] + REDDIT_NOTIFS_USERS = { + 'idio3': IDIO_ID, + 'aevann': AEVANN_ID, + 'carpflo': CARP_ID, + 'carpathianflorist': CARP_ID, + 'carpathian florist': CARP_ID, + 'the_homocracy': HOMO_ID, + } +elif SITE_NAME == 'PCM': + REDDIT_NOTIFS_SITE = ['pcmemes.net',] discounts = { # Big Spender badges, 2pp additive discount each diff --git a/files/helpers/cron.py b/files/helpers/cron.py index aae3d8f5e..a0546a063 100644 --- a/files/helpers/cron.py +++ b/files/helpers/cron.py @@ -1,7 +1,9 @@ from files.cli import g, app, db_session import click import files.helpers.const as const + import files.helpers.lottery as lottery +import files.helpers.offsitementions as offsitementions import files.helpers.stats as stats @app.cli.command('cron', help='Run scheduled tasks.') @@ -13,6 +15,7 @@ def cron(every_5m, every_1h, every_1d): if every_5m: lottery.check_if_end_lottery_task() + offsitementions.offsite_mentions_task() if every_1h: pass diff --git a/files/helpers/offsitementions.py b/files/helpers/offsitementions.py new file mode 100644 index 000000000..b1415123f --- /dev/null +++ b/files/helpers/offsitementions.py @@ -0,0 +1,74 @@ +from flask import g +import itertools +import requests +import files.helpers.const as const +from files.classes.user import User +from files.classes.comment import Comment +from files.classes.notifications import Notification + +# https://api.pushshift.io/meta provides key server_ratelimit_per_minute +# At time of writing, the ratelimit is 120 req/min. We get nowhere near this +# with current keyword quantities. If this ever changes, consider reading the +# value from /meta and doing a random selection of keywords. + +def offsite_mentions_task(): + if const.REDDIT_NOTIFS_SITE: + # Site-specific logic: send to JL1+, except on PCM JL3+ + jl_min = 3 if const.SITE_NAME == 'PCM' else 1 + row_send_to = g.db.query(User.id).filter(User.admin_level >= jl_min).all() + send_to = [x[0] for x in row_send_to] + + site_mentions = get_mentions(const.REDDIT_NOTIFS_SITE) + notify_mentions(send_to, site_mentions) + + if const.REDDIT_NOTIFS_USERS: + for query, send_user in const.REDDIT_NOTIFS_USERS.items(): + if not send_user: continue + + user_mentions = get_mentions([query]) + notify_mentions([send_user], user_mentions, mention_str='mention of you') + +def get_mentions(queries): + kinds = ['submission', 'comment'] + mentions = [] + for kind, query in itertools.product(kinds, queries): + try: + data = requests.get(f'https://api.pushshift.io/reddit/{kind}/search' + + f'?html_decode=true&q={query}&size=1', timeout=5).json()['data'] + except: break + + for i in data: + # Special case: PokemonGoRaids says 'Marsey' a lot unrelated to us. + if i['subreddit'] == 'PokemonGoRaids': continue + + mentions.append(i['permalink']) + + return mentions + +def notify_mentions(send_to, mentions, mention_str='site mention'): + for m in mentions: + notif_text = f'

New {mention_str}: ' \ + f'https://old.reddit.com{m}?context=89

' + + existing_comment = g.db.query(Comment.id).filter_by( + author_id=const.NOTIFICATIONS_ID, + parent_submission=None, + body_html=notif_text).one_or_none() + if existing_comment: break + + new_comment = Comment( + author_id=const.NOTIFICATIONS_ID, + parent_submission=None, + body_html=notif_text, + distinguish_level=6) + g.db.add(new_comment) + g.db.flush() + new_comment.top_comment_id = new_comment.id + + for user_id in send_to: + notif = Notification(comment_id=new_comment.id, user_id=user_id) + g.db.add(notif) + + g.db.commit() + diff --git a/files/routes/posts.py b/files/routes/posts.py index 6043efc19..f3d6981d2 100644 --- a/files/routes/posts.py +++ b/files/routes/posts.py @@ -707,100 +707,6 @@ def thumbnail_thread(pid): post.thumburl = process_image(0, name, resize=100) db.add(post) db.commit() - - if SITE_NAME == 'rDrama': - for t in ("submission","comment"): - word = random.choice(('rdrama','marsey')) - - try: - data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q={word}&size=1', timeout=5).json()["data"] - except: break - - for i in data: - - if i["subreddit"] == 'PokemonGoRaids': continue - - body_html = f'''

New site mention: https://old.reddit.com{i["permalink"]}?context=89

''' - - existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None, body_html=body_html).one_or_none() - if existing_comment: break - - new_comment = Comment(author_id=NOTIFICATIONS_ID, - parent_submission=None, - body_html=body_html, - distinguish_level=6 - ) - db.add(new_comment) - db.flush() - - new_comment.top_comment_id = new_comment.id - - - admins = db.query(User).filter(User.admin_level > 0).all() - for admin in admins: - notif = Notification(comment_id=new_comment.id, user_id=admin.id) - db.add(notif) - - k,val = random.choice(tuple(REDDIT_NOTIFS.items())) - - try: - data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q={k}&size=1', timeout=5).json()["data"] - except: break - - for i in data: - - body_html = f'''

New mention of you: https://old.reddit.com{i["permalink"]}?context=89

''' - - existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None,body_html=body_html).one_or_none() - if existing_comment: break - - new_comment = Comment(author_id=NOTIFICATIONS_ID, - parent_submission=None, - body_html=body_html, - distinguish_level=6 - ) - - db.add(new_comment) - db.flush() - - new_comment.top_comment_id = new_comment.id - - - notif = Notification(comment_id=new_comment.id, user_id=val) - db.add(notif) - - - if SITE == 'pcmemes.net': - for t in ("submission","comment"): - - try: - data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q=pcmemes.net&size=1', timeout=5).json()["data"] - except: break - - for i in data: - body_html = f'''

New site mention: https://old.reddit.com{i["permalink"]}?context=89

''' - - existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None, body_html=body_html).one_or_none() - - if existing_comment: break - - new_comment = Comment(author_id=NOTIFICATIONS_ID, - parent_submission=None, - body_html=body_html, - distinguish_level=6 - ) - db.add(new_comment) - db.flush() - - new_comment.top_comment_id = new_comment.id - - - admins = db.query(User).filter(User.admin_level > 2).all() - for admin in admins: - notif = Notification(comment_id=new_comment.id, user_id=admin.id) - db.add(notif) - - db.commit() db.close() stdout.flush() return