forked from MarseyWorld/MarseyWorld
Refactor reddit mentions, move to cron.
The reddit mentions system contained much duplicated code and was grafted onto the post thumbnail pipeline to achieve semi-regular invocation. Instead, we now run it through the new cron system, and the duplicate code has been refactored out.master
parent
152315f73e
commit
995375decc
|
|
@ -799,14 +799,21 @@ FACTCHECK_REPLIES = ('<b style="color:#6023f8">Factcheck: This claim has been co
|
|||
if SITE_NAME == 'rDrama': patron = 'Paypig'
|
||||
else: patron = 'Patron'
|
||||
|
||||
REDDIT_NOTIFS = {
|
||||
'idio3': IDIO_ID,
|
||||
'aevann': AEVANN_ID,
|
||||
'carpflo': CARP_ID,
|
||||
'carpathianflorist': CARP_ID,
|
||||
'carpathian florist': CARP_ID,
|
||||
'the_homocracy': HOMO_ID
|
||||
}
|
||||
REDDIT_NOTIFS_SITE = []
|
||||
REDDIT_NOTIFS_USERS = {}
|
||||
|
||||
if SITE_NAME == 'rDrama':
|
||||
REDDIT_NOTIFS_SITE = ['rdrama', 'marsey',]
|
||||
REDDIT_NOTIFS_USERS = {
|
||||
'idio3': IDIO_ID,
|
||||
'aevann': AEVANN_ID,
|
||||
'carpflo': CARP_ID,
|
||||
'carpathianflorist': CARP_ID,
|
||||
'carpathian florist': CARP_ID,
|
||||
'the_homocracy': HOMO_ID,
|
||||
}
|
||||
elif SITE_NAME == 'PCM':
|
||||
REDDIT_NOTIFS_SITE = ['pcmemes.net',]
|
||||
|
||||
discounts = {
|
||||
# Big Spender badges, 2pp additive discount each
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from files.cli import g, app, db_session
|
||||
import click
|
||||
import files.helpers.const as const
|
||||
|
||||
import files.helpers.lottery as lottery
|
||||
import files.helpers.offsitementions as offsitementions
|
||||
import files.helpers.stats as stats
|
||||
|
||||
@app.cli.command('cron', help='Run scheduled tasks.')
|
||||
|
|
@ -13,6 +15,7 @@ def cron(every_5m, every_1h, every_1d):
|
|||
|
||||
if every_5m:
|
||||
lottery.check_if_end_lottery_task()
|
||||
offsitementions.offsite_mentions_task()
|
||||
|
||||
if every_1h:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
from flask import g
|
||||
import itertools
|
||||
import requests
|
||||
import files.helpers.const as const
|
||||
from files.classes.user import User
|
||||
from files.classes.comment import Comment
|
||||
from files.classes.notifications import Notification
|
||||
|
||||
# https://api.pushshift.io/meta provides key server_ratelimit_per_minute
|
||||
# At time of writing, the ratelimit is 120 req/min. We get nowhere near this
|
||||
# with current keyword quantities. If this ever changes, consider reading the
|
||||
# value from /meta and doing a random selection of keywords.
|
||||
|
||||
def offsite_mentions_task():
|
||||
if const.REDDIT_NOTIFS_SITE:
|
||||
# Site-specific logic: send to JL1+, except on PCM JL3+
|
||||
jl_min = 3 if const.SITE_NAME == 'PCM' else 1
|
||||
row_send_to = g.db.query(User.id).filter(User.admin_level >= jl_min).all()
|
||||
send_to = [x[0] for x in row_send_to]
|
||||
|
||||
site_mentions = get_mentions(const.REDDIT_NOTIFS_SITE)
|
||||
notify_mentions(send_to, site_mentions)
|
||||
|
||||
if const.REDDIT_NOTIFS_USERS:
|
||||
for query, send_user in const.REDDIT_NOTIFS_USERS.items():
|
||||
if not send_user: continue
|
||||
|
||||
user_mentions = get_mentions([query])
|
||||
notify_mentions([send_user], user_mentions, mention_str='mention of you')
|
||||
|
||||
def get_mentions(queries):
|
||||
kinds = ['submission', 'comment']
|
||||
mentions = []
|
||||
for kind, query in itertools.product(kinds, queries):
|
||||
try:
|
||||
data = requests.get(f'https://api.pushshift.io/reddit/{kind}/search'
|
||||
+ f'?html_decode=true&q={query}&size=1', timeout=5).json()['data']
|
||||
except: break
|
||||
|
||||
for i in data:
|
||||
# Special case: PokemonGoRaids says 'Marsey' a lot unrelated to us.
|
||||
if i['subreddit'] == 'PokemonGoRaids': continue
|
||||
|
||||
mentions.append(i['permalink'])
|
||||
|
||||
return mentions
|
||||
|
||||
def notify_mentions(send_to, mentions, mention_str='site mention'):
|
||||
for m in mentions:
|
||||
notif_text = f'<p>New {mention_str}: <a href="https://old.reddit.com{m}' \
|
||||
f'?context=89" rel="nofollow noopener noreferrer" target="_blank">' \
|
||||
f'https://old.reddit.com{m}?context=89</a></p>'
|
||||
|
||||
existing_comment = g.db.query(Comment.id).filter_by(
|
||||
author_id=const.NOTIFICATIONS_ID,
|
||||
parent_submission=None,
|
||||
body_html=notif_text).one_or_none()
|
||||
if existing_comment: break
|
||||
|
||||
new_comment = Comment(
|
||||
author_id=const.NOTIFICATIONS_ID,
|
||||
parent_submission=None,
|
||||
body_html=notif_text,
|
||||
distinguish_level=6)
|
||||
g.db.add(new_comment)
|
||||
g.db.flush()
|
||||
new_comment.top_comment_id = new_comment.id
|
||||
|
||||
for user_id in send_to:
|
||||
notif = Notification(comment_id=new_comment.id, user_id=user_id)
|
||||
g.db.add(notif)
|
||||
|
||||
g.db.commit()
|
||||
|
||||
|
|
@ -707,100 +707,6 @@ def thumbnail_thread(pid):
|
|||
post.thumburl = process_image(0, name, resize=100)
|
||||
db.add(post)
|
||||
db.commit()
|
||||
|
||||
if SITE_NAME == 'rDrama':
|
||||
for t in ("submission","comment"):
|
||||
word = random.choice(('rdrama','marsey'))
|
||||
|
||||
try:
|
||||
data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q={word}&size=1', timeout=5).json()["data"]
|
||||
except: break
|
||||
|
||||
for i in data:
|
||||
|
||||
if i["subreddit"] == 'PokemonGoRaids': continue
|
||||
|
||||
body_html = f'''<p>New site mention: <a href="https://old.reddit.com{i["permalink"]}?context=89" rel="nofollow noopener noreferrer" target="_blank">https://old.reddit.com{i["permalink"]}?context=89</a></p>'''
|
||||
|
||||
existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None, body_html=body_html).one_or_none()
|
||||
if existing_comment: break
|
||||
|
||||
new_comment = Comment(author_id=NOTIFICATIONS_ID,
|
||||
parent_submission=None,
|
||||
body_html=body_html,
|
||||
distinguish_level=6
|
||||
)
|
||||
db.add(new_comment)
|
||||
db.flush()
|
||||
|
||||
new_comment.top_comment_id = new_comment.id
|
||||
|
||||
|
||||
admins = db.query(User).filter(User.admin_level > 0).all()
|
||||
for admin in admins:
|
||||
notif = Notification(comment_id=new_comment.id, user_id=admin.id)
|
||||
db.add(notif)
|
||||
|
||||
k,val = random.choice(tuple(REDDIT_NOTIFS.items()))
|
||||
|
||||
try:
|
||||
data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q={k}&size=1', timeout=5).json()["data"]
|
||||
except: break
|
||||
|
||||
for i in data:
|
||||
|
||||
body_html = f'''<p>New mention of you: <a href="https://old.reddit.com{i["permalink"]}?context=89" rel="nofollow noopener noreferrer" target="_blank">https://old.reddit.com{i["permalink"]}?context=89</a></p>'''
|
||||
|
||||
existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None,body_html=body_html).one_or_none()
|
||||
if existing_comment: break
|
||||
|
||||
new_comment = Comment(author_id=NOTIFICATIONS_ID,
|
||||
parent_submission=None,
|
||||
body_html=body_html,
|
||||
distinguish_level=6
|
||||
)
|
||||
|
||||
db.add(new_comment)
|
||||
db.flush()
|
||||
|
||||
new_comment.top_comment_id = new_comment.id
|
||||
|
||||
|
||||
notif = Notification(comment_id=new_comment.id, user_id=val)
|
||||
db.add(notif)
|
||||
|
||||
|
||||
if SITE == 'pcmemes.net':
|
||||
for t in ("submission","comment"):
|
||||
|
||||
try:
|
||||
data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q=pcmemes.net&size=1', timeout=5).json()["data"]
|
||||
except: break
|
||||
|
||||
for i in data:
|
||||
body_html = f'''<p>New site mention: <a href="https://old.reddit.com{i["permalink"]}?context=89" rel="nofollow noopener noreferrer" target="_blank">https://old.reddit.com{i["permalink"]}?context=89</a></p>'''
|
||||
|
||||
existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None, body_html=body_html).one_or_none()
|
||||
|
||||
if existing_comment: break
|
||||
|
||||
new_comment = Comment(author_id=NOTIFICATIONS_ID,
|
||||
parent_submission=None,
|
||||
body_html=body_html,
|
||||
distinguish_level=6
|
||||
)
|
||||
db.add(new_comment)
|
||||
db.flush()
|
||||
|
||||
new_comment.top_comment_id = new_comment.id
|
||||
|
||||
|
||||
admins = db.query(User).filter(User.admin_level > 2).all()
|
||||
for admin in admins:
|
||||
notif = Notification(comment_id=new_comment.id, user_id=admin.id)
|
||||
db.add(notif)
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
stdout.flush()
|
||||
return
|
||||
|
|
|
|||
Loading…
Reference in New Issue