Refactor reddit mentions, move to cron.

The reddit mentions system contained much duplicated code and was
grafted onto the post thumbnail pipeline to achieve semi-regular
invocation. Instead, we now run it through the new cron system,
and the duplicate code has been refactored out.

files/helpers/const.py

@@ -799,14 +799,21 @@ FACTCHECK_REPLIES = ('<b style="color:#6023f8">Factcheck: This claim has been co
 if SITE_NAME == 'rDrama': patron = 'Paypig'
 else: patron = 'Patron'
 
-REDDIT_NOTIFS = {
+REDDIT_NOTIFS_SITE = []
-	'idio3': IDIO_ID,
+REDDIT_NOTIFS_USERS = {}
-	'aevann': AEVANN_ID,
+
-	'carpflo': CARP_ID,
+if SITE_NAME == 'rDrama':
-	'carpathianflorist': CARP_ID,
+	REDDIT_NOTIFS_SITE = ['rdrama', 'marsey',]
-	'carpathian florist': CARP_ID,
+	REDDIT_NOTIFS_USERS = {
-	'the_homocracy': HOMO_ID
+		'idio3': IDIO_ID,
-}
+		'aevann': AEVANN_ID,
+		'carpflo': CARP_ID,
+		'carpathianflorist': CARP_ID,
+		'carpathian florist': CARP_ID,
+		'the_homocracy': HOMO_ID,
+	}
+elif SITE_NAME == 'PCM':
+	REDDIT_NOTIFS_SITE = ['pcmemes.net',]
 
 discounts = {
 	# Big Spender badges, 2pp additive discount each

files/helpers/cron.py

@@ -1,7 +1,9 @@
 from files.cli import g, app, db_session
 import click
 import files.helpers.const as const
+
 import files.helpers.lottery as lottery
+import files.helpers.offsitementions as offsitementions
 import files.helpers.stats as stats
 
 @app.cli.command('cron', help='Run scheduled tasks.')
@@ -13,6 +15,7 @@ def cron(every_5m, every_1h, every_1d):
 
 	if every_5m:
 		lottery.check_if_end_lottery_task()
+		offsitementions.offsite_mentions_task()
 
 	if every_1h:
 		pass

files/helpers/offsitementions.py

@@ -0,0 +1,74 @@
+from flask import g
+import itertools
+import requests
+import files.helpers.const as const
+from files.classes.user import User
+from files.classes.comment import Comment
+from files.classes.notifications import Notification
+
+# https://api.pushshift.io/meta provides key server_ratelimit_per_minute
+# At time of writing, the ratelimit is 120 req/min. We get nowhere near this 
+# with current keyword quantities. If this ever changes, consider reading the 
+# value from /meta and doing a random selection of keywords.
+
+def offsite_mentions_task():
+	if const.REDDIT_NOTIFS_SITE:
+		# Site-specific logic: send to JL1+, except on PCM JL3+
+		jl_min = 3 if const.SITE_NAME == 'PCM' else 1
+		row_send_to = g.db.query(User.id).filter(User.admin_level >= jl_min).all()
+		send_to = [x[0] for x in row_send_to]
+
+		site_mentions = get_mentions(const.REDDIT_NOTIFS_SITE)
+		notify_mentions(send_to, site_mentions)
+
+	if const.REDDIT_NOTIFS_USERS:
+		for query, send_user in const.REDDIT_NOTIFS_USERS.items():
+			if not send_user: continue
+
+			user_mentions = get_mentions([query])
+			notify_mentions([send_user], user_mentions, mention_str='mention of you')
+
+def get_mentions(queries):
+	kinds = ['submission', 'comment']
+	mentions = []
+	for kind, query in itertools.product(kinds, queries):
+		try:
+			data = requests.get(f'https://api.pushshift.io/reddit/{kind}/search'
+				+ f'?html_decode=true&q={query}&size=1', timeout=5).json()['data']
+		except: break
+
+		for i in data: 
+			# Special case: PokemonGoRaids says 'Marsey' a lot unrelated to us.
+			if i['subreddit'] == 'PokemonGoRaids': continue
+
+			mentions.append(i['permalink'])
+
+	return mentions
+
+def notify_mentions(send_to, mentions, mention_str='site mention'):
+	for m in mentions:
+		notif_text = f'<p>New {mention_str}: <a href="https://old.reddit.com{m}' \
+			f'?context=89" rel="nofollow noopener noreferrer" target="_blank">' \
+			f'https://old.reddit.com{m}?context=89</a></p>'
+
+		existing_comment = g.db.query(Comment.id).filter_by(
+			author_id=const.NOTIFICATIONS_ID, 
+			parent_submission=None, 
+			body_html=notif_text).one_or_none()
+		if existing_comment: break
+
+		new_comment = Comment(
+						author_id=const.NOTIFICATIONS_ID,
+						parent_submission=None,
+						body_html=notif_text,
+						distinguish_level=6)
+		g.db.add(new_comment)
+		g.db.flush()
+		new_comment.top_comment_id = new_comment.id
+
+		for user_id in send_to:
+			notif = Notification(comment_id=new_comment.id, user_id=user_id)
+			g.db.add(notif)
+
+	g.db.commit()
+

files/routes/posts.py

@@ -707,100 +707,6 @@ def thumbnail_thread(pid):
 	post.thumburl = process_image(0, name, resize=100)
 	db.add(post)
 	db.commit()
-
-	if SITE_NAME == 'rDrama':
-		for t in ("submission","comment"):
-			word = random.choice(('rdrama','marsey'))
-
-			try:
-				data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q={word}&size=1', timeout=5).json()["data"]
-			except: break
-
-			for i in data:
-
-				if i["subreddit"] == 'PokemonGoRaids': continue
-
-				body_html = f'''<p>New site mention: <a href="https://old.reddit.com{i["permalink"]}?context=89" rel="nofollow noopener noreferrer" target="_blank">https://old.reddit.com{i["permalink"]}?context=89</a></p>'''
-
-				existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None, body_html=body_html).one_or_none()
-				if existing_comment: break
-
-				new_comment = Comment(author_id=NOTIFICATIONS_ID,
-									parent_submission=None,
-									body_html=body_html,
-									distinguish_level=6
-									)
-				db.add(new_comment)
-				db.flush()
-
-				new_comment.top_comment_id = new_comment.id
-
-
-				admins = db.query(User).filter(User.admin_level > 0).all()
-				for admin in admins:
-					notif = Notification(comment_id=new_comment.id, user_id=admin.id)
-					db.add(notif)
-
-			k,val = random.choice(tuple(REDDIT_NOTIFS.items()))
-			
-			try:
-				data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q={k}&size=1', timeout=5).json()["data"]
-			except: break
-
-			for i in data:
-
-				body_html = f'''<p>New mention of you: <a href="https://old.reddit.com{i["permalink"]}?context=89" rel="nofollow noopener noreferrer" target="_blank">https://old.reddit.com{i["permalink"]}?context=89</a></p>'''
-
-				existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None,body_html=body_html).one_or_none()
-				if existing_comment: break
-
-				new_comment = Comment(author_id=NOTIFICATIONS_ID,
-									parent_submission=None,
-									body_html=body_html,
-									distinguish_level=6
-									)
-
-				db.add(new_comment)
-				db.flush()
-
-				new_comment.top_comment_id = new_comment.id
-
-
-				notif = Notification(comment_id=new_comment.id, user_id=val)
-				db.add(notif)
-
-
-	if SITE == 'pcmemes.net':
-		for t in ("submission","comment"):
-
-			try:
-				data = requests.get(f'https://api.pushshift.io/reddit/{t}/search?html_decode=true&q=pcmemes.net&size=1', timeout=5).json()["data"]
-			except: break
-
-			for i in data:
-				body_html = f'''<p>New site mention: <a href="https://old.reddit.com{i["permalink"]}?context=89" rel="nofollow noopener noreferrer" target="_blank">https://old.reddit.com{i["permalink"]}?context=89</a></p>'''
-
-				existing_comment = db.query(Comment.id).filter_by(author_id=NOTIFICATIONS_ID, parent_submission=None, body_html=body_html).one_or_none()
-
-				if existing_comment: break
-
-				new_comment = Comment(author_id=NOTIFICATIONS_ID,
-									parent_submission=None,
-									body_html=body_html,
-									distinguish_level=6
-									)
-				db.add(new_comment)
-				db.flush()
-
-				new_comment.top_comment_id = new_comment.id
-
-
-				admins = db.query(User).filter(User.admin_level > 2).all()
-				for admin in admins:
-					notif = Notification(comment_id=new_comment.id, user_id=admin.id)
-					db.add(notif)
-
-	db.commit()
 	db.close()
 	stdout.flush()
 	return