all seeing eye: rework reddit notifs a bit (#445)
* all seeing eye: rework reddit notifs a bit this should be more reliable and catch more of these when they're sent * fix reddit notifs to properly use fewer queriesremotes/1693176582716663532/tmp_refs/heads/watchparty
parent
8fee66c894
commit
f4f03f6582
|
@ -31,7 +31,7 @@ def cron(every_5m, every_1h, every_1d, every_1mo):
|
|||
if FEATURES['GAMBLING']:
|
||||
lottery.check_if_end_lottery_task()
|
||||
spin_roulette_wheel()
|
||||
offsitementions.offsite_mentions_task()
|
||||
offsitementions.offsite_mentions_task(cache)
|
||||
if SITE == 'pcmemes.net':
|
||||
route_static.live_cached()
|
||||
|
||||
|
|
|
@ -1,3 +1,6 @@
|
|||
import time
|
||||
from typing import Iterable
|
||||
from flask_caching import Cache
|
||||
from flask import g
|
||||
import itertools
|
||||
import requests
|
||||
|
@ -15,7 +18,7 @@ from files.helpers.sanitize import sanitize
|
|||
# with current keyword quantities. If this ever changes, consider reading the
|
||||
# value from /meta (or just guessing) and doing a random selection of keywords.
|
||||
|
||||
def offsite_mentions_task():
|
||||
def offsite_mentions_task(cache:Cache):
|
||||
if const.REDDIT_NOTIFS_SITE:
|
||||
row_send_to = g.db.query(Badge.user_id).filter_by(badge_id=140).all()
|
||||
row_send_to += g.db.query(User.id).filter(or_(User.admin_level >= const.PERMS['NOTIFICATIONS_REDDIT'])).all()
|
||||
|
@ -23,49 +26,59 @@ def offsite_mentions_task():
|
|||
send_to = [x[0] for x in row_send_to]
|
||||
send_to = set(send_to)
|
||||
|
||||
site_mentions = get_mentions(const.REDDIT_NOTIFS_SITE)
|
||||
site_mentions = get_mentions(cache, const.REDDIT_NOTIFS_SITE)
|
||||
notify_mentions(send_to, site_mentions)
|
||||
|
||||
if const.REDDIT_NOTIFS_USERS:
|
||||
for query, send_user in const.REDDIT_NOTIFS_USERS.items():
|
||||
user_mentions = get_mentions([query])
|
||||
user_mentions = get_mentions(cache, [query], reddit_notifs_users=True)
|
||||
notify_mentions([send_user], user_mentions, mention_str='mention of you')
|
||||
|
||||
def get_mentions(queries):
|
||||
def get_mentions(cache:Cache, queries:Iterable[str], reddit_notifs_users=False):
|
||||
CACHE_KEY = 'reddit_notifications'
|
||||
kinds = ['submission', 'comment']
|
||||
mentions = []
|
||||
for kind, query in itertools.product(kinds, queries):
|
||||
exclude_subreddits = ['PokemonGoRaids', 'SubSimulatorGPT2']
|
||||
try:
|
||||
# Special cases: PokemonGoRaids says 'Marsey' a lot unrelated to us.
|
||||
# SubSimulatorGPT2 is just bots
|
||||
data = requests.get(f'https://api.pushshift.io/reddit/{kind}/search?html_decode=true&q={query}&subreddit=!PokemonGoRaids,!SubSimulatorGPT2&size=1', timeout=5).json()['data']
|
||||
except: break
|
||||
|
||||
for i in data:
|
||||
if 'bot' in i['author'].lower(): continue
|
||||
after = int(cache.get(CACHE_KEY) or time.time())
|
||||
except:
|
||||
print("Failed to retrieve last mention time from cache")
|
||||
after = time.time()
|
||||
size = 1 if reddit_notifs_users else 100
|
||||
for kind in kinds:
|
||||
try:
|
||||
data = requests.get(f'https://api.pushshift.io/reddit/{kind}/search?html_decode=true&q={"%7C".join(queries)}&subreddit=!{",!".join(exclude_subreddits)}&after={after}&size={size}', timeout=15).json()['data']
|
||||
except:
|
||||
continue
|
||||
|
||||
for thing in data:
|
||||
if 'bot' in thing['author'].lower(): continue
|
||||
after = max(after, thing["created_utc"]) if thing["created_utc"] else after
|
||||
if kind == 'comment':
|
||||
body = i["body"].replace('>', '> ')
|
||||
body = thing["body"].replace('>', '> ')
|
||||
text = f'<blockquote><p>{body}</p></blockquote>'
|
||||
else:
|
||||
title = i["title"].replace('>', '> ')
|
||||
title = thing["title"].replace('>', '> ')
|
||||
|
||||
# Special case: a spambot says 'WPD' a lot unrelated to us.
|
||||
if 'Kathrine Mclaurin' in title: continue
|
||||
|
||||
text = f'<blockquote><p>{title}</p></blockquote>'
|
||||
|
||||
if i["selftext"]:
|
||||
selftext = i["selftext"].replace('>', '> ')[:5000]
|
||||
if thing["selftext"]:
|
||||
selftext = thing["selftext"].replace('>', '> ')[:5000]
|
||||
text += f'<br><blockquote><p>{selftext}</p></blockquote>'
|
||||
|
||||
|
||||
mentions.append({
|
||||
'permalink': i['permalink'],
|
||||
'author': i['author'],
|
||||
'permalink': thing['permalink'],
|
||||
'author': thing['author'],
|
||||
'text': text,
|
||||
})
|
||||
|
||||
try:
|
||||
if not reddit_notifs_users:
|
||||
cache.set(CACHE_KEY, after + 1)
|
||||
except:
|
||||
print("Failed to set cache value; there may be duplication of reddit notifications")
|
||||
return mentions
|
||||
|
||||
def notify_mentions(send_to, mentions, mention_str='site mention'):
|
||||
|
|
Loading…
Reference in New Issue