diff --git a/files/classes/user.py b/files/classes/user.py index fc2291bad..94c595165 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -136,7 +136,7 @@ class User(Base): last_viewed_modmail_notifs = Column(Integer, default=0) last_viewed_post_notifs = Column(Integer, default=0) last_viewed_log_notifs = Column(Integer, default=0) - last_viewed_reddit_notifs = Column(Integer, default=0) + last_viewed_offsite_notifs = Column(Integer, default=0) bite = Column(Integer, default=0) owoify = Column(Integer, default=0) sharpen = Column(Integer, default=0) @@ -202,7 +202,7 @@ class User(Base): kwargs["last_viewed_modmail_notifs"] = kwargs["created_utc"] kwargs["last_viewed_post_notifs"] = kwargs["created_utc"] kwargs["last_viewed_log_notifs"] = kwargs["created_utc"] - kwargs["last_viewed_reddit_notifs"] = kwargs["created_utc"] + kwargs["last_viewed_offsite_notifs"] = kwargs["created_utc"] super().__init__(**kwargs) @@ -556,8 +556,8 @@ class User(Base): @property @lazy - def can_view_offsitementions(self): - return self.offsitementions or self.admin_level >= PERMS['NOTIFICATIONS_REDDIT'] + def can_view_offsite_mentions(self): + return self.has_badge(140) or self.admin_level >= PERMS['NOTIFICATIONS_OFFSITE'] @lazy def can_edit(self, target): @@ -786,7 +786,7 @@ class User(Base): Comment.deleted_utc == 0, ) - return notifs.count() + self.modmail_notifications_count + self.post_notifications_count + self.modaction_notifications_count + self.reddit_notifications_count + return notifs.count() + self.modmail_notifications_count + self.post_notifications_count + self.modaction_notifications_count + self.offsite_notifications_count @property @lazy @@ -796,7 +796,7 @@ class User(Base): - self.modmail_notifications_count \ - self.post_notifications_count \ - self.modaction_notifications_count \ - - self.reddit_notifications_count + - self.offsite_notifications_count @property @lazy @@ -885,13 +885,13 @@ class User(Base): @property @lazy - def reddit_notifications_count(self): - if not self.can_view_offsitementions or (SITE == "watchpeopledie.tv" and self.id == AEVANN_ID): + def offsite_notifications_count(self): + if not self.can_view_offsite_mentions or (SITE == "watchpeopledie.tv" and self.id == AEVANN_ID): return 0 return g.db.query(Comment).filter( - Comment.created_utc > self.last_viewed_reddit_notifs, + Comment.created_utc > self.last_viewed_offsite_notifs, Comment.is_banned == False, Comment.deleted_utc == 0, - Comment.body_html.like('%

New site mention% 5: - REDDIT_NOTIFS_SITE.add(SITE_NAME.lower()) + OFFSITE_NOTIF_QUERIES.add(SITE_NAME.lower()) if not IS_LOCALHOST: - REDDIT_NOTIFS_SITE.add(SITE) + OFFSITE_NOTIF_QUERIES.add(SITE) TAGLINES = () @@ -154,7 +154,7 @@ PERMS = { # Minimum admin_level to perform action. 'BYPASS_CHAT_TRUESCORE_REQUIREMENT': 1, 'BYPASS_ANTISPAM_CHECKS': 1, 'WARN_ON_FAILED_LOGIN': 1, - 'NOTIFICATIONS_REDDIT': 1, + 'NOTIFICATIONS_OFFSITE': 1, 'NOTIFICATIONS_SPECIFIC_WPD_COMMENTS': 1, 'MESSAGE_BLOCKED_USERS': 1, 'ADMIN_MOP_VISIBLE': 1, @@ -316,10 +316,10 @@ if SITE_NAME == 'rDrama': 'atheism', } - REDDIT_NOTIFS_SITE.update({'marsey', 'r/drama', 'justice4darrell', 'cringetopia.org'}) + OFFSITE_NOTIF_QUERIES.update({'marsey', 'r/drama', 'justice4darrell', 'cringetopia.org'}) elif SITE_NAME == 'WPD': - REDDIT_NOTIFS_SITE.update({'marsey', 'watchpeopledie', 'makemycoffin'}) + OFFSITE_NOTIF_QUERIES.update({'marsey', 'watchpeopledie', 'makemycoffin'}) LONGPOSTBOT_REPLIES = ( diff --git a/files/helpers/cron.py b/files/helpers/cron.py index 8f51450b8..25cf318a6 100644 --- a/files/helpers/cron.py +++ b/files/helpers/cron.py @@ -14,7 +14,6 @@ import requests import ffmpeg -import files.helpers.offsitementions as offsitementions import files.helpers.stats as stats import files.routes.static as route_static from files.routes.front import frontlist @@ -27,6 +26,9 @@ from files.helpers.lottery import check_if_end_lottery_task from files.helpers.roulette import spin_roulette_wheel from files.helpers.sanitize import filter_emojis_only, sanitize from files.helpers.useractions import * +from files.helpers.reddit_mentions import * +from files.helpers.lemmy_mentions import * + from files.cli import app, db_session, g CRON_CACHE_TIMEOUT = 172800 @@ -57,9 +59,11 @@ def cron_fn(every_5m, every_1d, every_1mo): _grant_two_year_badges() g.db.commit() - if not IS_LOCALHOST: - offsitementions.offsite_mentions_task(cache) - g.db.commit() + reddit_mentions_task() + g.db.commit() + + lemmy_mentions_task() + g.db.commit() if every_1d or (not cache.get('stats') and not IS_LOCALHOST): if IS_HOMOWEEN(): diff --git a/files/helpers/lemmy_mentions.py b/files/helpers/lemmy_mentions.py new file mode 100644 index 000000000..86e9e0d36 --- /dev/null +++ b/files/helpers/lemmy_mentions.py @@ -0,0 +1,54 @@ +import requests +from flask import g + +from files.helpers.config.const import * +from files.classes.comment import Comment +from files.helpers.sanitize import * +from files.helpers.alerts import push_notif +from files.classes.notifications import Notification + +def lemmy_mentions_task(): + for q in OFFSITE_NOTIF_QUERIES: + url = f'https://lemm.ee/api/v3/search?q={q}' + data = requests.get(url, headers=HEADERS, timeout=5).json() + + for kind in ("post", "comment"): + for thing in data[f'{kind}s']: + creator = thing['creator'] + author_string = f"[/u/{creator['name']}]({creator['actor_id']})" + thing = thing[kind] + if kind == 'comment': + body = thing["content"] + text = f'

{body}

' + else: + title = thing["name"] + text = f'

{title}

' + + if thing["body"]: + selftext = thing["body"][:5000] + text += f'

{selftext}

' + + permalink = thing['ap_id'] + text = f'New site mention by {author_string}\n\n{permalink}\n\n{text}' + text = sanitize(text, blackjack="lemmy mention", golden=False) + + existing_comment = g.db.query(Comment.id).filter_by( + author_id=AUTOJANNY_ID, + parent_post=None, + body_html=text).one_or_none() + if existing_comment: break + + try: created_utc = int(time.mktime(time.strptime(thing['published'].split('.')[0], "%Y-%m-%dT%H:%M:%S"))) + except: created_utc = int(time.mktime(time.strptime(thing['published'].split('.')[0], "%Y-%m-%dT%H:%M:%SZ"))) + + new_comment = Comment( + author_id=AUTOJANNY_ID, + parent_post=None, + body_html=text, + distinguished=True, + created_utc=created_utc, + ) + + g.db.add(new_comment) + g.db.flush() + new_comment.top_comment_id = new_comment.id diff --git a/files/helpers/offsitementions.py b/files/helpers/reddit_mentions.py similarity index 70% rename from files/helpers/offsitementions.py rename to files/helpers/reddit_mentions.py index c42c826da..7a4667bd8 100644 --- a/files/helpers/offsitementions.py +++ b/files/helpers/reddit_mentions.py @@ -1,36 +1,23 @@ -import time -import itertools - import requests -from flask_caching import Cache from flask import g -from sqlalchemy import or_ -import files.helpers.config.const as const -from files.classes.badges import Badge +from files.helpers.config.const import * from files.classes.comment import Comment -from files.classes.user import User from files.helpers.sanitize import * from files.helpers.alerts import push_notif from files.classes.notifications import Notification -# Note: while https://api.pushshift.io/meta provides the key -# server_ratelimit_per_minute, in practice Cloudflare puts stricter, -# unofficially documented limits at around 60/minute. We get nowhere near this -# with current keyword quantities. If this ever changes, consider reading the -# value from /meta (or just guessing) and doing a random selection of keywords. - -def offsite_mentions_task(cache): - site_mentions = get_mentions(cache, const.REDDIT_NOTIFS_SITE) +def reddit_mentions_task(): + site_mentions = get_mentions(OFFSITE_NOTIF_QUERIES) notify_mentions(site_mentions) - if const.REDDIT_NOTIFS_USERS: - for query, send_user in const.REDDIT_NOTIFS_USERS.items(): + if REDDIT_NOTIFS_USERS: + for query, send_user in REDDIT_NOTIFS_USERS.items(): user_mentions = get_mentions(cache, [query], reddit_notifs_users=True) if user_mentions: notify_mentions(user_mentions, send_to=send_user, mention_str='mention of you') -def get_mentions(cache, queries, reddit_notifs_users=False): +def get_mentions(queries, reddit_notifs_users=False): mentions = [] for kind in ('submission', 'comment'): q = " or ".join(queries) @@ -73,32 +60,31 @@ def notify_mentions(mentions, send_to=None, mention_str='site mention'): for m in mentions: author = m['author'] permalink = m['permalink'] - text = sanitize(m['text'], blackjack="reddit mention", golden=False) - notif_text = ( + text = ( f'

New {mention_str} by /u/{author}

' f'

' f'https://old.reddit.com{permalink}?context=89

' - f'{text}' + f'{m["text"]}' ) + text = sanitize(text, blackjack="reddit mention", golden=False) + g.db.flush() try: existing_comment = g.db.query(Comment.id).filter_by( - author_id=const.AUTOJANNY_ID, + author_id=AUTOJANNY_ID, parent_post=None, - body_html=notif_text).one_or_none() + body_html=text).one_or_none() if existing_comment: break - # todo: handle this exception by removing one of the existing - # means that multiple rows were found, happens on new install for some reason except: pass new_comment = Comment( - author_id=const.AUTOJANNY_ID, + author_id=AUTOJANNY_ID, parent_post=None, - body_html=notif_text, + body_html=text, distinguished=True, created_utc=int(m['created_utc']), ) diff --git a/files/routes/notifications.py b/files/routes/notifications.py index d9d34305b..d94a2b6d0 100644 --- a/files/routes/notifications.py +++ b/files/routes/notifications.py @@ -32,7 +32,7 @@ def clear(v): v.last_viewed_modmail_notifs = int(time.time()) v.last_viewed_post_notifs = int(time.time()) v.last_viewed_log_notifs = int(time.time()) - v.last_viewed_reddit_notifs = int(time.time()) + v.last_viewed_offsite_notifs = int(time.time()) g.db.add(v) return {"message": "Notifications marked as read!"} @@ -268,17 +268,17 @@ def notifications_modactions(v): -@app.get("/notifications/reddit") +@app.get("/notifications/offsite") @limiter.limit(DEFAULT_RATELIMIT, deduct_when=lambda response: response.status_code < 400) @limiter.limit(DEFAULT_RATELIMIT, deduct_when=lambda response: response.status_code < 400, key_func=get_ID) @auth_required -def notifications_reddit(v): +def notifications_offsite(v): page = get_page() - if not v.can_view_offsitementions: abort(403) + if not v.can_view_offsite_mentions: abort(403) listing = g.db.query(Comment).filter( - Comment.body_html.like('%

New site mention% {% endif %} - {% if v.can_view_offsitementions %} + {% if v.can_view_offsite_mentions %}

  • - - Reddit {% if v.reddit_notifications_count %}({{v.reddit_notifications_count}}){% endif %} + + Offsite {% if v.offsite_notifications_count %}({{v.offsite_notifications_count}}){% endif %}
    • {% endif %} diff --git a/migrations/20240216-add-lemmy-mentions.sql b/migrations/20240216-add-lemmy-mentions.sql new file mode 100644 index 000000000..4f2e17f04 --- /dev/null +++ b/migrations/20240216-add-lemmy-mentions.sql @@ -0,0 +1 @@ +alter table users rename column last_viewed_reddit_notifs to last_viewed_offsite_notifs; diff --git a/schema.sql b/schema.sql index b19722a1a..cb2762359 100644 --- a/schema.sql +++ b/schema.sql @@ -163,7 +163,7 @@ CREATE TABLE public.users ( rainbow integer, spider integer, profanityreplacer integer DEFAULT 1 NOT NULL, - last_viewed_reddit_notifs integer NOT NULL, + last_viewed_offsite_notifs integer NOT NULL, profile_background character varying(167), chudded_by integer, blacklisted_by integer, @@ -3073,4 +3073,3 @@ ALTER TABLE ONLY public.comments -- -- PostgreSQL database dump complete -- - diff --git a/seed-users.sql b/seed-users.sql index d4b86bd5d..cd7d6c385 100644 --- a/seed-users.sql +++ b/seed-users.sql @@ -2,7 +2,7 @@ INSERT INTO public.users ( username, passhash, created_utc, admin_level, email_verified, original_username, defaultsorting, defaultsortingcomments, defaulttime, namecolor, flaircolor, theme, themecolor, reddit, pronouns, verified, profileurl, highres, - marsify, last_viewed_modmail_notifs, last_viewed_post_notifs, last_viewed_log_notifs, last_viewed_reddit_notifs, lifetimedonated, lifetimedonated_visible, show_sigs, grinch + marsify, last_viewed_modmail_notifs, last_viewed_post_notifs, last_viewed_log_notifs, last_viewed_offsite_notifs, lifetimedonated, lifetimedonated_visible, show_sigs, grinch ) VALUES ('AutoJanny', '', extract(epoch from now()), 0, true, 'AutoJanny', 'hot', 'top', 'day', 'ff459a', 'ff459a', 'dark', 'ff459a',