From 0e5218b2dce29154041a8b7b846fb98983254364 Mon Sep 17 00:00:00 2001 From: Aevann1 Date: Tue, 11 Jan 2022 21:46:50 +0200 Subject: [PATCH] bbbb --- files/helpers/alerts.py | 21 ++------ files/helpers/const.py | 6 +-- files/helpers/markdown.py | 102 -------------------------------------- files/helpers/sanitize.py | 25 +++++++++- files/helpers/wrappers.py | 20 +++----- files/routes/admin.py | 2 - files/routes/comments.py | 50 +++++++------------ files/routes/posts.py | 35 ++++++------- files/routes/settings.py | 15 ++---- files/routes/users.py | 8 +-- requirements.txt | 5 +- 11 files changed, 82 insertions(+), 207 deletions(-) delete mode 100644 files/helpers/markdown.py diff --git a/files/helpers/alerts.py b/files/helpers/alerts.py index f955350da..30c93a75b 100644 --- a/files/helpers/alerts.py +++ b/files/helpers/alerts.py @@ -1,8 +1,5 @@ -import mistletoe - from files.classes import * from flask import g -from .markdown import * from .sanitize import * from .const import * @@ -24,12 +21,7 @@ def send_repeatable_notification(uid, text, autojanny=False): if autojanny: author_id = AUTOJANNY_ID else: author_id = NOTIFICATIONS_ID - text_html = sanitize(Renderer2().render(mistletoe.Document(text))) - - for i in re.finditer("

@((\w|-){1,25})", text_html): - u = get_user(i.group(1), graceful=True) - if u: - text_html = text_html.replace(f'

@{u.username}', f'

@{u.username}') + text_html = sanitize(text, alert=True) existing_comment = g.db.query(Comment.id).filter_by(author_id=author_id, parent_submission=None, distinguish_level=6, body_html=text_html, created_utc=0).first() @@ -54,12 +46,7 @@ def notif_comment(text, autojanny=False): if autojanny: author_id = AUTOJANNY_ID else: author_id = NOTIFICATIONS_ID - text_html = sanitize(Renderer2().render(mistletoe.Document(text))) - - for i in re.finditer("

@((\w|-){1,25})", text_html): - u = get_user(i.group(1), graceful=True) - if u: - text_html = text_html.replace(f'

@{u.username}', f'

@{u.username}') + text_html = sanitize(text, alert=True) existing = g.db.query(Comment.id).filter_by(author_id=author_id, parent_submission=None, distinguish_level=6, body_html=text_html, created_utc=0).first() @@ -76,9 +63,7 @@ def add_notif(cid, uid): def send_admin(vid, text): - text_html = Renderer().render(mistletoe.Document(text)) - - text_html = sanitize(text_html, True) + text_html = sanitize(text, noimages=True) new_comment = Comment(author_id=vid, parent_submission=None, diff --git a/files/helpers/const.py b/files/helpers/const.py index 707ef22e8..ca38bb48d 100644 --- a/files/helpers/const.py +++ b/files/helpers/const.py @@ -154,7 +154,7 @@ if SITE == 'rdrama.net': AUTOPOLLER_ID = 6176 AUTOBETTER_ID = 7668 TAX_RECEIVER_ID = 995 - PIZZA_SHILL_ID = 2424 + AUTO_UPVOTE_IDS = (2424,4245) IDIO_ID = 30 CARP_ID = 995 JOAN_ID = 28 @@ -180,7 +180,7 @@ elif SITE == "pcmemes.net": AUTOPOLLER_ID = 3369 AUTOBETTER_ID = 1867 TAX_RECEIVER_ID = 1592 - PIZZA_SHILL_ID = 0 + AUTO_UPVOTE_IDS = () IDIO_ID = 0 CARP_ID = 0 JOAN_ID = 0 @@ -206,7 +206,7 @@ else: AUTOPOLLER_ID = 6 AUTOBETTER_ID = 7 TAX_RECEIVER_ID = 8 - PIZZA_SHILL_ID = 0 + AUTO_UPVOTE_IDS = () IDIO_ID = 0 CARP_ID = 0 JOAN_ID = 0 diff --git a/files/helpers/markdown.py b/files/helpers/markdown.py deleted file mode 100644 index 37d5f7831..000000000 --- a/files/helpers/markdown.py +++ /dev/null @@ -1,102 +0,0 @@ -from .get import * - -from mistletoe.span_token import SpanToken -from mistletoe.html_renderer import HTMLRenderer -import re -from flask import g - - -class UserMention(SpanToken): - - pattern = re.compile("(^|\s|\n)@((\w|-){1,25})") - parse_inner = False - def __init__(self, match_obj): - self.target = (match_obj.group(1), match_obj.group(2)) - -class SubMention(SpanToken): - - pattern = re.compile("(^|\s|\n)(r/|/r/)(\w{3,25})") - parse_inner = False - def __init__(self, match_obj): - self.target = (match_obj.group(1), match_obj.group(3)) - -class RedditorMention(SpanToken): - - pattern = re.compile("(^|\s|\n)(u/|/u/)((\w|-){3,25})") - parse_inner = False - def __init__(self, match_obj): - self.target = (match_obj.group(1), match_obj.group(3)) - -class CustomRenderer(HTMLRenderer): - - def __init__(self, **kwargs): - super().__init__(UserMention, SubMention, RedditorMention) - for i in kwargs: self.__dict__[i] = kwargs[i] - - def render_user_mention(self, token): - space = token.target[0] - target = token.target[1] - - user = get_user(target, graceful=True) - - if not user: return f"{space}@{target}" - - return f'''{space}@{user.username}'s profile picture@{user.username}''' - - def render_sub_mention(self, token): - space = token.target[0] - target = token.target[1] - return f'{space}r/{target}' - - def render_redditor_mention(self, token): - space = token.target[0] - target = token.target[1] - return f'{space}u/{target}' - - -class Renderer(HTMLRenderer): - - def __init__(self, **kwargs): - super().__init__(UserMention, SubMention, RedditorMention) - for i in kwargs: self.__dict__[i] = kwargs[i] - - def render_user_mention(self, token): - space = token.target[0] - target = token.target[1] - - user = get_user(target, graceful=True) - - if not user: return f"{space}@{target}" - - return f'{space}@{user.username}' - - def render_sub_mention(self, token): - space = token.target[0] - target = token.target[1] - return f'{space}r/{target}' - - def render_redditor_mention(self, token): - space = token.target[0] - target = token.target[1] - return f'{space}u/{target}' - -class Renderer2(HTMLRenderer): - - def __init__(self, **kwargs): - super().__init__(UserMention, SubMention, RedditorMention) - for i in kwargs: self.__dict__[i] = kwargs[i] - - def render_user_mention(self, token): - space = token.target[0] - target = token.target[1] - return f"{space}@{target}" - - def render_sub_mention(self, token): - space = token.target[0] - target = token.target[1] - return f"{space}r/{target}" - - def render_redditor_mention(self, token): - space = token.target[0] - target = token.target[1] - return f"{space}u/{target}" \ No newline at end of file diff --git a/files/helpers/sanitize.py b/files/helpers/sanitize.py index 068b43322..c8fd15b4e 100644 --- a/files/helpers/sanitize.py +++ b/files/helpers/sanitize.py @@ -5,6 +5,7 @@ from functools import partial from .get import * from os import path, environ import re +from markdown import markdown site = environ.get("DOMAIN").strip() @@ -97,9 +98,29 @@ allowed_protocols = ['http', 'https'] allowed_styles = ['color', 'background-color', 'font-weight', 'transform', '-webkit-transform'] -def sanitize(sanitized, noimages=False): +def sanitize(sanitized, noimages=False, alert=False): + + sanitized = markdown(sanitized) + + sanitized = sanitized.replace("\ufeff", "").replace("𒐪","").replace("@((\w|-){1,25})", sanitized): + u = get_user(i.group(1), graceful=True) + if u: + sanitized = re.sub("

@((\w|-){1,25})", f'

@{u.username}', sanitized) + else: + sanitized = re.sub('(^|\s|\n|

)\/?((r|u)\/\w{3,25})', r'\1\2', sanitized) + + for i in re.finditer('(^|\s|\n|

)@((\w|-){1,25})', sanitized): + u = get_user(i.group(2), graceful=True) + + if u and (not g.v.any_block_exists(u) or g.v.admin_level > 1): + if noimages: + sanitized = re.sub("(^|\s|\n|

)@((\w|-){1,25})", rf'\1@{u.username}', sanitized) + else: + sanitized = re.sub("(^|\s|\n|

)@((\w|-){1,25})", rf'\1@{u.username}', sanitized) - sanitized = sanitized.replace("\ufeff", "").replace("𒐪","").replace("[^<\s+]|[^>\s+]<', body_html))) > 0: return {"error":"You can only type marseys!"}, 403 @@ -322,9 +322,7 @@ def api_comment(v): body2 = f"@{basedguy.username}'s Based Count has increased by 1. Their Based Count is now {basedguy.basedcount}." if basedguy.pills: body2 += f"\n\nPills: {basedguy.pills}" - body_md = CustomRenderer().render(mistletoe.Document(body2)) - - body_based_html = sanitize(body_md) + body_based_html = sanitize(body2) c_based = Comment(author_id=BASEDBOT_ID, parent_submission=parent_submission, @@ -352,9 +350,7 @@ def api_comment(v): body = AGENDAPOSTER_MSG.format(username=v.username, type='comment') - body_md = CustomRenderer().render(mistletoe.Document(body)) - - body_jannied_html = sanitize(body_md) + body_jannied_html = sanitize(body) @@ -374,22 +370,11 @@ def api_comment(v): n = Notification(comment_id=c_jannied.id, user_id=v.id) g.db.add(n) - if v.id == PIZZA_SHILL_ID: - cratvote = CommentVote(user_id=TAX_RECEIVER_ID, comment_id=c.id, vote_type=1) - g.db.add(cratvote) - v.coins += 1 - v.truecoins += 1 - g.db.add(v) - c.upvotes += 1 - g.db.add(c) - if request.host == "rdrama.net" and len(c.body) >= 1000 and "<" not in body and "" not in body_html: body = random.choice(LONGPOST_REPLIES) - body_md = CustomRenderer().render(mistletoe.Document(body)) - body_html2 = sanitize(body_md) - + body_html2 = sanitize(body) c2 = Comment(author_id=LONGPOSTBOT_ID, parent_submission=parent_submission, @@ -421,8 +406,7 @@ def api_comment(v): if request.host == "rdrama.net" and random.random() < 0.001: body = "zoz" - body_md = CustomRenderer().render(mistletoe.Document(body)) - body_html2 = sanitize(body_md) + body_html2 = sanitize(body) @@ -446,8 +430,7 @@ def api_comment(v): body = "zle" - body_md = CustomRenderer().render(mistletoe.Document(body)) - body_html2 = sanitize(body_md) + body_html2 = sanitize(body) @@ -464,8 +447,7 @@ def api_comment(v): g.db.flush() body = "zozzle" - body_md = CustomRenderer().render(mistletoe.Document(body)) - body_html2 = sanitize(body_md) + body_html2 = sanitize(body) c4 = Comment(author_id=ZOZBOT_ID, @@ -536,6 +518,15 @@ def api_comment(v): c.voted = 1 + if v.id in AUTO_UPVOTE_IDS: + autovote = CommentVote(user_id=TAX_RECEIVER_ID, comment_id=c.id, vote_type=1) + g.db.add(autovote) + v.coins += 1 + v.truecoins += 1 + g.db.add(v) + c.upvotes += 1 + g.db.add(c) + g.db.commit() if request.headers.get("Authorization"): return c.json @@ -598,7 +589,7 @@ def edit_comment(cid, v): ) g.db.add(c_option) - body_html = sanitize(CustomRenderer().render(mistletoe.Document(body))) + body_html = sanitize(body) if v.marseyawarded and len(list(re.finditer('>[^<\s+]|[^>\s+]<', body_html))) > 0: return {"error":"You can only type marseys!"}, 403 @@ -675,8 +666,7 @@ def edit_comment(cid, v): body += f"\n\n{url}" else: return {"error": "Image/Video files only"}, 400 - body_md = CustomRenderer().render(mistletoe.Document(body)) - body_html = sanitize(body_md) + body_html = sanitize(body) if len(body_html) > 20000: abort(400) @@ -693,9 +683,7 @@ def edit_comment(cid, v): body = AGENDAPOSTER_MSG.format(username=v.username, type='comment') - body_md = CustomRenderer().render(mistletoe.Document(body)) - - body_jannied_html = sanitize(body_md) + body_jannied_html = sanitize(body) diff --git a/files/routes/posts.py b/files/routes/posts.py index ce5152c11..c3dfc6cfe 100644 --- a/files/routes/posts.py +++ b/files/routes/posts.py @@ -1,11 +1,9 @@ import time -import mistletoe import gevent import requests from files.helpers.wrappers import * from files.helpers.sanitize import * from files.helpers.filters import * -from files.helpers.markdown import * from files.helpers.alerts import * from files.helpers.discord import send_discord_message from files.helpers.const import * @@ -68,7 +66,7 @@ def publish(pid, v): add_notif(cid, x) - cid = notif_comment(f"@{v.username} has made a new post: [{post.title}]({post.permalink})", True) + cid = notif_comment(f"@{v.username} has made a new post: [{post.title}]({post.permalink})", autojanny=True) for follow in v.followers: user = get_account(follow.user_id) if post.club and not user.paid_dues: continue @@ -475,7 +473,7 @@ def edit_post(pid, v): ) g.db.add(c) - body_html = sanitize(CustomRenderer().render(mistletoe.Document(body))) + body_html = sanitize(body) bans = filter_comment_html(body_html) if bans: @@ -507,9 +505,7 @@ def edit_post(pid, v): body = AGENDAPOSTER_MSG.format(username=v.username, type='post') - body_md = CustomRenderer().render(mistletoe.Document(body)) - - body_jannied_html = sanitize(body_md) + body_jannied_html = sanitize(body) c_jannied = Comment(author_id=NOTIFICATIONS_ID, parent_submission=p.id, @@ -698,7 +694,7 @@ def thumbnail_thread(pid): for chunk in image_req.iter_content(1024): file.write(chunk) - post.thumburl = process_image(name, True) + post.thumburl = process_image(name, resize=True) db.add(post) db.commit() db.close() @@ -946,7 +942,7 @@ def submit_post(v): else: template = 'CHRISTMAS/' return render_template(f"{template}submit.html", v=v, error=f"Image/Video files only."), 400 - body_html = sanitize(CustomRenderer().render(mistletoe.Document(body))) + body_html = sanitize(body) if v.marseyawarded and len(list(re.finditer('>[^<\s+]|[^>\s+]<', body_html))) > 0: return {"error":"You can only type marseys!"}, 400 @@ -1029,7 +1025,7 @@ def submit_post(v): name2 = name.replace('.webp', 'r.webp') copyfile(name, name2) - new_post.thumburl = process_image(name2, True) + new_post.thumburl = process_image(name2, resize=True) elif file.content_type.startswith('video/'): file.save("video.mp4") with open("video.mp4", 'rb') as f: @@ -1051,7 +1047,7 @@ def submit_post(v): for x in notify_users: add_notif(cid, x) - cid = notif_comment(f"@{v.username} has made a new post: [{title}]({new_post.permalink})", True) + cid = notif_comment(f"@{v.username} has made a new post: [{title}]({new_post.permalink})", autojanny=True) for follow in v.followers: user = get_account(follow.user_id) if new_post.club and not user.paid_dues: continue @@ -1063,9 +1059,7 @@ def submit_post(v): body = AGENDAPOSTER_MSG.format(username=v.username, type='post') - body_md = CustomRenderer().render(mistletoe.Document(body)) - - body_jannied_html = sanitize(body_md) + body_jannied_html = sanitize(body) @@ -1122,8 +1116,7 @@ def submit_post(v): body += f'* [archive.ph](https://archive.ph/?url={quote(href)}&run=1) (click to archive)\n\n' gevent.spawn(archiveorg, href) - body_md = CustomRenderer().render(mistletoe.Document(body)) - body_html = sanitize(body_md) + body_html = sanitize(body) if len(body_html) < 20000: c = Comment(author_id=SNAPPY_ID, @@ -1158,8 +1151,16 @@ def submit_post(v): send_discord_message(f"https://{site}{new_post.permalink}") cache.delete_memoized(changeloglist) - g.db.commit() + if v.id in AUTO_UPVOTE_IDS: + autovote = Vote(user_id=TAX_RECEIVER_ID, submission_id=new_post.id, vote_type=1) + g.db.add(autovote) + v.coins += 1 + v.truecoins += 1 + g.db.add(v) + new_post.upvotes += 1 + g.db.add(new_post) + g.db.commit() if request.headers.get("Authorization"): return new_post.json else: return redirect(new_post.permalink) diff --git a/files/routes/settings.py b/files/routes/settings.py index 431332e26..523a4cdb0 100644 --- a/files/routes/settings.py +++ b/files/routes/settings.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from files.helpers.alerts import * from files.helpers.sanitize import * from files.helpers.filters import filter_comment_html -from files.helpers.markdown import * from files.helpers.discord import remove_user, set_nick from files.helpers.const import * from files.mail import * @@ -166,8 +165,7 @@ def settings_profile_post(v): for i in re.finditer('^(https:\/\/.*\.(png|jpg|jpeg|gif|webp|PNG|JPG|JPEG|GIF|WEBP|9999))', sig, re.MULTILINE): if "wikipedia" not in i.group(1): sig = sig.replace(i.group(1), f'![]({i.group(1)})') - sig_html = CustomRenderer().render(mistletoe.Document(sig)) - sig_html = sanitize(sig_html) + sig_html = sanitize(sig) bans = filter_comment_html(sig_html) @@ -205,8 +203,7 @@ def settings_profile_post(v): for i in re.finditer('^(https:\/\/.*\.(png|jpg|jpeg|gif|webp|PNG|JPG|JPEG|GIF|WEBP|9999))', friends, re.MULTILINE): if "wikipedia" not in i.group(1): friends = friends.replace(i.group(1), f'![]({i.group(1)})') - friends_html = CustomRenderer().render(mistletoe.Document(friends)) - friends_html = sanitize(friends_html) + friends_html = sanitize(friends) bans = filter_comment_html(friends_html) if bans: @@ -247,8 +244,7 @@ def settings_profile_post(v): for i in re.finditer('^(https:\/\/.*\.(png|jpg|jpeg|gif|webp|PNG|JPG|JPEG|GIF|WEBP|9999))', enemies, re.MULTILINE): if "wikipedia" not in i.group(1): enemies = enemies.replace(i.group(1), f'![]({i.group(1)})') - enemies_html = CustomRenderer().render(mistletoe.Document(enemies)) - enemies_html = sanitize(enemies_html) + enemies_html = sanitize(enemies) bans = filter_comment_html(enemies_html) if bans: @@ -309,8 +305,7 @@ def settings_profile_post(v): else: template = 'CHRISTMAS/' return render_template(f"{template}settings_profile.html", v=v, error="Image/Video files only."), 400 - bio_html = CustomRenderer().render(mistletoe.Document(bio)) - bio_html = sanitize(bio_html) + bio_html = sanitize(bio) bans = filter_comment_html(bio_html) if len(bio_html) > 10000: @@ -696,7 +691,7 @@ def settings_images_profile(v): name2 = name.replace('.webp', 'r.webp') copyfile(name, name2) - imageurl = process_image(name2, True) + imageurl = process_image(name2, resize=True) if not imageurl: abort(400) diff --git a/files/routes/users.py b/files/routes/users.py index bc16b976b..c4ac30be6 100644 --- a/files/routes/users.py +++ b/files/routes/users.py @@ -5,7 +5,6 @@ import math from files.classes.user import ViewerRelationship from files.helpers.alerts import * from files.helpers.sanitize import * -from files.helpers.markdown import * from files.helpers.const import * from files.mail import * from flask import * @@ -415,9 +414,7 @@ def message2(v, username): message = re.sub('!\[\]\((.*?)\)', r'\1', message) - text_html = Renderer().render(mistletoe.Document(message)) - - text_html = sanitize(text_html, True) + text_html = sanitize(message, noimages=True) existing = g.db.query(Comment.id).filter(Comment.author_id == v.id, Comment.sentto == user.id, @@ -482,8 +479,7 @@ def messagereply(v): if v.id == user_id: user_id = parent.sentto - text_html = Renderer().render(mistletoe.Document(message)) - text_html = sanitize(text_html, True) + text_html = sanitize(message, noimages=True) new_comment = Comment(author_id=v.id, parent_submission=None, diff --git a/requirements.txt b/requirements.txt index 0b6cb11cb..f444eafa5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -assertpy beautifulsoup4 bleach Flask @@ -9,10 +8,8 @@ Flask-Mail==0.9.1 gevent greenlet gunicorn -ImageHash +markdown matplotlib -mistletoe -piexif Pillow psutil pyotp