From 7047953ce7ffe458c71ae7effb7cc6039a384c35 Mon Sep 17 00:00:00 2001 From: Aevann1 Date: Sat, 16 Apr 2022 00:39:17 +0200 Subject: [PATCH] fdsdfs --- files/helpers/sanitize.py | 297 +++++++++--------------- files/templates/authforms.html | 4 +- files/templates/chat.html | 4 +- files/templates/default.html | 6 +- files/templates/formatting.html | 15 ++ files/templates/header.html | 2 +- files/templates/log.html | 4 +- files/templates/login.html | 2 +- files/templates/login_2fa.html | 2 +- files/templates/settings.html | 2 +- files/templates/settings2.html | 4 +- files/templates/sign_up.html | 2 +- files/templates/sign_up_failed_ref.html | 2 +- requirements.txt | 2 +- 14 files changed, 142 insertions(+), 206 deletions(-) diff --git a/files/helpers/sanitize.py b/files/helpers/sanitize.py index 478026f14..225b38f04 100644 --- a/files/helpers/sanitize.py +++ b/files/helpers/sanitize.py @@ -13,115 +13,23 @@ import signal import time import requests -allowed_tags = tags = ['b', - 'blockquote', - 'br', - 'code', - 'del', - 'em', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', - 'hr', - 'i', - 'li', - 'ol', - 'p', - 'pre', - 'strong', - 'sub', - 'sup', - 'table', - 'tbody', - 'th', - 'thead', - 'td', - 'tr', - 'ul', - 'marquee', - 'a', - 'img', - 'span', - 'ruby', - 'rp', - 'rt', - ] - -no_images = ['b', - 'blockquote', - 'br', - 'code', - 'del', - 'em', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', - 'hr', - 'i', - 'li', - 'ol', - 'p', - 'pre', - 'strong', - 'sub', - 'sup', - 'table', - 'tbody', - 'th', - 'thead', - 'td', - 'tr', - 'ul', - 'marquee', - 'a', - 'span', - 'ruby', - 'rp', - 'rt', - ] - -def sanitize_marquee(tag, name, value): - if name in allowed_attributes['*'] or name in ['direction', 'behavior', 'scrollamount']: return True - - if name in {'height', 'width'}: - try: value = int(value.replace('px', '')) - except: return False - if 0 < value <= 250: return True - - return False - -allowed_attributes = { - '*': ['href', 'style', 'src', 'title', 'loading'], - 'marquee': sanitize_marquee} - -allowed_protocols = ['http', 'https'] - -allowed_styles = ['color', 'background-color', 'font-weight', 'text-align'] - - -def handler(signum, frame): - print("Forever is over!") - raise Exception("end of time") - def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): + def handler(signum, frame): + print("Timeout!") + raise Exception("Timeout") + signal.signal(signal.SIGALRM, handler) signal.alarm(1) - + sanitized = image_check_regex.sub(r'\1', sanitized) sanitized = markdown(sanitized) sanitized = strikethrough_regex.sub(r'\1', sanitized) - sanitized = sanitized.replace('‎','').replace('​','').replace("\ufeff", "").replace("𒐪","").replace("","") + sanitized = sanitized.replace('‎','').replace('​','').replace("\ufeff", "").replace("𒐪","") if alert: captured = [] @@ -153,41 +61,14 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): sanitized = imgur_regex.sub(r'\1_d.webp?maxwidth=9999&fidelity=high', sanitized) - if noimages: - sanitized = bleach.Cleaner(tags=no_images, - attributes=allowed_attributes, - protocols=allowed_protocols, - styles=allowed_styles, - filters=[partial(LinkifyFilter, - skip_tags=["pre"], - parse_email=False, - ) - ] - ).clean(sanitized) - else: - sanitized = bleach.Cleaner(tags=allowed_tags, - attributes=allowed_attributes, - protocols=['http', 'https'], - styles=['color','font-weight','transform','-webkit-transform'], - filters=[partial(LinkifyFilter, - skip_tags=["pre"], - parse_email=False, - ) - ] - ).clean(sanitized) - soup = BeautifulSoup(sanitized, 'lxml') for tag in soup.find_all("img"): if tag.get("src") and not tag["src"].startswith('/pp/'): - tag["class"] = "img" tag["loading"] = "lazy" tag["data-src"] = tag["src"] tag["src"] = "/assets/images/loading.webp" tag['alt'] = f'![]({tag["data-src"]})' - tag["onclick"] = "expandDesktopImage(this.src);" - tag["data-bs-toggle"] = "modal" - tag["data-bs-target"] = "#expandImageModal" tag['referrerpolicy'] = "no-referrer" for tag in soup.find_all("a"): @@ -205,7 +86,7 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): sanitized = str(soup) - sanitized = spoiler_regex.sub(r'\1', sanitized) + sanitized = spoiler_regex.sub(r'\1', sanitized) if comment: marseys_used = set() @@ -218,38 +99,27 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): captured.append(i.group(0)) old = i.group(0) - if 'marseylong1' in old or 'marseylong2' in old or 'marseyllama1' in old or 'marseyllama2' in old: new = old.lower().replace(">", " class='mb-0'>") - else: new = old.lower() - + new = old.lower() + captured2 = [] for i in emoji_regex2.finditer(new): if i.group(0) in captured2: continue captured2.append(i.group(0)) emoji = i.group(1).lower() - if emoji.startswith("#!") or emoji.startswith("!#"): - classes = 'emoji-lg mirrored' - remoji = emoji[2:] - elif emoji.startswith("#"): - classes = 'emoji-lg' - remoji = emoji[1:] - elif emoji.startswith("!"): - classes = 'emoji-md mirrored' - remoji = emoji[1:] - else: - classes = 'emoji-md' - remoji = emoji + remoji = emoji.replace('!','').replace('#','') - if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): classes += ' golden' + golden = ' ' + if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): golden = 'g ' if remoji == 'marseyrandom': remoji = choice(marseys_const2) if path.isfile(f'files/assets/images/emojis/{remoji}.webp'): - new = re.sub(f'(?', new, flags=re.I|re.A) + new = re.sub(f'(?', new, flags=re.I|re.A) if comment: marseys_used.add(emoji) elif remoji.endswith('pat') and path.isfile(f"files/assets/images/emojis/{remoji.replace('pat','')}.webp"): pat(remoji.replace('pat','')) - new = re.sub(f'(?', new, flags=re.I|re.A) + new = re.sub(f'(?', new, flags=re.I|re.A) requests.post(f'https://api.cloudflare.com/client/v4/zones/{CF_ZONE}/purge_cache', headers=CF_HEADERS, data={'files': [f"https://{request.host}/e/{emoji}.webp"]}, timeout=5) @@ -263,30 +133,21 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): if i.group(0) in captured: continue captured.append(i.group(0)) - emoji = i.group(1).lower().replace('#','') - if emoji.startswith("!"): - emoji = emoji[1:] - classes = 'emoji mirrored' - if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): classes += ' golden' + emoji = i.group(1).lower() + golden = ' ' + if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): golden = 'g ' - old = emoji - if emoji == 'marseyrandom': emoji = choice(marseys_const2) - else: emoji = old - else: - classes = 'emoji' - if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): classes += ' golden' - - old = emoji - if emoji == 'marseyrandom': emoji = choice(marseys_const2) - else: emoji = old + old = emoji + emoji = emoji.replace('!','').replace('#','') + if emoji == 'marseyrandom': emoji = choice(marseys_const2) if path.isfile(f'files/assets/images/emojis/{emoji}.webp'): - sanitized = re.sub(f'(?', sanitized, flags=re.I|re.A) + sanitized = re.sub(f'(?', sanitized, flags=re.I|re.A) if comment: marseys_used.add(emoji) elif emoji.endswith('pat') and path.isfile(f"files/assets/images/emojis/{emoji.replace('pat','')}.webp"): pat(emoji.replace('pat','')) - sanitized = re.sub(f'(?', sanitized, flags=re.I|re.A) + sanitized = re.sub(f'(?', sanitized, flags=re.I|re.A) requests.post(f'https://api.cloudflare.com/client/v4/zones/{CF_ZONE}/purge_cache', headers=CF_HEADERS, data={'files': [f"https://{request.host}/e/{emoji}.webp"]}, timeout=5) @@ -299,6 +160,7 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): if "https://youtube.com/watch?v=" in sanitized: sanitized = sanitized.replace("?t=", "&t=") captured = [] + print(sanitized,flush=True) for i in youtube_regex.finditer(sanitized): if i.group(0) in captured: continue captured.append(i.group(0)) @@ -323,7 +185,7 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): sanitized = unlinked_regex.sub(r'\1\2', sanitized) if not noimages: - sanitized = video_regex.sub(r'

', sanitized) + sanitized = video_regex.sub(r'

', sanitized) if comment: for marsey in g.db.query(Marsey).filter(Marsey.name.in_(marseys_used)).all(): @@ -338,22 +200,81 @@ def sanitize(sanitized, noimages=False, alert=False, comment=False, edit=False): sanitized = utm_regex.sub('', sanitized) sanitized = utm_regex2.sub('', sanitized) + + sanitized = sanitized.replace('','').replace('','') + + + allowed_tags = ['b','blockquote','br','code','del','em','h1','h2','h3','h4','h5','h6','hr','i','li','ol','p','pre','strong','sub','sup','table','tbody','th','thead','td','tr','ul','marquee','a','span','ruby','rp','rt','spoiler'] + if not noimages: allowed_tags += ['img','video','lite-youtube','source'] + + def allowed_attributes(tag, name, value): + + if name == 'style': return True + + if tag == 'marquee': + if name in ['direction', 'behavior', 'scrollamount']: return True + if name in {'height', 'width'}: + try: value = int(value.replace('px', '')) + except: return False + if 0 < value <= 250: return True + return False + + if tag == 'a': + if name == 'href': return True + if name == 'rel' and value == 'nofollow noopener noreferrer': return True + if name == 'target' and value == '_blank': return True + return False + + if tag == 'img': + if name == 'loading' and value == 'lazy': return True + if name == 'referrpolicy' and value == 'no-referrer': return True + if name == 'data-bs-toggle' and value == 'tooltip': return True + if name in ['src','data-src','alt','title','g','b']: return True + return False + + if tag == 'lite-youtube': + if name == 'params' and value.startswith('autoplay=1&modestbranding=1'): return True + if name == 'videoid': return True + return False + + if tag == 'video': + if name == 'controls' and value == '': return True + if name == 'preload' and value == 'none': return True + return False + + if tag == 'source': + if name == 'src': return True + return False + + + + sanitized = bleach.Cleaner(tags=allowed_tags, + attributes=allowed_attributes, + protocols=['http', 'https'], + styles=['color', 'background-color', 'font-weight', 'text-align'], + filters=[partial(LinkifyFilter,skip_tags=["pre"],parse_email=False)] + ).clean(sanitized) + + + signal.alarm(0) return sanitized -def handler2(signum, frame): - print("Forever is over!") - raise Exception("end of time") + + + def filter_emojis_only(title, edit=False, graceful=False): - signal.signal(signal.SIGALRM, handler2) + def handler(signum, frame): + print("Timeout!") + raise Exception("Timeout") + + signal.signal(signal.SIGALRM, handler) signal.alarm(1) - title = title.replace('‎','').replace('​','').replace("\ufeff", "").replace("𒐪","").replace("\n", "").replace("\r", "").replace("\t", "").replace("","").replace("&", "&").replace('<','<').replace('>','>').replace('"', '"').replace("'", "'").strip() - - title = bleach.clean(title, tags=[]) + title = title.replace('‎','').replace('​','').replace("\ufeff", "").replace("𒐪","").replace("\n", "").replace("\r", "").replace("\t", "").replace("&", "&").replace('<','<').replace('>','>').replace('"', '"').replace("'", "'").strip() emojis = list(emoji_regex4.finditer(title)) if len(emojis) > 20: edit = True @@ -364,35 +285,35 @@ def filter_emojis_only(title, edit=False, graceful=False): captured.append(i.group(0)) emoji = i.group(1).lower() + golden = ' ' + if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): golden = 'g ' - if emoji.startswith("!"): - emoji = emoji[1:] - classes = 'emoji mirrored' - if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): classes += ' golden' - - old = emoji - if emoji == 'marseyrandom': emoji = choice(marseys_const2) - else: emoji = old - old = '!' + emoji - else: - classes = 'emoji' - if not edit and random() < 0.0025 and ('marsey' in emoji or emoji in marseys_const2): classes += ' golden' - - old = emoji - if emoji == 'marseyrandom': emoji = choice(marseys_const2) - else: emoji = old - + old = emoji + emoji = emoji.replace('!','').replace('#','') + if emoji == 'marseyrandom': emoji = choice(marseys_const2) if path.isfile(f'files/assets/images/emojis/{emoji}.webp'): - title = re.sub(f'(?', title, flags=re.I|re.A) + title = re.sub(f'(?', title, flags=re.I|re.A) elif emoji.endswith('pat') and path.isfile(f"files/assets/images/emojis/{emoji.replace('pat','')}.webp"): pat(emoji.replace('pat','')) - title = re.sub(f'(?', title, flags=re.I|re.A) + title = re.sub(f'(?', title, flags=re.I|re.A) requests.post(f'https://api.cloudflare.com/client/v4/zones/{CF_ZONE}/purge_cache', headers=CF_HEADERS, data={'files': [f"https://{request.host}/e/{emoji}.webp"]}, timeout=5) title = strikethrough_regex.sub(r'\1', title) + + def allowed_attributes(tag, name, value): + + if tag == 'img': + if name == 'loading' and value == 'lazy': return True + if name == 'data-bs-toggle' and value == 'tooltip': return True + if name in ['src','alt','title','g']: return True + return False + + + sanitized = bleach.clean(title, tags=['img','del'], attributes=allowed_attributes, protocols=['http','https']) + signal.alarm(0) if len(title) > 1500 and not graceful: abort(400) diff --git a/files/templates/authforms.html b/files/templates/authforms.html index 79d6ad88c..2a8de8a7c 100644 --- a/files/templates/authforms.html +++ b/files/templates/authforms.html @@ -15,7 +15,7 @@ {% if v %} - + {% if v.agendaposter %} - + {% endif %} diff --git a/files/templates/chat.html b/files/templates/chat.html index 3743a0c86..660fe4313 100644 --- a/files/templates/chat.html +++ b/files/templates/chat.html @@ -14,7 +14,7 @@ Chat - + {% if v.css %} @@ -197,7 +197,7 @@ {% include "emoji_modal.html" %} {% include "expanded_image_modal.html" %} - + \ No newline at end of file diff --git a/files/templates/default.html b/files/templates/default.html index 998b458b5..75cca4b24 100644 --- a/files/templates/default.html +++ b/files/templates/default.html @@ -7,7 +7,7 @@ {% if v %} - + {% if v.agendaposter %} - + {% endif %} @@ -325,7 +325,7 @@ - + {% if v %} diff --git a/files/templates/formatting.html b/files/templates/formatting.html index 96e22dcef..e18e04387 100644 --- a/files/templates/formatting.html +++ b/files/templates/formatting.html @@ -46,6 +46,21 @@ You can use Markdown formatting: ~~text~~ text + + Horizontal Rule + + 

+Text 1
+---
+Text 2
+
+ + + Text 1 +
+ Text 2 + + Links [{{SITE_NAME}}]({{SITE_FULL}}/) diff --git a/files/templates/header.html b/files/templates/header.html index 32d4be076..c97fcd18c 100644 --- a/files/templates/header.html +++ b/files/templates/header.html @@ -259,7 +259,7 @@ - + {% if v and not err %}
{{v.formkey}}
diff --git a/files/templates/log.html b/files/templates/log.html index 5c79ba170..dac5e0391 100644 --- a/files/templates/log.html +++ b/files/templates/log.html @@ -6,7 +6,7 @@ {% block content %} {% if v %} - + {% if v.agendaposter %} - + {% endif %}
diff --git a/files/templates/login.html b/files/templates/login.html index 62ed41239..92aa130b6 100644 --- a/files/templates/login.html +++ b/files/templates/login.html @@ -18,7 +18,7 @@ {% endblock %} - + diff --git a/files/templates/login_2fa.html b/files/templates/login_2fa.html index 119bf7aef..82bb95ba1 100644 --- a/files/templates/login_2fa.html +++ b/files/templates/login_2fa.html @@ -14,7 +14,7 @@ 2-Step Login - {{SITE_NAME}} - + diff --git a/files/templates/settings.html b/files/templates/settings.html index 9c986d300..682fd4fd3 100644 --- a/files/templates/settings.html +++ b/files/templates/settings.html @@ -34,7 +34,7 @@ - + {% if v.agendaposter %} - + {% else %} - + {% endif %} diff --git a/files/templates/sign_up.html b/files/templates/sign_up.html index bc524d49e..3ddb1c340 100644 --- a/files/templates/sign_up.html +++ b/files/templates/sign_up.html @@ -31,7 +31,7 @@ {% if ref_user %}{{ref_user.username}} invites you to {{SITE_NAME}}{% else %}Sign up - {{SITE_NAME}}{% endif %} - + diff --git a/files/templates/sign_up_failed_ref.html b/files/templates/sign_up_failed_ref.html index 2ed139ec3..85638f5e8 100644 --- a/files/templates/sign_up_failed_ref.html +++ b/files/templates/sign_up_failed_ref.html @@ -32,7 +32,7 @@ {% if ref_user %}{{ref_user.username}} invites you to {{SITE_NAME}}{% else %}{{SITE_NAME}}{% endif %} - + diff --git a/requirements.txt b/requirements.txt index a0093943e..72acaa12c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4 -bleach +bleach==4.1.0 Flask Flask-Caching Flask-Compress