carpathianflorist
/
MarseyWorld
forked from MarseyWorld/MarseyWorld
1
0
Fork 0
Code Pull Requests Activity
MarseyWorld/files/helpers/sanitize.py

258 lines
8.7 KiB
Python
Raw Normal View History

sneed
2021-07-21 01:12:26 +00:00
import bleach
from bs4 import BeautifulSoup
from bleach.linkifier import LinkifyFilter
from functools import partial
from .get import *
fd
2021-09-14 13:40:52 +00:00
from os import path, environ
fd
2021-09-14 13:43:35 +00:00
import re
sneed
2021-07-21 01:12:26 +00:00
fggfgf
2021-08-05 14:41:32 +00:00
site = environ.get("DOMAIN").strip()
fdfddf
2021-08-02 14:27:20 +00:00
sneed
2021-07-21 01:12:26 +00:00
_allowed_tags = tags = ['b',
'blockquote',
'br',
'code',
'del',
'em',
'h1',
'h2',
'h3',
'h4',
'h5',
'h6',
'hr',
'i',
'li',
'ol',
'p',
'pre',
'strong',
'sub',
'sup',
'table',
'tbody',
'th',
'thead',
'td',
'tr',
'ul',
'marquee',
'a',
'img',
'span',
]
fdfd
2021-08-31 21:58:25 +00:00
no_images = ['b',
fdfd
2021-08-31 21:54:34 +00:00
'blockquote',
'br',
'code',
'del',
'em',
'h1',
'h2',
'h3',
'h4',
'h5',
'h6',
'hr',
'i',
'li',
'ol',
'p',
'pre',
'strong',
'sub',
'sup',
'table',
'tbody',
'th',
'thead',
'td',
'tr',
'ul',
'marquee',
'a',
'span',
]
sneed
2021-07-21 01:12:26 +00:00
_allowed_attributes = {
gfdg
2021-10-06 00:25:59 +00:00
'*': ['href', 'style', 'src', 'class', 'title', 'rel', 'data-bs-original-name', 'direction']
sneed
2021-07-21 01:12:26 +00:00
}
_allowed_protocols = [
'http',
'https'
]
_allowed_styles =[
'color',
dfsfsd
2021-10-05 14:37:39 +00:00
'font-weight',
dsfsfd
2021-10-06 00:41:58 +00:00
'transform',
'-webkit-transform',
sneed
2021-07-21 01:12:26 +00:00
]
fdfd
2021-08-31 21:54:34 +00:00
def sanitize(sanitized, noimages=False):
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
sanitized = sanitized.replace("\ufeff", "").replace("m.youtube.com", "youtube.com")
sneed
2021-07-21 01:12:26 +00:00
dffd
2021-08-21 16:27:23 +00:00
for i in re.finditer('https://i.imgur.com/(([^_]*?)\.(jpg|png|jpeg))', sanitized):
fgfg
2021-08-21 16:25:34 +00:00
sanitized = sanitized.replace(i.group(1), i.group(2) + "_d." + i.group(3) + "?maxwidth=9999")
sneed
2021-07-21 01:12:26 +00:00
fdfd
2021-08-31 21:54:34 +00:00
if noimages:
fdfd
2021-08-31 21:58:25 +00:00
sanitized = bleach.Cleaner(tags=no_images,
fdfd
2021-08-31 21:54:34 +00:00
attributes=_allowed_attributes,
protocols=_allowed_protocols,
styles=_allowed_styles,
filters=[partial(LinkifyFilter,
skip_tags=["pre"],
parse_email=False,
)
]
).clean(sanitized)
else:
sanitized = bleach.Cleaner(tags=_allowed_tags,
attributes=_allowed_attributes,
protocols=_allowed_protocols,
styles=_allowed_styles,
filters=[partial(LinkifyFilter,
skip_tags=["pre"],
parse_email=False,
)
]
).clean(sanitized)
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
soup = BeautifulSoup(sanitized, features="html.parser")
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
for tag in soup.find_all("img"):
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
url = tag.get("src", "")
if not url: continue
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
if "profile-pic-20" not in tag.get("class", ""):
sneed
2021-07-21 01:12:26 +00:00
fd
2021-09-02 20:33:48 +00:00
tag["rel"] = "nofollow noopener noreferrer"
df
2021-08-21 12:30:54 +00:00
tag["style"] = "max-height: 100px; max-width: 100%;"
dfsfdsfsd
2021-10-06 09:23:55 +00:00
tag["class"] = "in-comment-image rounded-sm my-2"
dffd
2021-09-03 14:08:32 +00:00
tag["loading"] = "lazy"
dffds
2021-10-06 09:14:08 +00:00
tag["data-src"] = tag["src"]
tag["src"] = ""
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
link = soup.new_tag("a")
link["href"] = tag["src"]
fd
2021-09-02 20:33:48 +00:00
link["rel"] = "nofollow noopener noreferrer"
df
2021-08-21 12:30:54 +00:00
link["target"] = "_blank"
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
link["onclick"] = f"expandDesktopImage('{tag['src']}');"
gf
2021-09-26 09:04:49 +00:00
link["data-bs-toggle"] = "modal"
link["data-bs-target"] = "#expandImageModal"
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
tag.wrap(link)
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
#disguised link preventer
for tag in soup.find_all("a"):
sneed
2021-07-21 01:12:26 +00:00
gfdg
2021-10-06 00:25:59 +00:00
tag["target"] = "_blank"
if site not in tag["href"]: tag["rel"] = "nofollow noopener noreferrer"
df
2021-08-21 12:30:54 +00:00
if re.match("https?://\S+", str(tag.string)):
try:
tag.string = tag["href"]
except:
tag.string = ""
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
#clean up tags in code
for tag in soup.find_all("code"):
tag.contents=[x.string for x in tag.contents if x.string]
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
#whatever else happens with images, there are only two sets of classes allowed
for tag in soup.find_all("img"):
if 'profile-pic-20' not in tag.attrs.get("class",""):
dfsfdsfsd
2021-10-06 09:23:55 +00:00
tag.attrs['class']="in-comment-image rounded-sm my-2"
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
#table format
for tag in soup.find_all("table"):
tag.attrs['class']="table table-striped"
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
for tag in soup.find_all("thead"):
tag.attrs['class']="bg-primary text-white"
sneed
2021-07-21 01:12:26 +00:00
df
2021-08-21 12:30:54 +00:00
sanitized = str(soup)
sneed
2021-07-21 01:12:26 +00:00
start = '<s>'
end = '</s>'
gf
2021-09-05 19:47:05 +00:00
fd
2021-09-06 22:26:28 +00:00
try:
if not session.get("favorite_emojis"): session["favorite_emojis"] = {}
fd
2021-09-07 02:07:11 +00:00
except:
fd
2021-09-06 22:26:28 +00:00
pass
gf
2021-09-05 19:47:05 +00:00
gfd
2021-10-06 05:40:27 +00:00
if start in sanitized and end in sanitized and start in sanitized.split(end)[0] and end in sanitized.split(start)[1]: sanitized = sanitized.replace(start, '<span class="spoiler">').replace(end, '</span>')
sneed
2021-07-21 01:12:26 +00:00
dfsdfs
2021-10-06 05:37:07 +00:00
for i in re.finditer("[^a]>\s*(:!?\w+:\s*)+<\/", sanitized):
dfsfds
2021-10-01 00:23:26 +00:00
old = i.group(0)
dfsfds
2021-10-06 09:09:14 +00:00
if 'marseylong' in old: new = old.lower().replace(">", " style='margin-bottom:0 !important'>")
fdgf
2021-10-06 09:12:58 +00:00
else: new = old.lower()
fgfggf
2021-10-05 20:45:15 +00:00
for i in re.finditer('\w*(?<!"):([^ ]{1,30}?):', new):
emoji = i.group(1).lower()
dfssfd
2021-10-05 20:59:15 +00:00
if emoji.startswith("!"):
df
2021-10-05 21:26:16 +00:00
emoji = emoji[1:]
if path.isfile(f'./files/assets/images/emojis/{emoji}.webp'):
dfsfdsfsd
2021-10-06 09:23:55 +00:00
new = re.sub(f'\w*(?<!"):!{emoji}:', f'<img loading="lazy" data-bs-toggle="tooltip" alt=":!{emoji}:" title=":!{emoji}:" delay="0" height=60 data-src="https://{site}/assets/images/emojis/{emoji}.webp" class="mirrored">', new)
df
2021-10-05 21:26:16 +00:00
elif path.isfile(f'./files/assets/images/emojis/{emoji}.webp'):
dfsfdsfsd
2021-10-06 09:23:55 +00:00
new = re.sub(f'\w*(?<!"):{emoji}:', f'<img loading="lazy" data-bs-toggle="tooltip" alt=":{emoji}:" title=":{emoji}:" delay="0" height=60 data-src="https://{site}/assets/images/emojis/{emoji}.webp">', new)
df
2021-10-05 21:26:16 +00:00
if emoji in session["favorite_emojis"]: session["favorite_emojis"][emoji] += 1
else: session["favorite_emojis"][emoji] = 1
dfssfd
2021-10-01 00:21:33 +00:00
dfssfd
2021-10-05 20:59:15 +00:00
sanitized = sanitized.replace(old, new)
dfsfds
2021-10-01 00:13:20 +00:00
fd
2021-10-01 00:24:39 +00:00
for i in re.finditer('\w*(?<!"):([^ ]{1,30}?):', sanitized):
emoji = i.group(1).lower()
dfssfd
2021-10-05 20:59:15 +00:00
if emoji.startswith("!"):
df
2021-10-05 21:26:16 +00:00
emoji = emoji[1:]
if path.isfile(f'./files/assets/images/emojis/{emoji}.webp'):
dfsfdsfsd
2021-10-06 09:23:55 +00:00
sanitized = re.sub(f'\w*(?<!"):!{emoji}:', f'<img loading="lazy" data-bs-toggle="tooltip" alt=":!{emoji}:" title=":!{emoji}:" delay="0" height=30 data-src="https://{site}/assets/images/emojis/{emoji}.webp" class="mirrored">', sanitized)
df
2021-10-05 21:26:16 +00:00
elif path.isfile(f'./files/assets/images/emojis/{emoji}.webp'):
dfsfdsfsd
2021-10-06 09:23:55 +00:00
sanitized = re.sub(f'\w*(?<!"):{emoji}:', f'<img loading="lazy" data-bs-toggle="tooltip" alt=":{emoji}:" title=":{emoji}:" delay="0" height=30 data-src="https://{site}/assets/images/emojis/{emoji}.webp">', sanitized)
df
2021-10-05 21:26:16 +00:00
if emoji in session["favorite_emojis"]: session["favorite_emojis"][emoji] += 1
else: session["favorite_emojis"][emoji] = 1
sneed
2021-07-21 01:12:26 +00:00
gfdg
2021-10-05 20:23:31 +00:00
gf
2021-09-23 20:43:11 +00:00
sanitized = sanitized.replace("https://www.", "https://").replace("https://youtu.be/", "https://youtube.com/watch?v=").replace("https://music.youtube.com/watch?v=", "https://youtube.com/watch?v=").replace("https://open.spotify.com/", "https://open.spotify.com/embed/").replace("https://streamable.com/", "https://streamable.com/e/").replace("https://youtube.com/shorts/", "https://youtube.com/watch?v=").replace("https://mobile.twitter", "https://twitter").replace("https://m.facebook", "https://facebook").replace("https://m.wikipedia", "https://wikipedia").replace("https://m.youtube", "https://youtube")
gfgf
2021-08-22 19:51:19 +00:00
for i in re.finditer('" target="_blank">(https://youtube.com/watch\?v\=.*?)</a>', sanitized):
url = i.group(1)
fsdsfd
2021-10-06 02:06:07 +00:00
replacing = f'<a href="{url}" rel="nofollow noopener noreferrer" target="_blank">{url}</a>'
dfsfdsfsd
2021-10-06 09:23:55 +00:00
htmlsource = f'<div class="embed-responsive embed-responsive-16by9 mb-3"><iframe loading="lazy" data-src="{url}" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></div>'
gfgf
2021-08-22 19:51:19 +00:00
sanitized = sanitized.replace(replacing, htmlsource.replace("watch?v=", "embed/"))
for i in re.finditer('<a href="(https://streamable.com/e/.*?)"', sanitized):
sneed
2021-07-21 01:12:26 +00:00
url = i.group(1)
fsdsfd
2021-10-06 02:06:07 +00:00
replacing = f'<a href="{url}" rel="nofollow noopener noreferrer" target="_blank">{url}</a>'
dfsfdsfsd
2021-10-06 09:23:55 +00:00
htmlsource = f'<div class="embed-responsive embed-responsive-16by9 mb-3"><iframe loading="lazy" data-src="{url}" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></div>'
sneed
2021-07-21 01:12:26 +00:00
sanitized = sanitized.replace(replacing, htmlsource)
gfgf
2021-08-22 19:51:19 +00:00
fdsfsdgfd
2021-10-01 20:30:05 +00:00
for i in re.finditer('<p>(https:.*?\.mp4)</p>', sanitized):
dfsfdsfsd
2021-10-06 09:23:55 +00:00
sanitized = sanitized.replace(i.group(0), f'<p><video controls loop preload="metadata" style="max-width: 100%"><source data-src="{i.group(1)}" type="video/mp4"></video>')
sfdsdf
2021-10-01 20:25:15 +00:00
sneed
2021-07-21 01:12:26 +00:00
for i in re.finditer('<a href="(https://open.spotify.com/embed/.*?)"', sanitized):
url = i.group(1)
fsdsfd
2021-10-06 02:06:07 +00:00
replacing = f'<a href="{url}" rel="nofollow noopener noreferrer" target="_blank">{url}</a>'
dfsfdsfsd
2021-10-06 09:23:55 +00:00
htmlsource = f'<iframe data-src="{url}" width="100%" height="80" frameBorder="0" allowtransparency="true" allow="encrypted-media"></iframe>'
sneed
2021-07-21 01:12:26 +00:00
sanitized = sanitized.replace(replacing, htmlsource)
for rd in ["https://reddit.com/", "https://new.reddit.com/", "https://www.reddit.com/", "https://redd.it/"]:
sanitized = sanitized.replace(rd, "https://old.reddit.com/")
fdfd
2021-09-01 21:16:51 +00:00
fd
2021-09-02 20:33:48 +00:00
sanitized = re.sub(' (https:\/\/[^ <>]*)', r' <a target="_blank" rel="nofollow noopener noreferrer" href="\1">\1</a>', sanitized)
fix emoji tags
2021-09-19 11:02:35 +00:00
sanitized = re.sub('<p>(https:\/\/[^ <>]*)', r'<p><a target="_blank" rel="nofollow noopener noreferrer" href="\1">\1</a></p>', sanitized)
fd
2021-09-01 20:58:53 +00:00
fix controversial sorting setting for comments
2021-08-15 19:17:19 +00:00
return sanitized