2022-07-05 22:11:45 +00:00
import functools
2022-05-04 23:09:46 +00:00
import bleach
from bs4 import BeautifulSoup
2022-05-25 00:27:41 +00:00
from bleach . css_sanitizer import CSSSanitizer
2022-07-15 13:27:45 +00:00
from bleach . linkifier import LinkifyFilter
2022-05-04 23:09:46 +00:00
from functools import partial
from . get import *
from os import path , environ
import re
from mistletoe import markdown
from json import loads , dump
from random import random , choice
import signal
import time
import requests
2022-05-24 19:16:55 +00:00
TLDS = ( # Original gTLDs and ccTLDs
2022-06-12 02:08:48 +00:00
' ac ' , ' ad ' , ' ae ' , ' aero ' , ' af ' , ' ag ' , ' ai ' , ' al ' , ' am ' , ' an ' , ' ao ' , ' aq ' , ' ar ' , ' arpa ' , ' as ' , ' asia ' , ' at ' ,
2022-05-24 19:16:55 +00:00
' au ' , ' aw ' , ' ax ' , ' az ' , ' ba ' , ' bb ' , ' bd ' , ' be ' , ' bf ' , ' bg ' , ' bh ' , ' bi ' , ' biz ' , ' bj ' , ' bm ' , ' bn ' , ' bo ' , ' br ' ,
' bs ' , ' bt ' , ' bv ' , ' bw ' , ' by ' , ' bz ' , ' ca ' , ' cafe ' , ' cat ' , ' cc ' , ' cd ' , ' cf ' , ' cg ' , ' ch ' , ' ci ' , ' ck ' , ' cl ' ,
' cm ' , ' cn ' , ' co ' , ' com ' , ' coop ' , ' cr ' , ' cu ' , ' cv ' , ' cx ' , ' cy ' , ' cz ' , ' de ' , ' dj ' , ' dk ' , ' dm ' , ' do ' , ' dz ' , ' ec ' ,
' edu ' , ' ee ' , ' eg ' , ' er ' , ' es ' , ' et ' , ' eu ' , ' fi ' , ' fj ' , ' fk ' , ' fm ' , ' fo ' , ' fr ' , ' ga ' , ' gb ' , ' gd ' , ' ge ' , ' gf ' ,
' gg ' , ' gh ' , ' gi ' , ' gl ' , ' gm ' , ' gn ' , ' gov ' , ' gp ' , ' gq ' , ' gr ' , ' gs ' , ' gt ' , ' gu ' , ' gw ' , ' gy ' , ' hk ' , ' hm ' , ' hn ' ,
' hr ' , ' ht ' , ' hu ' , ' id ' , ' ie ' , ' il ' , ' im ' , ' in ' , ' info ' , ' int ' , ' io ' , ' iq ' , ' ir ' , ' is ' , ' it ' , ' je ' , ' jm ' , ' jo ' ,
' jobs ' , ' jp ' , ' ke ' , ' kg ' , ' kh ' , ' ki ' , ' km ' , ' kn ' , ' kp ' , ' kr ' , ' kw ' , ' ky ' , ' kz ' , ' la ' , ' lb ' , ' lc ' , ' li ' , ' lk ' ,
' lr ' , ' ls ' , ' lt ' , ' lu ' , ' lv ' , ' ly ' , ' ma ' , ' mc ' , ' md ' , ' me ' , ' mg ' , ' mh ' , ' mil ' , ' mk ' , ' ml ' , ' mm ' , ' mn ' , ' mo ' ,
' mobi ' , ' mp ' , ' mq ' , ' mr ' , ' ms ' , ' mt ' , ' mu ' , ' museum ' , ' mv ' , ' mw ' , ' mx ' , ' my ' , ' mz ' , ' na ' , ' name ' ,
' nc ' , ' ne ' , ' net ' , ' nf ' , ' ng ' , ' ni ' , ' nl ' , ' no ' , ' np ' , ' nr ' , ' nu ' , ' nz ' , ' om ' , ' org ' , ' pa ' , ' pe ' , ' pf ' , ' pg ' ,
' ph ' , ' pk ' , ' pl ' , ' pm ' , ' pn ' , ' post ' , ' pr ' , ' pro ' , ' ps ' , ' pt ' , ' pw ' , ' py ' , ' qa ' , ' re ' , ' ro ' , ' rs ' , ' ru ' , ' rw ' ,
' sa ' , ' sb ' , ' sc ' , ' sd ' , ' se ' , ' sg ' , ' sh ' , ' si ' , ' sj ' , ' sk ' , ' sl ' , ' sm ' , ' sn ' , ' so ' , ' social ' , ' sr ' , ' ss ' , ' st ' ,
' su ' , ' sv ' , ' sx ' , ' sy ' , ' sz ' , ' tc ' , ' td ' , ' tel ' , ' tf ' , ' tg ' , ' th ' , ' tj ' , ' tk ' , ' tl ' , ' tm ' , ' tn ' , ' to ' , ' tp ' ,
' tr ' , ' travel ' , ' tt ' , ' tv ' , ' tw ' , ' tz ' , ' ua ' , ' ug ' , ' uk ' , ' us ' , ' uy ' , ' uz ' , ' va ' , ' vc ' , ' ve ' , ' vg ' , ' vi ' , ' vn ' ,
' vu ' , ' wf ' , ' ws ' , ' xn ' , ' xxx ' , ' ye ' , ' yt ' , ' yu ' , ' za ' , ' zm ' , ' zw ' ,
# New gTLDs
2022-08-21 13:29:35 +00:00
' app ' , ' cleaning ' , ' club ' , ' dev ' , ' farm ' , ' florist ' , ' fun ' , ' gay ' , ' lgbt ' , ' life ' , ' lol ' ,
2022-06-21 05:03:33 +00:00
' moe ' , ' mom ' , ' monster ' , ' new ' , ' news ' , ' online ' , ' pics ' , ' press ' , ' pub ' , ' site ' ,
2022-08-22 22:13:32 +00:00
' vip ' , ' win ' , ' world ' , ' wtf ' , ' xyz ' ,
2022-05-24 19:16:55 +00:00
)
2022-05-04 23:09:46 +00:00
2022-05-24 19:16:55 +00:00
allowed_tags = ( ' b ' , ' blockquote ' , ' br ' , ' code ' , ' del ' , ' em ' , ' h1 ' , ' h2 ' , ' h3 ' , ' h4 ' , ' h5 ' , ' h6 ' , ' hr ' , ' i ' ,
' li ' , ' ol ' , ' p ' , ' pre ' , ' strong ' , ' sub ' , ' sup ' , ' table ' , ' tbody ' , ' th ' , ' thead ' , ' td ' , ' tr ' , ' ul ' ,
2022-06-19 15:05:50 +00:00
' marquee ' , ' a ' , ' span ' , ' ruby ' , ' rp ' , ' rt ' , ' spoiler ' , ' img ' , ' lite-youtube ' , ' video ' , ' source ' , ' audio ' , ' g ' )
2022-05-04 23:09:46 +00:00
2022-06-30 22:18:05 +00:00
allowed_styles = [ ' color ' , ' background-color ' , ' font-weight ' , ' text-align ' , ' filter ' , ]
2022-05-25 00:27:41 +00:00
2022-05-04 23:09:46 +00:00
def allowed_attributes ( tag , name , value ) :
if name == ' style ' : return True
if tag == ' marquee ' :
if name in [ ' direction ' , ' behavior ' , ' scrollamount ' ] : return True
if name in { ' height ' , ' width ' } :
try : value = int ( value . replace ( ' px ' , ' ' ) )
except : return False
if 0 < value < = 250 : return True
return False
2022-07-05 22:11:45 +00:00
2022-05-04 23:09:46 +00:00
if tag == ' a ' :
2022-06-19 17:25:55 +00:00
if name == ' href ' and ' \\ ' not in value and ' xn-- ' not in value :
return True
2022-05-04 23:09:46 +00:00
if name == ' rel ' and value == ' nofollow noopener noreferrer ' : return True
if name == ' target ' and value == ' _blank ' : return True
return False
if tag == ' img ' :
2022-05-25 18:29:22 +00:00
if name in [ ' src ' , ' data-src ' ] : return is_safe_url ( value )
2022-05-04 23:09:46 +00:00
if name == ' loading ' and value == ' lazy ' : return True
if name == ' data-bs-toggle ' and value == ' tooltip ' : return True
2022-06-23 00:34:37 +00:00
if name in [ ' g ' , ' b ' , ' glow ' ] and not value : return True
2022-05-18 18:45:04 +00:00
if name in [ ' alt ' , ' title ' ] : return True
2022-07-02 10:44:05 +00:00
if name == ' referrerpolicy ' and value == ' no-referrer ' : return True
2022-05-04 23:09:46 +00:00
return False
if tag == ' lite-youtube ' :
if name == ' params ' and value . startswith ( ' autoplay=1&modestbranding=1 ' ) : return True
if name == ' videoid ' : return True
return False
if tag == ' video ' :
if name == ' controls ' and value == ' ' : return True
if name == ' preload ' and value == ' none ' : return True
return False
if tag == ' source ' :
2022-05-25 18:29:22 +00:00
if name == ' src ' : return is_safe_url ( value )
2022-05-04 23:09:46 +00:00
2022-05-15 22:47:37 +00:00
if tag == ' audio ' :
2022-05-25 18:29:22 +00:00
if name == ' src ' : return is_safe_url ( value )
2022-05-15 22:47:37 +00:00
if name == ' controls ' and value == ' ' : return True
if name == ' preload ' and value == ' none ' : return True
return False
2022-05-04 23:09:46 +00:00
if tag == ' p ' :
if name == ' class ' and value == ' mb-0 ' : return True
return False
if tag == ' span ' :
if name == ' data-bs-toggle ' and value == ' tooltip ' : return True
if name == ' title ' : return True
if name == ' alt ' : return True
return False
2022-07-15 13:27:45 +00:00
def build_url_re ( tlds , protocols ) :
""" Builds the url regex used by linkifier
If you want a different set of tlds or allowed protocols , pass those in
and stomp on the existing ` ` url_re ` ` : :
from bleach import linkifier
my_url_re = linkifier . build_url_re ( my_tlds_list , my_protocols )
linker = LinkifyFilter ( url_re = my_url_re )
"""
return re . compile (
r """ \ (* # Match any opening parentheses.
\b ( ? < ! [ @ . ] ) ( ? : ( ? : { 0 } ) : / { { 0 , 3 } } ( ? : ( ? : \w + : ) ? \w + @ ) ? ) ? # http://
( [ \w - ] + \. ) + ( ? : { 1 } ) ( ? : \: [ 0 - 9 ] + ) ? ( ? ! \. \w ) \b # xx.yy.tld(:##)?
( ? : [ / ? ] [ ^ #\s\{{\}}\|\\\^\[\]`<>"]*)?
# /path/zz (excluding "unsafe" chars from RFC 1738,
# except for ~, which happens in practice)
( ? : \#[^#\s\|\\\^\[\]`<>"]*)?
# #hash (excluding "unsafe" chars from RFC 1738,
# except for ~, which happens in practice)
""" .format(
" | " . join ( sorted ( protocols ) ) , " | " . join ( sorted ( tlds ) )
) ,
re . IGNORECASE | re . VERBOSE | re . UNICODE ,
)
2022-05-04 23:09:46 +00:00
url_re = build_url_re ( tlds = TLDS , protocols = [ ' http ' , ' https ' ] )
def callback ( attrs , new = False ) :
2022-05-17 18:59:07 +00:00
if ( None , " href " ) not in attrs :
return # Incorrect <a> tag
2022-05-04 23:09:46 +00:00
href = attrs [ ( None , " href " ) ]
2022-05-17 18:59:07 +00:00
# \ in href right after / makes most browsers ditch site hostname and allows for a host injection bypassing the check, see <a href="/\google.com">cool</a>
2022-06-19 17:25:55 +00:00
if " \\ " in href or not ascii_only_regex . fullmatch ( href ) :
2022-05-17 18:59:07 +00:00
attrs [ " _text " ] = href # Laugh at this user
del attrs [ ( None , " href " ) ] # Make unclickable and reset harmful payload
return attrs
2022-05-04 23:09:46 +00:00
if not href . startswith ( ' / ' ) and not href . startswith ( f ' { SITE_FULL } / ' ) :
attrs [ ( None , " target " ) ] = " _blank "
2022-06-11 09:56:16 +00:00
attrs [ ( None , " rel " ) ] = " nofollow noopener noreferrer "
2022-05-04 23:09:46 +00:00
return attrs
2022-06-13 18:05:24 +00:00
def render_emoji ( html , regexp , edit , marseys_used , b = False ) :
2022-05-04 23:09:46 +00:00
emojis = list ( regexp . finditer ( html ) )
captured = set ( )
for i in emojis :
if i . group ( 0 ) in captured : continue
captured . add ( i . group ( 0 ) )
emoji = i . group ( 1 ) . lower ( )
attrs = ' '
if b : attrs + = ' b '
2022-06-23 00:34:37 +00:00
if not edit and len ( emojis ) < = 20 and ( ' marsey ' in emoji or emoji in marseys_const2 ) :
if random ( ) < 0.0025 : attrs + = ' g '
elif random ( ) < 0.00125 : attrs + = ' glow '
2022-05-04 23:09:46 +00:00
old = emoji
emoji = emoji . replace ( ' ! ' , ' ' ) . replace ( ' # ' , ' ' )
if emoji == ' marseyrandom ' : emoji = choice ( marseys_const2 )
emoji_partial_pat = ' <img loading= " lazy " alt= " : {0} : " src= " {1} " {2} > '
emoji_partial = ' <img loading= " lazy " data-bs-toggle= " tooltip " alt= " : {0} : " title= " : {0} : " src= " {1} " {2} > '
emoji_html = None
2022-07-08 15:39:54 +00:00
if emoji . endswith ( ' pat ' ) and emoji != ' marseyunpettablepat ' :
2022-05-04 23:09:46 +00:00
if path . isfile ( f " files/assets/images/emojis/ { emoji . replace ( ' pat ' , ' ' ) } .webp " ) :
2022-06-22 15:51:19 +00:00
emoji_html = f ' <span data-bs-toggle= " tooltip " alt= " : { old } : " title= " : { old } : " ><img src= " /i/hand.webp " > { emoji_partial_pat . format ( old , f " / e/ { emoji [ : - 3 ] } .webp " , attrs ) } </span> '
2022-05-04 23:09:46 +00:00
elif emoji . startswith ( ' @ ' ) :
if u := get_user ( emoji [ 1 : - 3 ] , graceful = True ) :
2022-06-22 15:51:19 +00:00
emoji_html = f ' <span data-bs-toggle= " tooltip " alt= " : { old } : " title= " : { old } : " ><img src= " /i/hand.webp " > { emoji_partial_pat . format ( old , f " /pp/ { u . id } " , attrs ) } </span> '
2022-05-04 23:09:46 +00:00
elif path . isfile ( f ' files/assets/images/emojis/ { emoji } .webp ' ) :
emoji_html = emoji_partial . format ( old , f ' /e/ { emoji } .webp ' , attrs )
if emoji_html :
2022-05-17 18:59:07 +00:00
marseys_used . add ( emoji )
2022-05-04 23:09:46 +00:00
html = re . sub ( f ' (?<! " ) { i . group ( 0 ) } ' , emoji_html , html )
return html
2022-07-05 22:11:45 +00:00
def with_sigalrm_timeout ( timeout : int ) :
' Use SIGALRM to raise an exception if the function executes for longer than timeout seconds '
2022-05-04 23:09:46 +00:00
2022-07-05 22:11:45 +00:00
# while trying to test this using time.sleep I discovered that gunicorn does in fact do some
# async so if we timeout on that (or on a db op) then the process is crashed without returning
# a proper 500 error. Oh well.
def sig_handler ( signum , frame ) :
print ( " Timeout! " , flush = True )
raise Exception ( " Timeout " )
2022-05-04 23:09:46 +00:00
2022-07-05 22:11:45 +00:00
def inner ( func ) :
2022-07-06 09:01:48 +00:00
@functools.wraps ( func )
2022-07-05 22:11:45 +00:00
def wrapped ( * args , * * kwargs ) :
signal . signal ( signal . SIGALRM , sig_handler )
signal . alarm ( timeout )
try :
return func ( * args , * * kwargs )
finally :
signal . alarm ( 0 )
return wrapped
return inner
@with_sigalrm_timeout ( 2 )
2022-08-21 15:05:32 +00:00
def sanitize ( sanitized , edit = False , limit_pings = 0 , showmore = True ) :
2022-06-18 15:53:34 +00:00
sanitized = sanitized . strip ( )
2022-06-23 19:43:49 +00:00
sanitized = normalize_url ( sanitized )
2022-05-27 18:28:54 +00:00
if ' ``` ' not in sanitized and ' <pre> ' not in sanitized :
2022-05-08 09:06:01 +00:00
sanitized = linefeeds_regex . sub ( r ' \ 1 \ n \ n \ 2 ' , sanitized )
2022-05-04 23:09:46 +00:00
2022-06-19 15:22:06 +00:00
sanitized = greentext_regex . sub ( r ' \ 1<g> \ > \ 2</g> ' , sanitized )
2022-06-19 15:05:50 +00:00
2022-06-11 12:21:59 +00:00
sanitized = image_regex . sub ( r ' \ 1![]( \ 2) \ 5 ' , sanitized )
2022-05-04 23:09:46 +00:00
sanitized = image_check_regex . sub ( r ' \ 1 ' , sanitized )
2022-06-25 05:28:43 +00:00
sanitized = link_fix_regex . sub ( r ' \ 1https:// \ 2 ' , sanitized )
2022-05-07 05:28:51 +00:00
2022-07-20 00:07:38 +00:00
if FEATURES [ ' MARKUP_COMMANDS ' ] :
sanitized = command_regex . sub ( command_regex_matcher , sanitized )
2022-07-11 12:14:18 +00:00
2022-05-04 23:09:46 +00:00
sanitized = markdown ( sanitized )
2022-06-28 05:52:29 +00:00
sanitized = strikethrough_regex . sub ( r ' \ 1<del> \ 2</del> ' , sanitized )
2022-05-04 23:09:46 +00:00
sanitized = sanitized . replace ( ' ' , ' ' ) . replace ( ' ' , ' ' ) . replace ( " \ufeff " , " " ) . replace ( " 𒐪 " , " " )
2022-07-12 20:29:27 +00:00
sanitized = reddit_regex . sub ( r ' \ 1<a href= " https://old.reddit.com/ \ 2 " rel= " nofollow noopener noreferrer " target= " _blank " >/ \ 2</a> ' , sanitized )
2022-06-22 22:12:47 +00:00
sanitized = sub_regex . sub ( r ' \ 1<a href= " / \ 2 " >/ \ 2</a> ' , sanitized )
2022-07-29 13:23:34 +00:00
v = getattr ( g , ' v ' , None )
2022-08-21 17:20:09 +00:00
names = set ( m . group ( 2 ) for m in mention_regex . finditer ( sanitized ) )
2022-08-21 17:22:18 +00:00
if limit_pings and len ( names ) > limit_pings and not v . admin_level : abort ( 406 )
2022-08-21 17:20:09 +00:00
users_list = get_users ( names , graceful = True )
users_dict = { }
for u in users_list :
users_dict [ u . username . lower ( ) ] = u
if u . original_username :
users_dict [ u . original_username . lower ( ) ] = u
def replacer ( m ) :
u = users_dict . get ( m . group ( 2 ) . lower ( ) )
if not u :
return m . group ( 0 )
return f ' { m . group ( 1 ) } <a href= " /id/ { u . id } " ><img loading= " lazy " src= " /pp/ { u . id } " >@ { u . username } </a> '
sanitized = mention_regex . sub ( replacer , sanitized )
2022-05-04 23:09:46 +00:00
soup = BeautifulSoup ( sanitized , ' lxml ' )
for tag in soup . find_all ( " img " ) :
if tag . get ( " src " ) and not tag [ " src " ] . startswith ( ' /pp/ ' ) :
2022-07-12 20:30:00 +00:00
if not is_safe_url ( tag [ " src " ] ) :
a = soup . new_tag ( " a " , href = tag [ " src " ] , rel = " nofollow noopener noreferrer " , target = " _blank " )
a . string = tag [ " src " ]
tag . replace_with ( a )
continue
2022-05-04 23:09:46 +00:00
tag [ " loading " ] = " lazy "
tag [ " data-src " ] = tag [ " src " ]
2022-06-22 15:51:19 +00:00
tag [ " src " ] = " /i/l.webp "
2022-05-04 23:09:46 +00:00
tag [ ' alt ' ] = f ' ![]( { tag [ " data-src " ] } ) '
2022-07-02 10:44:05 +00:00
if not is_site_url ( tag [ " data-src " ] ) :
tag [ ' referrerpolicy ' ] = " no-referrer "
2022-05-04 23:09:46 +00:00
2022-07-02 00:25:58 +00:00
if tag . parent . name != ' a ' :
2022-07-02 10:44:05 +00:00
a = soup . new_tag ( " a " , href = tag [ " data-src " ] )
if not is_site_url ( a [ " href " ] ) :
a [ " rel " ] = " nofollow noopener noreferrer "
2022-07-12 20:29:27 +00:00
a [ " target " ] = " _blank "
2022-07-02 00:25:58 +00:00
tag = tag . replace_with ( a )
a . append ( tag )
2022-06-27 01:00:45 +00:00
2022-05-04 23:09:46 +00:00
for tag in soup . find_all ( " a " ) :
2022-07-02 00:54:59 +00:00
if not tag . contents or not str ( tag . contents [ 0 ] ) . strip ( ) :
tag . extract ( )
2022-06-22 21:59:30 +00:00
if tag . get ( " href " ) and fishylinks_regex . fullmatch ( str ( tag . string ) ) :
2022-05-04 23:09:46 +00:00
tag . string = tag [ " href " ]
sanitized = str ( soup )
2022-07-05 22:11:45 +00:00
2022-05-04 23:09:46 +00:00
sanitized = spoiler_regex . sub ( r ' <spoiler> \ 1</spoiler> ' , sanitized )
2022-07-05 22:11:45 +00:00
2022-05-04 23:09:46 +00:00
marseys_used = set ( )
emojis = list ( emoji_regex . finditer ( sanitized ) )
if len ( emojis ) > 20 : edit = True
captured = [ ]
for i in emojis :
if i . group ( 0 ) in captured : continue
captured . append ( i . group ( 0 ) )
old = i . group ( 0 )
if ' marseylong1 ' in old or ' marseylong2 ' in old or ' marseyllama1 ' in old or ' marseyllama2 ' in old : new = old . lower ( ) . replace ( " > " , " class= ' mb-0 ' > " )
else : new = old . lower ( )
new = render_emoji ( new , emoji_regex2 , edit , marseys_used , True )
sanitized = sanitized . replace ( old , new )
emojis = list ( emoji_regex2 . finditer ( sanitized ) )
if len ( emojis ) > 20 : edit = True
sanitized = render_emoji ( sanitized , emoji_regex2 , edit , marseys_used )
2022-05-22 10:20:11 +00:00
sanitized = sanitized . replace ( ' & ' , ' & ' )
2022-05-04 23:09:46 +00:00
if " https://youtube.com/watch?v= " in sanitized : sanitized = sanitized . replace ( " ?t= " , " &t= " )
captured = [ ]
for i in youtube_regex . finditer ( sanitized ) :
if i . group ( 0 ) in captured : continue
captured . append ( i . group ( 0 ) )
2022-05-22 10:20:11 +00:00
params = parse_qs ( urlparse ( i . group ( 2 ) ) . query )
2022-05-04 23:09:46 +00:00
t = params . get ( ' t ' , params . get ( ' start ' , [ 0 ] ) ) [ 0 ]
if isinstance ( t , str ) : t = t . replace ( ' s ' , ' ' )
htmlsource = f ' { i . group ( 1 ) } <lite-youtube videoid= " { i . group ( 3 ) } " params= " autoplay=1&modestbranding=1 '
if t : htmlsource + = f ' &start= { t } '
htmlsource + = ' " ></lite-youtube> '
sanitized = sanitized . replace ( i . group ( 0 ) , htmlsource )
2022-08-20 20:32:28 +00:00
sanitized = video_sub_regex . sub ( r ' <video controls preload= " none " ><source src= " \ 1 " ></video> ' , sanitized )
sanitized = audio_sub_regex . sub ( r ' <audio controls preload= " none " src= " \ 1 " ></audio> ' , sanitized )
2022-05-04 23:09:46 +00:00
2022-06-13 18:05:24 +00:00
if not edit :
2022-05-04 23:09:46 +00:00
for marsey in g . db . query ( Marsey ) . filter ( Marsey . name . in_ ( marseys_used ) ) . all ( ) :
marsey . count + = 1
g . db . add ( marsey )
2022-05-15 08:45:57 +00:00
sanitized = sanitized . replace ( ' <p></p> ' , ' ' )
2022-05-04 23:09:46 +00:00
sanitized = utm_regex . sub ( ' ' , sanitized )
sanitized = utm_regex2 . sub ( ' ' , sanitized )
sanitized = sanitized . replace ( ' <html><body> ' , ' ' ) . replace ( ' </body></html> ' , ' ' )
2022-05-25 00:27:41 +00:00
css_sanitizer = CSSSanitizer ( allowed_css_properties = allowed_styles )
2022-05-04 23:09:46 +00:00
sanitized = bleach . Cleaner ( tags = allowed_tags ,
attributes = allowed_attributes ,
protocols = [ ' http ' , ' https ' ] ,
2022-05-25 00:27:41 +00:00
css_sanitizer = css_sanitizer ,
2022-07-05 22:11:45 +00:00
filters = [ partial ( LinkifyFilter , skip_tags = [ " pre " ] ,
2022-05-25 00:27:41 +00:00
parse_email = False , callbacks = [ callback ] , url_re = url_re ) ]
2022-05-04 23:09:46 +00:00
) . clean ( sanitized )
soup = BeautifulSoup ( sanitized , ' lxml ' )
links = soup . find_all ( " a " )
domain_list = set ( )
for link in links :
href = link . get ( " href " )
if not href : continue
2022-07-05 22:11:45 +00:00
2022-05-04 23:09:46 +00:00
url = urlparse ( href )
domain = url . netloc
url_path = url . path
domain_list . add ( domain + url_path )
parts = domain . split ( " . " )
for i in range ( len ( parts ) ) :
new_domain = parts [ i ]
for j in range ( i + 1 , len ( parts ) ) :
new_domain + = " . " + parts [ j ]
domain_list . add ( new_domain )
bans = g . db . query ( BannedDomain . domain ) . filter ( BannedDomain . domain . in_ ( list ( domain_list ) ) ) . all ( )
if bans : abort ( 403 , description = f " Remove the banned domains { bans } and try again! " )
2022-06-30 23:01:10 +00:00
if ' <pre> ' not in sanitized :
sanitized = sanitized . replace ( ' \n ' , ' ' )
2022-06-29 00:55:44 +00:00
2022-08-10 21:34:15 +00:00
if showmore and len ( sanitized ) > 5000 :
2022-08-04 19:31:13 +00:00
sanitized = showmore_regex . sub ( r ' \ 1<p><button class= " showmore " onclick= " showmore() " >SHOW MORE</button></p><d class= " d-none " > \ 2</d> ' , sanitized )
2022-05-04 23:09:46 +00:00
2022-07-02 10:12:52 +00:00
return sanitized . strip ( )
2022-05-04 23:09:46 +00:00
def allowed_attributes_emojis ( tag , name , value ) :
if tag == ' img ' :
2022-05-25 18:29:22 +00:00
if name == ' src ' and value . startswith ( ' / ' ) and ' \\ ' not in value : return True
2022-05-04 23:09:46 +00:00
if name == ' loading ' and value == ' lazy ' : return True
if name == ' data-bs-toggle ' and value == ' tooltip ' : return True
2022-06-23 00:34:37 +00:00
if name in [ ' g ' , ' glow ' ] and not value : return True
2022-05-18 18:45:04 +00:00
if name in [ ' alt ' , ' title ' ] : return True
2022-05-17 19:58:41 +00:00
if tag == ' span ' :
if name == ' data-bs-toggle ' and value == ' tooltip ' : return True
if name == ' title ' : return True
if name == ' alt ' : return True
return False
2022-05-04 23:09:46 +00:00
return False
2022-07-05 22:11:45 +00:00
@with_sigalrm_timeout ( 1 )
2022-05-04 23:09:46 +00:00
def filter_emojis_only ( title , edit = False , graceful = False ) :
title = title . replace ( ' ' , ' ' ) . replace ( ' ' , ' ' ) . replace ( " \ufeff " , " " ) . replace ( " 𒐪 " , " " ) . replace ( " \n " , " " ) . replace ( " \r " , " " ) . replace ( " \t " , " " ) . replace ( " & " , " & " ) . replace ( ' < ' , ' < ' ) . replace ( ' > ' , ' > ' ) . replace ( ' " ' , ' " ' ) . replace ( " ' " , " ' " ) . strip ( )
2022-06-13 18:05:24 +00:00
marseys_used = set ( )
title = render_emoji ( title , emoji_regex3 , edit , marseys_used )
if not edit :
for marsey in g . db . query ( Marsey ) . filter ( Marsey . name . in_ ( marseys_used ) ) . all ( ) :
marsey . count + = 1
g . db . add ( marsey )
2022-05-04 23:09:46 +00:00
2022-06-28 05:41:21 +00:00
title = strikethrough_regex . sub ( r ' \ 1<del> \ 2</del> ' , title )
2022-05-04 23:09:46 +00:00
2022-05-17 19:58:41 +00:00
title = bleach . clean ( title , tags = [ ' img ' , ' del ' , ' span ' ] , attributes = allowed_attributes_emojis , protocols = [ ' http ' , ' https ' ] )
2022-05-04 23:09:46 +00:00
if len ( title ) > 1500 and not graceful : abort ( 400 )
2022-07-02 10:12:52 +00:00
else : return title . replace ( ' \n ' , ' ' ) . strip ( )
2022-05-25 08:43:16 +00:00
2022-06-10 20:02:15 +00:00
def normalize_url ( url ) :
2022-07-04 03:08:33 +00:00
url = reddit_domain_regex . sub ( r ' \ 1https://old.reddit.com/ \ 3/ ' , url )
2022-06-10 20:02:15 +00:00
2022-06-23 15:47:57 +00:00
url = url . replace ( " https://youtu.be/ " , " https://youtube.com/watch?v= " ) \
2022-05-25 08:43:16 +00:00
. replace ( " https://music.youtube.com/watch?v= " , " https://youtube.com/watch?v= " ) \
2022-08-24 22:02:06 +00:00
. replace ( " https://www.youtube.com " , " https://youtube.com " ) \
2022-05-25 08:43:16 +00:00
. replace ( " https://youtube.com/shorts/ " , " https://youtube.com/watch?v= " ) \
2022-08-24 22:02:06 +00:00
. replace ( " https://youtube.com/v/ " , " https://youtube.com/watch?v= " ) \
2022-06-23 15:47:57 +00:00
. replace ( " https://mobile.twitter.com " , " https://twitter.com " ) \
. replace ( " https://m.facebook.com " , " https://facebook.com " ) \
. replace ( " https://m.wikipedia.org " , " https://wikipedia.org " ) \
. replace ( " https://m.youtube.com " , " https://youtube.com " ) \
. replace ( " https://www.twitter.com " , " https://twitter.com " ) \
. replace ( " https://www.instagram.com " , " https://instagram.com " ) \
. replace ( " https://www.tiktok.com " , " https://tiktok.com " ) \
. replace ( " https://www.streamable.com " , " https://streamable.com " ) \
2022-06-10 14:35:09 +00:00
. replace ( " https://streamable.com/ " , " https://streamable.com/e/ " ) \
2022-07-15 13:00:51 +00:00
. replace ( " https://streamable.com/e/e/ " , " https://streamable.com/e/ " ) \
2022-08-13 05:06:53 +00:00
. replace ( " https://search.marsey.cat/# " , " https://camas.unddit.com/# " ) \
. replace ( " https://imgur.com/ " , " https://i.imgur.com/ " )
2022-05-25 08:43:16 +00:00
2022-06-10 20:02:15 +00:00
url = imgur_regex . sub ( r ' \ 1_d.webp?maxwidth=9999&fidelity=high ' , url )
2022-06-11 12:21:59 +00:00
url = giphy_regex . sub ( r ' \ 1.webp ' , url )
2022-05-25 08:43:16 +00:00
2022-06-11 09:56:16 +00:00
return url
2022-08-05 17:09:41 +00:00
def validate_css ( css ) :
if ' @import ' in css :
return False , " @import statements not allowed. "
for i in css_url_regex . finditer ( css ) :
url = i . group ( 1 )
if not is_safe_url ( url ) :
domain = tldextract . extract ( url ) . registered_domain
return False , f " The domain ' { domain } ' is not allowed, please use one of these domains \n \n { approved_embed_hosts } . "
return True , " "