From 43822b3322e8605a6e7422f7212f5ca0f086a311 Mon Sep 17 00:00:00 2001 From: Aevann Date: Fri, 29 Sep 2023 10:15:29 +0300 Subject: [PATCH] add nsfw emojis --- files/assets/js/submit_emojis.js | 1 + files/classes/emoji.py | 1 + files/helpers/const_stateful.py | 11 +++-- files/helpers/sanitize.py | 46 +++++++++++++-------- files/routes/asset_submissions.py | 27 +++++++++--- files/routes/comments.py | 48 +++++++++++----------- files/routes/posts.py | 36 ++++++++-------- files/routes/static.py | 4 +- files/templates/submit_emojis.html | 14 +++++++ migrations/20230929-add-over-18-emojis.sql | 2 + 10 files changed, 121 insertions(+), 69 deletions(-) create mode 100644 migrations/20230929-add-over-18-emojis.sql diff --git a/files/assets/js/submit_emojis.js b/files/assets/js/submit_emojis.js index f22f33c46..3b648af40 100644 --- a/files/assets/js/submit_emojis.js +++ b/files/assets/js/submit_emojis.js @@ -4,6 +4,7 @@ function approve_emoji(t, name) { "tags": document.getElementById(`${name}-tags`).value, "name": document.getElementById(`${name}-name`).value, "kind": document.getElementById(`${name}-kind`).value, + "over_18": document.getElementById(`${name}-over-18`).value, }, () => { document.getElementById(`${name}-emoji`).remove() diff --git a/files/classes/emoji.py b/files/classes/emoji.py index 073be9ee4..cf3939711 100644 --- a/files/classes/emoji.py +++ b/files/classes/emoji.py @@ -15,6 +15,7 @@ class Emoji(Base): count = Column(Integer, default=0) submitter_id = Column(Integer, ForeignKey("users.id")) created_utc = Column(Integer) + over_18 = Column(Boolean, default=False) def __init__(self, *args, **kwargs): if "created_utc" not in kwargs: kwargs["created_utc"] = int(time.time()) diff --git a/files/helpers/const_stateful.py b/files/helpers/const_stateful.py index 03dd75e2e..fe6061f00 100644 --- a/files/helpers/const_stateful.py +++ b/files/helpers/const_stateful.py @@ -12,15 +12,16 @@ SNAPPY_QUOTES = [] SNAPPY_QUOTES_FISTMAS = [] SNAPPY_QUOTES_HOMOWEEN = [] STEALTH_HOLES = [] +OVER_18_EMOJIS = [] def const_initialize(): - global MARSEYS_CONST, MARSEYS_CONST2, MARSEY_MAPPINGS, SNAPPY_KONGS, SNAPPY_MARSEYS, SNAPPY_QUOTES, SNAPPY_QUOTES_FISTMAS, SNAPPY_QUOTES_HOMOWEEN, STEALTH_HOLES + global MARSEYS_CONST, MARSEYS_CONST2, MARSEY_MAPPINGS, SNAPPY_KONGS, SNAPPY_MARSEYS, SNAPPY_QUOTES, SNAPPY_QUOTES_FISTMAS, SNAPPY_QUOTES_HOMOWEEN, STEALTH_HOLES, OVER_18_EMOJIS db = db_session() - MARSEYS_CONST = [x[0] for x in db.query(Emoji.name).filter(Emoji.kind=="Marsey", Emoji.submitter_id==None, Emoji.name!='chudsey')] + MARSEYS_CONST = [x[0] for x in db.query(Emoji.name).filter(Emoji.kind == "Marsey", Emoji.submitter_id == None, Emoji.name != 'chudsey', Emoji.over_18 == False)] MARSEYS_CONST2 = MARSEYS_CONST + ['chudsey','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','0','1','2','3','4','5','6','7','8','9','exclamationpoint','period','questionmark'] - marseys = db.query(Emoji).filter(Emoji.kind=="Marsey", Emoji.submitter_id==None).all() + marseys = db.query(Emoji).filter(Emoji.kind=="Marsey", Emoji.submitter_id == None, Emoji.over_18 == False).all() for marsey in marseys: for tag in marsey.tags.split(): if tag in MARSEY_MAPPINGS: @@ -28,11 +29,13 @@ def const_initialize(): else: MARSEY_MAPPINGS[tag] = [marsey.name] - SNAPPY_KONGS = db.query(Emoji.name).filter(Emoji.kind=="Donkey Kong", Emoji.submitter_id==None).all() + SNAPPY_KONGS = db.query(Emoji.name).filter(Emoji.kind=="Donkey Kong", Emoji.submitter_id==None, Emoji.over_18 == False).all() SNAPPY_KONGS = [f':#{x[0]}:' for x in SNAPPY_KONGS] STEALTH_HOLES = [x[0] for x in db.query(Sub.name).filter_by(stealth=True)] + OVER_18_EMOJIS = [x[0] for x in db.query(Emoji.name).filter_by(over_18=True)] + db.commit() db.close() diff --git a/files/helpers/sanitize.py b/files/helpers/sanitize.py index 4c3f3ff90..f426d322b 100644 --- a/files/helpers/sanitize.py +++ b/files/helpers/sanitize.py @@ -218,58 +218,58 @@ def execute_blackjack(v, target, body, kind): send_repeatable_notification_duplicated(id, f"Blackjack by @{v.username}: {extra_info}") return True -def find_all_emote_endings(word): +def find_all_emote_endings(emoji): endings = [] - if path.isfile(f'files/assets/images/emojis/{word}.webp'): - return endings, word + if path.isfile(f'files/assets/images/emojis/{emoji}.webp'): + return endings, emoji is_non_ending_found = False while not is_non_ending_found: - if word.endswith('pat'): + if emoji.endswith('pat'): if 'pat' in endings: is_non_ending_found = True continue endings.append('pat') - word = word[:-3] + emoji = emoji[:-3] continue - if word.endswith('talking'): + if emoji.endswith('talking'): if 'talking' in endings: is_non_ending_found = True continue endings.append('talking') - word = word[:-7] + emoji = emoji[:-7] continue - if word.endswith('genocide'): + if emoji.endswith('genocide'): if 'genocide' in endings: is_non_ending_found = True continue endings.append('genocide') - word = word[:-8] + emoji = emoji[:-8] continue - if word.endswith('love'): + if emoji.endswith('love'): if 'love' in endings: is_non_ending_found = True continue endings.append('love') - word = word[:-4] + emoji = emoji[:-4] continue is_non_ending_found = True - if word.endswith('random'): - kind = word.split('random')[0].title() + if emoji.endswith('random'): + kind = emoji.split('random')[0].title() if kind == 'Donkeykong': kind = 'Donkey Kong' elif kind == 'Marseyflag': kind = 'Marsey Flags' elif kind == 'Marseyalphabet': kind = 'Marsey Alphabet' if kind in EMOJI_KINDS: - word = g.db.query(Emoji.name).filter_by(kind=kind).order_by(func.random()).first()[0] + emoji = g.db.query(Emoji.name).filter_by(kind=kind, over_18=False).order_by(func.random()).first()[0] - return endings, word + return endings, emoji def render_emoji(html, regexp, golden, emojis_used, b=False, is_title=False): @@ -411,7 +411,7 @@ def handle_youtube_links(url): return html @with_sigalrm_timeout(10) -def sanitize(sanitized, golden=True, limit_pings=0, showmore=False, count_emojis=False, snappy=False, chat=False, blackjack=None, post_mention_notif=False, commenters_ping_post_id=None): +def sanitize(sanitized, golden=True, limit_pings=0, showmore=False, count_emojis=False, snappy=False, chat=False, blackjack=None, post_mention_notif=False, commenters_ping_post_id=None, obj=None): def error(error): if chat: return error, 403 @@ -563,6 +563,12 @@ def sanitize(sanitized, golden=True, limit_pings=0, showmore=False, count_emojis emoji.count += 1 g.db.add(emoji) + if obj: + for emoji in emojis_used: + if emoji in OVER_18_EMOJIS: + obj.over_18 = True + break + sanitized = sanitized.replace('

', '') allowed_css_properties = allowed_styles.copy() @@ -704,7 +710,7 @@ def allowed_attributes_emojis(tag, name, value): @with_sigalrm_timeout(2) -def filter_emojis_only(title, golden=True, count_emojis=False): +def filter_emojis_only(title, golden=True, count_emojis=False, obj=None): title = title.replace("\n", "").replace("\r", "").replace("\t", "").replace('<','<').replace('>','>') @@ -719,6 +725,12 @@ def filter_emojis_only(title, golden=True, count_emojis=False): emoji.count += 1 g.db.add(emoji) + if obj: + for emoji in emojis_used: + if emoji in OVER_18_EMOJIS: + obj.over_18 = True + break + title = strikethrough_regex.sub(r'\1\2', title) title = bleach.clean(title, tags=['img','del','span'], attributes=allowed_attributes_emojis, protocols=['http','https']).replace('\n','') diff --git a/files/routes/asset_submissions.py b/files/routes/asset_submissions.py index 572149b7a..76a073a96 100644 --- a/files/routes/asset_submissions.py +++ b/files/routes/asset_submissions.py @@ -48,6 +48,7 @@ def submit_emoji(v): tags = request.values.get('tags', '').lower().strip() username = request.values.get('author', '').lower().strip() kind = request.values.get('kind', '').strip() + over_18 = request.values.get("over_18", False, bool) for modifier in emoji_modifiers: if name.endswith(modifier): @@ -91,7 +92,15 @@ def submit_emoji(v): copyfile(highquality, filename) process_image(filename, v, resize=300, trim=True) - emoji = Emoji(name=name, kind=kind, author_id=author.id, tags=tags, count=0, submitter_id=v.id) + emoji = Emoji( + name=name, + kind=kind, + author_id=author.id, + tags=tags, + count=0, + submitter_id=v.id, + over_18=over_18, + ) g.db.add(emoji) return {"message": f"'{name}' submitted successfully!"} @@ -139,10 +148,12 @@ def approve_emoji(v, name): if new_kind not in EMOJI_KINDS: abort(400, "Invalid kind!") + over_18 = request.values.get("over_18", False, bool) emoji.name = new_name emoji.kind = new_kind emoji.tags = tags + emoji.over_18 = over_18 g.db.add(emoji) author = get_account(emoji.author_id) @@ -177,9 +188,9 @@ def approve_emoji(v, name): badge_grant(badge_id=113, user=author) badge_grant(badge_id=112, user=author) - - cache.delete("emojis") - cache.delete(f"emoji_list_{emoji.kind}") + if not Emoji.over_18: + cache.delete("emojis") + cache.delete(f"emoji_list_{emoji.kind}") purge_files_in_cloudflare_cache(f"{SITE_FULL_IMAGES}/e/{emoji.name}/webp") @@ -210,6 +221,9 @@ def approve_emoji(v, name): ) g.db.add(ma) + if emoji.over_18: + OVER_18_EMOJIS.append(emoji.name) + return {"message": f"'{emoji.name}' approved!"} def remove_asset(cls, type_name, v, name): @@ -474,8 +488,9 @@ def update_emoji(v): ) g.db.add(ma) - cache.delete("emojis") - cache.delete(f"emoji_list_{existing.kind}") + if not Emoji.over_18: + cache.delete("emojis") + cache.delete(f"emoji_list_{existing.kind}") return {"message": f"'{name}' updated successfully!"} diff --git a/files/routes/comments.py b/files/routes/comments.py index be4011292..4092841c6 100644 --- a/files/routes/comments.py +++ b/files/routes/comments.py @@ -260,7 +260,29 @@ def comment(v): if v.marsify and not v.chud: body_for_sanitize = marsify(body_for_sanitize) if v.sharpen: body_for_sanitize = sharpen(body_for_sanitize) - body_html = sanitize(body_for_sanitize, limit_pings=5, showmore=(not v.marseyawarded), count_emojis=not v.marsify, commenters_ping_post_id=commenters_ping_post_id) + is_bot = v.client is not None and v.id not in BOT_SYMBOL_HIDDEN + + chudded = v.chud and not (posting_to_post and post_target.sub == 'chudrama') + + c = Comment(author_id=v.id, + parent_post=post_target.id if posting_to_post else None, + wall_user_id=post_target.id if not posting_to_post else None, + parent_comment_id=parent_comment_id, + level=level, + over_18=post_target.over_18 if posting_to_post else False, + is_bot=is_bot, + app_id=v.client.application.id if v.client else None, + body=body, + ghost=ghost, + chudded=chudded, + rainbowed=bool(v.rainbow), + queened=bool(v.queen), + sharpened=bool(v.sharpen), + ) + + c.upvotes = 1 + + body_html = sanitize(body_for_sanitize, limit_pings=5, showmore=(not v.marseyawarded), count_emojis=not v.marsify, commenters_ping_post_id=commenters_ping_post_id, obj=c) if post_target.id not in ADMIGGER_THREADS and not (v.chud and v.chud_phrase in body.lower()): existing = g.db.query(Comment.id).filter( @@ -282,28 +304,8 @@ def comment(v): if len(body_html) > COMMENT_BODY_HTML_LENGTH_LIMIT: abort(400, "Comment too long!") - is_bot = v.client is not None and v.id not in BOT_SYMBOL_HIDDEN + c.body_html = body_html - chudded = v.chud and not (posting_to_post and post_target.sub == 'chudrama') - - c = Comment(author_id=v.id, - parent_post=post_target.id if posting_to_post else None, - wall_user_id=post_target.id if not posting_to_post else None, - parent_comment_id=parent_comment_id, - level=level, - over_18=post_target.over_18 if posting_to_post else False, - is_bot=is_bot, - app_id=v.client.application.id if v.client else None, - body_html=body_html, - body=body, - ghost=ghost, - chudded=chudded, - rainbowed=bool(v.rainbow), - queened=bool(v.queen), - sharpened=bool(v.sharpen), - ) - - c.upvotes = 1 g.db.add(c) g.db.flush() @@ -683,7 +685,7 @@ def edit_comment(cid, v): if c.sharpened: body_for_sanitize = sharpen(body_for_sanitize) - body_html = sanitize(body_for_sanitize, golden=False, limit_pings=5, showmore=(not v.marseyawarded), commenters_ping_post_id=c.parent_post) + body_html = sanitize(body_for_sanitize, golden=False, limit_pings=5, showmore=(not v.marseyawarded), commenters_ping_post_id=c.parent_post, obj=c) if len(body_html) > COMMENT_BODY_HTML_LENGTH_LIMIT: abort(400) diff --git a/files/routes/posts.py b/files/routes/posts.py index 34301dde6..1ec52524c 100644 --- a/files/routes/posts.py +++ b/files/routes/posts.py @@ -457,11 +457,6 @@ def submit_post(v, sub=None): if SITE == 'rdrama.net' and v.id == 10947: sub = 'mnn' - title_html = filter_emojis_only(title, count_emojis=True) - - if v.marseyawarded and not marseyaward_title_regex.fullmatch(title_html): - abort(400, "You can only type marseys!") - if sub == 'changelog': abort(400, "/h/changelog is archived") @@ -541,14 +536,6 @@ def submit_post(v, sub=None): body_for_sanitize = body if v.sharpen: body_for_sanitize = sharpen(body_for_sanitize) - body_html = sanitize(body_for_sanitize, count_emojis=True, limit_pings=100) - - if v.marseyawarded and marseyaward_body_regex.search(body_html): - abort(400, "You can only type marseys!") - - if len(body_html) > POST_BODY_HTML_LENGTH_LIMIT: - abort(400, "Post body_html too long!") - flag_notify = (request.values.get("notify", "on") == "on") flag_new = request.values.get("new", False, bool) or 'megathread' in title.lower() flag_over_18 = FEATURES['NSFW_MARKING'] and request.values.get("over_18", False, bool) @@ -577,10 +564,8 @@ def submit_post(v, sub=None): is_bot=(v.client is not None), url=url, body=body, - body_html=body_html, embed=embed, title=title, - title_html=title_html, sub=sub, ghost=flag_ghost, chudded=flag_chudded, @@ -589,6 +574,23 @@ def submit_post(v, sub=None): sharpened=bool(v.sharpen), ) + title_html = filter_emojis_only(title, count_emojis=True, obj=p) + + if v.marseyawarded and not marseyaward_title_regex.fullmatch(title_html): + abort(400, "You can only type marseys!") + + p.title_html = title_html + + body_html = sanitize(body_for_sanitize, count_emojis=True, limit_pings=100, obj=p) + + if v.marseyawarded and marseyaward_body_regex.search(body_html): + abort(400, "You can only type marseys!") + + if len(body_html) > POST_BODY_HTML_LENGTH_LIMIT: + abort(400, "Post body_html too long!") + + p.body_html = body_html + g.db.add(p) g.db.flush() @@ -1015,7 +1017,7 @@ def edit_post(pid, v): if title != p.title: - title_html = filter_emojis_only(title, golden=False) + title_html = filter_emojis_only(title, golden=False, obj=p) if p.author.marseyawarded and not marseyaward_title_regex.fullmatch(title_html): abort(403, "You can only type marseys!") @@ -1033,7 +1035,7 @@ def edit_post(pid, v): body_for_sanitize = body if p.sharpened: body_for_sanitize = sharpen(body_for_sanitize) - body_html = sanitize(body_for_sanitize, golden=False, limit_pings=100) + body_html = sanitize(body_for_sanitize, golden=False, limit_pings=100, obj=p) if p.author.marseyawarded and marseyaward_body_regex.search(body_html): abort(403, "You can only type marseys!") diff --git a/files/routes/static.py b/files/routes/static.py index 98fa1e3f6..de5666019 100644 --- a/files/routes/static.py +++ b/files/routes/static.py @@ -45,7 +45,7 @@ def reddit_post(subreddit, v, path): @cache.cached(make_cache_key=lambda kind:f"emoji_list_{kind}") def get_emoji_list(kind): emojis = [] - for emoji, author in g.db.query(Emoji, User).join(User, Emoji.author_id == User.id).filter(Emoji.submitter_id == None, Emoji.kind == kind).order_by(Emoji.count.desc()): + for emoji, author in g.db.query(Emoji, User).join(User, Emoji.author_id == User.id).filter(Emoji.submitter_id == None, Emoji.kind == kind, Emoji.over_18 == False).order_by(Emoji.count.desc()): emoji.author = author.username if FEATURES['ASSET_SUBMISSIONS'] else None emojis.append(emoji) return emojis @@ -83,7 +83,7 @@ def emoji_list(v, kind): @cache.cached(make_cache_key=lambda:"emojis") def get_emojis(): - emojis = g.db.query(Emoji, User).join(User, Emoji.author_id == User.id).filter(Emoji.submitter_id == None) + emojis = g.db.query(Emoji, User).join(User, Emoji.author_id == User.id).filter(Emoji.submitter_id == None, Emoji.over_18 == False) emojis1 = emojis.filter(Emoji.kind != 'Marsey Alphabet').order_by(Emoji.count.desc()).all() emojis2 = emojis.filter(Emoji.kind == 'Marsey Alphabet').order_by(func.length(Emoji.name), Emoji.name).all() emojis = emojis1 + emojis2 diff --git a/files/templates/submit_emojis.html b/files/templates/submit_emojis.html index de0549b38..b348d3efa 100644 --- a/files/templates/submit_emojis.html +++ b/files/templates/submit_emojis.html @@ -43,6 +43,13 @@ + {% if FEATURES['NSFW_MARKING'] %} +
+ + +
+ {% endif %} +
@@ -91,6 +98,13 @@ + + {% if FEATURES['NSFW_MARKING'] %} +
+ + +
+ {% endif %}
{% if v.admin_level >= PERMS['MODERATE_PENDING_SUBMITTED_ASSETS'] or v.id == emoji.submitter_id %} diff --git a/migrations/20230929-add-over-18-emojis.sql b/migrations/20230929-add-over-18-emojis.sql new file mode 100644 index 000000000..eecc07b5a --- /dev/null +++ b/migrations/20230929-add-over-18-emojis.sql @@ -0,0 +1,2 @@ +alter table emojis add column over_18 bool default false not null; +alter table emojis alter column over_18 drop default;