From dee8eb515491ac2adc654b0cafcd77f160bb585d Mon Sep 17 00:00:00 2001 From: TLSM Date: Sun, 19 Jun 2022 22:15:33 -0400 Subject: [PATCH 01/17] Replace loading.webp with new marseyloading. --- files/classes/user.py | 4 +++- files/helpers/sanitize.py | 2 +- files/templates/header.html | 2 ++ files/templates/submission_listing.html | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/files/classes/user.py b/files/classes/user.py index 23f6a33a40..aaa750e188 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -345,7 +345,9 @@ class User(Base): @lazy def bio_html_eager(self): if self.bio_html == None: return '' - return self.bio_html.replace('data-src', 'src').replace('src="/assets/images/loading.webp"', '') + return self.bio_html.replace('data-src', 'src') \ + .replace('src="/assets/images/loading.webp?v=2"', '') \ + .replace('src="/assets/images/loading.webp"', '') @property @lazy diff --git a/files/helpers/sanitize.py b/files/helpers/sanitize.py index aa7f3ee06b..e0c30f8e0d 100644 --- a/files/helpers/sanitize.py +++ b/files/helpers/sanitize.py @@ -221,7 +221,7 @@ def sanitize(sanitized, alert=False, edit=False): if tag.get("src") and not tag["src"].startswith('/pp/'): tag["loading"] = "lazy" tag["data-src"] = tag["src"] - tag["src"] = "/assets/images/loading.webp" + tag["src"] = "/assets/images/loading.webp?v=2" tag['alt'] = f'![]({tag["data-src"]})' tag['referrerpolicy'] = "no-referrer" diff --git a/files/templates/header.html b/files/templates/header.html index d7eeef1613..8c18145b56 100644 --- a/files/templates/header.html +++ b/files/templates/header.html @@ -308,3 +308,5 @@ } {%- endif %} + + diff --git a/files/templates/submission_listing.html b/files/templates/submission_listing.html index d5cf76b37d..d0584e0c26 100644 --- a/files/templates/submission_listing.html +++ b/files/templates/submission_listing.html @@ -347,7 +347,7 @@ {% if p.is_image and not p.over_18 and ((v and v.cardview) or (not v and environ.get('CARD_VIEW') == '1')) %}
- Unable to load image + Unable to load image
{% elif p.is_video %} From e212eb567d276321334417d0f707f4ef5ee6e90d Mon Sep 17 00:00:00 2001 From: TLSM Date: Mon, 20 Jun 2022 07:39:45 -0400 Subject: [PATCH 03/17] Exclude comments on drafts from edit limit. Draft posts have already been excluded from the edit time limit for obvious reasons--drafts are intended to be edited, and people use them as personal megathreads on their profiles. Largely for the latter use case, this commit also excludes comments on drafts from the limit. --- files/routes/comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/routes/comments.py b/files/routes/comments.py index 4edf58eafb..64dfd0bb87 100644 --- a/files/routes/comments.py +++ b/files/routes/comments.py @@ -654,7 +654,7 @@ def edit_comment(cid, v): c = get_comment(cid, v=v) - if time.time() - c.created_utc > 7*24*60*60: + if time.time() - c.created_utc > 7*24*60*60 and not (c.post and c.post.private): return {"error":"You can't edit comments older than 1 week!"}, 403 if c.author_id != v.id: abort(403) From 35720c0a63dd37e1d2035da72543b0721217c1fb Mon Sep 17 00:00:00 2001 From: TLSM Date: Mon, 20 Jun 2022 11:58:58 -0400 Subject: [PATCH 05/17] Include holed post edit in mobile admin actions. --- files/templates/post_admin_actions_mobile.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/templates/post_admin_actions_mobile.html b/files/templates/post_admin_actions_mobile.html index fa7d5ad79c..9f363773f0 100644 --- a/files/templates/post_admin_actions_mobile.html +++ b/files/templates/post_admin_actions_mobile.html @@ -9,7 +9,7 @@
    - {% if request.path.startswith('/post/') and v.admin_level > 2 %} + {% if (request.path.startswith('/post/') or request.path.startswith('/h/')) and v.admin_level > 2 and p.id %} {% endif %} From f6b6997f6136d7b700acb7394771c1ef889ecaa7 Mon Sep 17 00:00:00 2001 From: TLSM Date: Mon, 20 Jun 2022 16:25:03 -0400 Subject: [PATCH 06/17] Persist activity timestamp to users. --- files/classes/user.py | 14 ++++++++++++++ files/helpers/const.py | 1 + files/helpers/wrappers.py | 10 ++++++++-- files/routes/admin.py | 4 ++-- files/templates/userpage.html | 8 ++++++++ schema.sql | 3 ++- 6 files changed, 35 insertions(+), 5 deletions(-) diff --git a/files/classes/user.py b/files/classes/user.py index aaa750e188..f7997ef1d7 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -70,6 +70,7 @@ class User(Base): received_award_count = Column(Integer, default=0) created_utc = Column(Integer) admin_level = Column(Integer, default=0) + last_active = Column(Integer, default=0, nullable=False) coins_spent = Column(Integer, default=0) lootboxes_bought = Column(Integer, default=0) agendaposter = Column(Integer, default=0) @@ -198,6 +199,12 @@ class User(Base): return time.strftime("%d %b %Y", time.gmtime(self.created_utc)) + @property + @lazy + def last_active_date(self): + if self.last_active == 0: + return "never" + return str(time.strftime("%d %b %Y", time.gmtime(self.last_active))) @property @lazy @@ -673,6 +680,13 @@ class User(Base): def created_datetime(self): return str(time.strftime("%d/%B/%Y %H:%M:%S UTC", time.gmtime(self.created_utc))) + @property + @lazy + def last_active_datetime(self): + if self.last_active == 0: + return "never" + return str(time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime(self.last_active))) + @lazy def subscribed_idlist(self, page=1): posts = g.db.query(Subscription.submission_id).filter_by(user_id=self.id).all() diff --git a/files/helpers/const.py b/files/helpers/const.py index 2a174f0935..12f2503561 100644 --- a/files/helpers/const.py +++ b/files/helpers/const.py @@ -143,6 +143,7 @@ HOLE_COST = 0 HOLE_INACTIVITY_DELETION = False PIN_LIMIT = 3 POST_RATE_LIMIT = '1/second;2/minute;10/hour;50/day' +LOGGEDIN_ACTIVE_TIME = 15 * 60 NOTIFICATIONS_ID = 1 AUTOJANNY_ID = 2 diff --git a/files/helpers/wrappers.py b/files/helpers/wrappers.py index 3c750939d2..600ff69b0a 100644 --- a/files/helpers/wrappers.py +++ b/files/helpers/wrappers.py @@ -53,15 +53,21 @@ def get_logged_in_user(): if v: if session["session_id"] in loggedout: del loggedout[session["session_id"]] loggedin[v.id] = timestamp + # Check against last_active + ACTIVE_TIME to reduce frequency of + # UPDATEs in exchange for a ±ACTIVE_TIME margin of error. + if (v.last_active + LOGGEDIN_ACTIVE_TIME) < timestamp: + v.last_active = timestamp + g.db.add(v) + g.db.commit() else: ua = str(user_agents.parse(g.agent)) if not ua.startswith('Spider') and 'bot' not in ua.lower(): loggedout[session["session_id"]] = (timestamp, ua) - g.loggedin_counter = len([x for x in loggedin.values() if timestamp-x<15*60]) + g.loggedin_counter = len([x for x in loggedin.values() if timestamp-x < LOGGEDIN_ACTIVE_TIME]) cache.set(f'{SITE}_loggedin', loggedin) - g.loggedout_counter = len([x for x in loggedout.values() if timestamp-x[0]<15*60]) + g.loggedout_counter = len([x for x in loggedout.values() if timestamp-x[0] < LOGGEDIN_ACTIVE_TIME]) cache.set(f'{SITE}_loggedout', loggedout) g.v = v diff --git a/files/routes/admin.py b/files/routes/admin.py index 929d0d594e..4a592676bc 100644 --- a/files/routes/admin.py +++ b/files/routes/admin.py @@ -63,14 +63,14 @@ def kippy(v): @app.get('/admin/loggedin') @admin_level_required(2) def loggedin_list(v): - ids = [x for x,val in cache.get(f'{SITE}_loggedin').items() if time.time()-val<15*60] + ids = [x for x,val in cache.get(f'{SITE}_loggedin').items() if time.time()-val < LOGGEDIN_ACTIVE_TIME] users = g.db.query(User).filter(User.id.in_(ids)).order_by(User.admin_level.desc(), User.truecoins.desc()).all() return render_template("loggedin.html", v=v, users=users) @app.get('/admin/loggedout') @admin_level_required(2) def loggedout_list(v): - users = sorted([val[1] for x,val in cache.get(f'{SITE}_loggedout').items() if time.time()-val[0]<15*60]) + users = sorted([val[1] for x,val in cache.get(f'{SITE}_loggedout').items() if time.time()-val[0] < LOGGEDIN_ACTIVE_TIME]) return render_template("loggedout.html", v=v, users=users) @app.get('/admin/merge//') diff --git a/files/templates/userpage.html b/files/templates/userpage.html index c1870e4ef2..60a8923a9a 100644 --- a/files/templates/userpage.html +++ b/files/templates/userpage.html @@ -125,6 +125,10 @@ follows {{u.follow_count}} user{{'s' if u.follow_count != 1 else ''}}   joined {{u.created_date}} + + {% if v and v.admin_level >= 2 -%} + last active {{u.last_active_date}} + {%- endif %}
{% if u.basedcount %}

Based Count: {{u.basedcount}}

{% endif %} @@ -427,6 +431,10 @@ {% endif %}
joined {{u.created_date}} + + {% if v and v.admin_level >= 2 -%} +
last active {{u.last_active_date}} + {%- endif %} {% if u.bio_html %}
{{u.bio_html | safe}}
diff --git a/schema.sql b/schema.sql index 748a1edaa8..0f0dc8c513 100644 --- a/schema.sql +++ b/schema.sql @@ -679,7 +679,8 @@ CREATE TABLE public.users ( total_held_lottery_tickets integer DEFAULT 0 NOT NULL, total_lottery_winnings integer DEFAULT 0 NOT NULL, can_gamble boolean DEFAULT true NOT NULL, - offsitementions boolean DEFAULT false NOT NULL + offsitementions boolean DEFAULT false NOT NULL, + last_active integer DEFAULT 0 NOT NULL ); From f59556cac3cebada1d5740369e515b7368ad9ef4 Mon Sep 17 00:00:00 2001 From: TLSM Date: Mon, 20 Jun 2022 16:33:47 -0400 Subject: [PATCH 07/17] stats: add WAU based on activity timestamp. --- files/helpers/stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/files/helpers/stats.py b/files/helpers/stats.py index 9000ce336e..f642e71ced 100644 --- a/files/helpers/stats.py +++ b/files/helpers/stats.py @@ -128,6 +128,7 @@ def stats(site=None): "total awards": g.db.query(AwardRelationship).count(), "awards given": g.db.query(AwardRelationship).filter(or_(AwardRelationship.submission_id != None, AwardRelationship.comment_id != None)).count(), "users who posted, commented, or voted in the past 7 days": len(active_users), + "users online in the past 7 days": g.db.query(User).filter(User.last_active > week).count(), } if site == 'rDrama': From 3c5e2c945537f4d8c968b6a53066e7bb20d7c8ee Mon Sep 17 00:00:00 2001 From: Marco Rebhan Date: Mon, 20 Jun 2022 23:01:42 +0200 Subject: [PATCH 10/17] Include message text in off-site mentions --- files/helpers/offsitementions.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/files/helpers/offsitementions.py b/files/helpers/offsitementions.py index 445fb31464..4e45f71bef 100644 --- a/files/helpers/offsitementions.py +++ b/files/helpers/offsitementions.py @@ -6,6 +6,7 @@ import files.helpers.const as const from files.classes.user import User from files.classes.comment import Comment from files.classes.notifications import Notification +from files.helpers.sanitize import sanitize # https://api.pushshift.io/meta provides key server_ratelimit_per_minute # At time of writing, the ratelimit is 120 req/min. We get nowhere near this @@ -38,23 +39,30 @@ def get_mentions(queries): + f'?html_decode=true&q={query}&size=1', timeout=5).json()['data'] except: break - for i in data: + for i in data: # Special case: PokemonGoRaids says 'Marsey' a lot unrelated to us. if i['subreddit'] == 'PokemonGoRaids': continue - mentions.append(i['permalink']) + mentions.append({ + 'permalink': i['permalink'], + 'text': i['body' if kind == 'comment' else 'title'], + }) return mentions def notify_mentions(send_to, mentions, mention_str='site mention'): for m in mentions: - notif_text = f'

New {mention_str}: ' \ - f'https://old.reddit.com{m}?context=89

' + f'https://old.reddit.com{permalink}?context=89

' \ + f'
{text}
' existing_comment = g.db.query(Comment.id).filter_by( - author_id=const.NOTIFICATIONS_ID, - parent_submission=None, + author_id=const.NOTIFICATIONS_ID, + parent_submission=None, body_html=notif_text).one_or_none() if existing_comment: continue From 6c00330042b57f3024fb86da4d8e4a19e5a5b053 Mon Sep 17 00:00:00 2001 From: TLSM Date: Mon, 20 Jun 2022 20:45:05 -0400 Subject: [PATCH 12/17] Fix markdown ordered lists breaking after index. Previously, Markdown ordered lists in user content (in posts, comments, previews, etc) would display like this: 1. Foo bar baz. This is because sanitize populates them as

  • Foo bar baz.

    • Rather than mess with the Markdown engine and still not have backwards compatibility, this has been solved in the frontend using CSS to force the

    to display inline. --- files/templates/util/assetcache.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/templates/util/assetcache.html b/files/templates/util/assetcache.html index fd7662b334..13f8006c29 100644 --- a/files/templates/util/assetcache.html +++ b/files/templates/util/assetcache.html @@ -1,6 +1,6 @@ {%- set CACHE_VER = { - 'css/main.css': 340, + 'css/main.css': 341, 'css/4chan.css': 61, 'css/classic.css': 61, From 4166b2d2f00b31d01c24164fe3725201562b2c8f Mon Sep 17 00:00:00 2001 From: TLSM Date: Tue, 21 Jun 2022 01:03:33 -0400 Subject: [PATCH 14/17] Content: chadsoy x2, more neo-gTLDs. --- files/helpers/sanitize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/files/helpers/sanitize.py b/files/helpers/sanitize.py index e0c30f8e0d..bb64653b11 100644 --- a/files/helpers/sanitize.py +++ b/files/helpers/sanitize.py @@ -32,7 +32,8 @@ TLDS = ( # Original gTLDs and ccTLDs 'vu','wf','ws','xn','xxx','ye','yt','yu','za','zm','zw', # New gTLDs 'app','cleaning','club','dev','florist','fun','gay','lgbt','life','lol', - 'moe','mom','monster','pics','press','pub','win','wtf','xyz', + 'moe','mom','monster','new','news','online','pics','press','pub','site', + 'vip','win','wtf','xyz', ) allowed_tags = ('b','blockquote','br','code','del','em','h1','h2','h3','h4','h5','h6','hr','i', From e2cd25db671e276101a73c83d3c23bd14f0aef21 Mon Sep 17 00:00:00 2001 From: TLSM Date: Tue, 21 Jun 2022 01:20:21 -0400 Subject: [PATCH 15/17] WPD: limit hole creation to JL2+. --- files/classes/user.py | 5 +++++ files/helpers/const.py | 3 ++- files/routes/subs.py | 8 ++++++-- files/templates/sidebar_Cringetopia.html | 8 ++++++-- files/templates/sidebar_PCM.html | 4 +++- files/templates/sidebar_WPD.html | 4 +++- files/templates/sidebar_rDrama.html | 8 +++++--- 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/files/classes/user.py b/files/classes/user.py index f7997ef1d7..a6b123c067 100644 --- a/files/classes/user.py +++ b/files/classes/user.py @@ -740,3 +740,8 @@ class User(Base): @lazy def lottery_stats(self): return { "winnings": self.total_lottery_winnings, "ticketsHeld": { "current": self.currently_held_lottery_tickets , "total": self.total_held_lottery_tickets } } + + @property + @lazy + def can_create_hole(self): + return self.admin_level >= HOLE_CREATE_JL_MIN diff --git a/files/helpers/const.py b/files/helpers/const.py index 12f2503561..efb2d531df 100644 --- a/files/helpers/const.py +++ b/files/helpers/const.py @@ -140,6 +140,7 @@ AGENDAPOSTER_MSG_HTML = """

    Hi {{sub.sidebar_html|safe}} {% endif %} {% if v %} - CREATE HOLE + {% if v.can_create_hole -%} + CREATE HOLE + {%- endif %} {% if v.mods(sub.name) %} HOLE SETTINGS {% endif %} @@ -29,7 +31,9 @@ HOLE FOLLOWERS HOLE BLOCKERS {% else %} - CREATE HOLE + {% if v and v.can_create_hole -%} + CREATE HOLE + {%- endif %} BROWSE HOLES

    diff --git a/files/templates/sidebar_PCM.html b/files/templates/sidebar_PCM.html index efb736c4f4..819a25107b 100644 --- a/files/templates/sidebar_PCM.html +++ b/files/templates/sidebar_PCM.html @@ -19,7 +19,9 @@ HOLE BLOCKERS {% endif %} - CREATE HOLE + {% if v and v.can_create_hole -%} + CREATE HOLE + {%- endif %} BROWSE HOLES STREAM LIST BUGS/SUGGESTIONS MEGATHREAD diff --git a/files/templates/sidebar_WPD.html b/files/templates/sidebar_WPD.html index 56b4e6b34f..9a9fb4f11e 100644 --- a/files/templates/sidebar_WPD.html +++ b/files/templates/sidebar_WPD.html @@ -19,7 +19,9 @@
    {{sub.sidebar_html|safe}}
    {% endif %} {% if v %} - CREATE HOLE + {% if v.can_create_hole -%} + CREATE HOLE + {%- endif %} {% if v.mods(sub.name) %} HOLE SETTINGS {% endif %} diff --git a/files/templates/sidebar_rDrama.html b/files/templates/sidebar_rDrama.html index 2275da22bd..df68ac4d71 100644 --- a/files/templates/sidebar_rDrama.html +++ b/files/templates/sidebar_rDrama.html @@ -63,7 +63,9 @@ set VISITORS_HERE_FLAVOR = [
    {{sub.sidebar_html|safe}}
    {% endif %} {% if v %} - CREATE HOLE + {% if v.can_create_hole -%} + CREATE HOLE + {%- endif %} {% if v.mods(sub.name) %} HOLE SETTINGS {% endif %} @@ -78,9 +80,9 @@ set VISITORS_HERE_FLAVOR = [ Submit Marseys & Art | Info Megathreads BROWSE HOLES - {% if v %} + {% if v and v.can_create_hole -%} CREATE HOLE - {% endif %} + {%- endif %}

    Rules:


    From f7a59f46724075a19507cc941c5e792451c94706 Mon Sep 17 00:00:00 2001 From: TLSM Date: Tue, 21 Jun 2022 01:31:31 -0400 Subject: [PATCH 16/17] Make search operators case insensitive. In light of the fact that all searching against the database is done using ILIKE pattern matching, the only truly case-sensitive part of the search query was search operator keys. Rather than lowercase the keys in `criteria` before returning, we instead lowercase the entire search string at the beginning of parsing. This will further enforce case-insensitivity on the design of search going forward. --- files/routes/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/routes/search.py b/files/routes/search.py index 097f6a4350..2591ce4db1 100644 --- a/files/routes/search.py +++ b/files/routes/search.py @@ -12,7 +12,7 @@ valid_params=[ ] def searchparse(text): - + text = text.lower() criteria = {x[0]:x[1] for x in query_regex.findall(text)} From 5609a96e05f782550c9b182fbdba433b18abae22 Mon Sep 17 00:00:00 2001 From: TLSM Date: Tue, 21 Jun 2022 02:38:18 -0400 Subject: [PATCH 17/17] Fix info leak of removed & deleted via post embeds. Embedded local posts (posts which link to posts on the same site) embed the linked post using submission_listing.html via helpers/jinja2:post_embed. This suffered from much the same issue recently fixed in submission.html through the addition of `v_forbid_deleted` in the template before outputting privileged information. A similar fix has been applied to submission_listing. Unfortunately, this is not the most elegant fix. Surely this would be better resolved more centrally in the submission model. However, I am not clear at present about the precise interaction between deletion, removal, and realbody & realurl in all of the different places they are used. This commit fixes the problem, but it also highlights a potential future refactoring target. --- files/templates/submission_listing.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/files/templates/submission_listing.html b/files/templates/submission_listing.html index d0584e0c26..c36db1001c 100644 --- a/files/templates/submission_listing.html +++ b/files/templates/submission_listing.html @@ -65,6 +65,8 @@ {% set voted=-2 %} {% endif %} +{% set v_forbid_deleted = (p.deleted_utc != 0 or p.is_banned) and not (v and v.admin_level >= 2) and not (v and v.id == p.author_id) %} + {% if p.active_flags(v) %}
    Reported by: @@ -112,6 +114,7 @@
    + {% if not v_forbid_deleted %}
    {% if p.club and not (v and (v.paid_dues or v.id == p.author_id)) %} post thumnail @@ -135,6 +138,7 @@ {% endif %}
    + {% endif %}
    @@ -337,7 +341,7 @@
    -{% if not p.club or v and (v.paid_dues or v.id == p.author_id) %} +{% if (not p.club or v and (v.paid_dues or v.id == p.author_id)) and not v_forbid_deleted %} {% if p.realbody(v) %}
    {{p.realbody(v) | safe}}