From a57ff00e09432f0c881c858d6ce4e7ec4e51f30e Mon Sep 17 00:00:00 2001 From: Aevann Date: Wed, 26 Jul 2023 00:26:34 +0300 Subject: [PATCH] make the site faster by not having GET endpoints waiting for UPDATE statements to finish (they can take really long bc of locks) --- files/helpers/cron.py | 17 +++++++++++++++ files/routes/comments.py | 19 +++++++++++----- files/routes/front.py | 5 ----- files/routes/posts.py | 22 +++++++++++-------- files/routes/users.py | 47 +++++++++++++++------------------------- files/routes/wrappers.py | 4 +--- 6 files changed, 63 insertions(+), 51 deletions(-) diff --git a/files/helpers/cron.py b/files/helpers/cron.py index 31469f513f..59f6a8c878 100644 --- a/files/helpers/cron.py +++ b/files/helpers/cron.py @@ -38,6 +38,7 @@ def cron_fn(every_5m, every_1d): spin_roulette_wheel() #offsitementions.offsite_mentions_task(cache) _award_timers_task() + _unpin_expired() if every_1d: stats.generate_charts_task(SITE) @@ -234,3 +235,19 @@ def _award_timers_task(): User.chudded_by: None, }) _process_timer(User.flairchanged, [96], "Your temporary flair-lock has expired. You can now change your flair!") + + +def _unpin_expired(): + t = int(time.time()) + pins = [] + + for cls in (Post, Comment): + pins += g.db.query(cls).options(load_only(cls.id)).filter(cls.stickied_utc < t) + + for pin in pins: + pin.stickied = None + pin.stickied_utc = None + g.db.add(pin) + + if pins: + cache.delete_memoized(frontlist) diff --git a/files/routes/comments.py b/files/routes/comments.py index 1f152db020..1e936b8780 100644 --- a/files/routes/comments.py +++ b/files/routes/comments.py @@ -23,6 +23,19 @@ from files.routes.routehelpers import execute_shadowban_viewers_and_voters from files.routes.wrappers import * from files.__main__ import app, cache, limiter +def _mark_comment_as_read(cid, vid): + db = db_session() + + notif = db.query(Notification).filter_by(comment_id=cid, user_id=vid, read=False).one_or_none() + + if notif: + notif.read = True + db.add(notif) + + db.commit() + db.close() + stdout.flush() + @app.get("/comment/") @app.get("/post///") @app.get("/h//comment/") @@ -35,11 +48,7 @@ def post_pid_comment_cid(cid, v, pid=None, anything=None, sub=None): if not User.can_see(v, comment): abort(403) if v and request.values.get("read"): - notif = g.db.query(Notification).filter_by(comment_id=cid, user_id=v.id, read=False).one_or_none() - if notif: - notif.read = True - g.db.add(notif) - g.db.flush() + gevent.spawn(_mark_comment_as_read, comment.id, v.id) if comment.parent_post: post = comment.parent_post diff --git a/files/routes/front.py b/files/routes/front.py index 83f9f37a32..b1acb1a1c1 100644 --- a/files/routes/front.py +++ b/files/routes/front.py @@ -153,11 +153,6 @@ def frontlist(v=None, sort="hot", page=1, t="all", ids_only=True, filter_words=' if v: pins = pins.filter(or_(Post.sub == None, Post.sub.notin_(v.sub_blocks))) - for pin in pins: - if pin.stickied_utc and int(time.time()) > pin.stickied_utc: - pin.stickied = None - pin.stickied_utc = None - g.db.add(pin) if v: pins = pins.filter(Post.author_id.notin_(v.userblocks)) diff --git a/files/routes/posts.py b/files/routes/posts.py index 67ed722210..7da11493d5 100644 --- a/files/routes/posts.py +++ b/files/routes/posts.py @@ -30,6 +30,17 @@ from .users import userpagelisting from files.__main__ import app, limiter, redis_instance +def _add_post_view(pid): + db = db_session() + + p = db.get(Post, pid) + p.views += 1 + db.add(p) + + db.commit() + db.close() + stdout.flush() + @app.post("/publish/") @limiter.limit('1/second', scope=rpath) @limiter.limit('1/second', scope=rpath, key_func=get_ID) @@ -90,10 +101,7 @@ def post_id(pid, v, anything=None, sub=None): if g.is_api_or_xhr: abort(451, "Must be 18+ to view") return render_template("errors/nsfw.html", v=v) - p.views += 1 - g.db.add(p) - try: g.db.flush() - except: g.db.rollback() + gevent.spawn(_add_post_view, pid) if p.new: defaultsortingcomments = 'new' elif v: defaultsortingcomments = v.defaultsortingcomments @@ -163,11 +171,7 @@ def post_id(pid, v, anything=None, sub=None): if sort == "hot": pinned2 = {} for pin in pinned: - if pin.stickied_utc and int(time.time()) > pin.stickied_utc: - pin.stickied = None - pin.stickied_utc = None - g.db.add(pin) - elif pin.level > 1: + if pin.level > 1: pinned2[pin.top_comment] = '' if pin.top_comment in comments: comments.remove(pin.top_comment) diff --git a/files/routes/users.py b/files/routes/users.py index 8fb5ded215..362eb8cb54 100644 --- a/files/routes/users.py +++ b/files/routes/users.py @@ -22,9 +22,22 @@ from files.helpers.sorting_and_time import * from files.helpers.useractions import badge_grant from files.routes.routehelpers import check_for_alts, add_alt from files.routes.wrappers import * +from files.routes.comments import _mark_comment_as_read from files.__main__ import app, cache, limiter +def _add_profile_view(vid, uid): + db = db_session() + + view = db.query(ViewerRelationship).filter_by(viewer_id=vid, user_id=uid).one_or_none() + if view: view.last_view_utc = int(time.time()) + else: view = ViewerRelationship(viewer_id=vid, user_id=uid) + db.add(view) + + db.commit() + db.close() + stdout.flush() + def claim_rewards_all_users(): emails = [x[0] for x in g.db.query(Transaction.email).filter_by(claimed=None).all()] users = g.db.query(User).filter(User.email.in_(emails)).order_by(User.truescore.desc()).all() @@ -941,12 +954,7 @@ def u_username_wall(v:Optional[User], username:str): is_following = v and u.has_follower(v) if v and v.id != u.id and not v.admin_level and not session.get("GLOBAL"): - view = g.db.query(ViewerRelationship).filter_by(viewer_id=v.id, user_id=u.id).one_or_none() - if view: view.last_view_utc = int(time.time()) - else: view = ViewerRelationship(viewer_id=v.id, user_id=u.id) - g.db.add(view) - try: g.db.flush() - except: g.db.rollback() + gevent.spawn(_add_profile_view, v.id, u.id) page = get_page() @@ -993,19 +1001,10 @@ def u_username_wall_comment(v:User, username:str, cid): is_following = v and u.has_follower(v) if v and v.id != u.id and not v.admin_level and not session.get("GLOBAL"): - view = g.db.query(ViewerRelationship).filter_by(viewer_id=v.id, user_id=u.id).one_or_none() - if view: view.last_view_utc = int(time.time()) - else: view = ViewerRelationship(viewer_id=v.id, user_id=u.id) - g.db.add(view) - try: g.db.flush() - except: g.db.rollback() + gevent.spawn(_add_profile_view, v.id, u.id) if v and request.values.get("read"): - notif = g.db.query(Notification).filter_by(comment_id=cid, user_id=v.id, read=False).one_or_none() - if notif: - notif.read = True - g.db.add(notif) - g.db.flush() + gevent.spawn(_mark_comment_as_read, comment.id, v.id) try: context = min(int(request.values.get("context", 8)), 8) except: context = 8 @@ -1047,12 +1046,7 @@ def u_username(v:Optional[User], username:str): return render_template("userpage/private.html", u=u, v=v, is_following=is_following), 403 if v and v.id != u.id and not v.admin_level and not session.get("GLOBAL"): - view = g.db.query(ViewerRelationship).filter_by(viewer_id=v.id, user_id=u.id).one_or_none() - if view: view.last_view_utc = int(time.time()) - else: view = ViewerRelationship(viewer_id=v.id, user_id=u.id) - g.db.add(view) - try: g.db.flush() - except: g.db.rollback() + gevent.spawn(_add_profile_view, v.id, u.id) sort = request.values.get("sort", "new") t = request.values.get("t", "all") @@ -1119,12 +1113,7 @@ def u_username_comments(username, v): return render_template("userpage/private.html", u=u, v=v, is_following=is_following), 403 if v and v.id != u.id and not v.admin_level and not session.get("GLOBAL"): - view = g.db.query(ViewerRelationship).filter_by(viewer_id=v.id, user_id=u.id).one_or_none() - if view: view.last_view_utc = int(time.time()) - else: view = ViewerRelationship(viewer_id=v.id, user_id=u.id) - g.db.add(view) - try: g.db.flush() - except: g.db.rollback() + gevent.spawn(_add_profile_view, v.id, u.id) page = get_page() diff --git a/files/routes/wrappers.py b/files/routes/wrappers.py index b3b12d631c..966f6d0c1c 100644 --- a/files/routes/wrappers.py +++ b/files/routes/wrappers.py @@ -110,13 +110,11 @@ def get_logged_in_user(): # Check against last_active + ACTIVE_TIME to reduce frequency of # UPDATEs in exchange for a ±ACTIVE_TIME margin of error. - if not session.get("GLOBAL"): + if not session.get("GLOBAL") and request.method == "POST": timestamp = int(time.time()) if (v.last_active + LOGGEDIN_ACTIVE_TIME) < timestamp: v.last_active = timestamp g.db.add(v) - try: g.db.flush() - except: g.db.rollback() if not v and SITE == 'rdrama.net' and request.headers.get("Cf-Ipcountry") == 'EG': abort(404)