diff --git a/files/cli.py b/files/cli.py index 85cbf5b6f..b16c34d98 100644 --- a/files/cli.py +++ b/files/cli.py @@ -1,4 +1,4 @@ -from .__main__ import app, db_session +from .__main__ import app, db_session, cache from flask import g import files.helpers.cron diff --git a/files/helpers/awards.py b/files/helpers/awards.py new file mode 100644 index 000000000..c2a40234b --- /dev/null +++ b/files/helpers/awards.py @@ -0,0 +1,83 @@ +from flask import g +import time +from files.helpers.alerts import send_repeatable_notification +from files.helpers.const import bots +from files.helpers.discord import remove_role +from files.classes.badges import Badge +from files.classes.user import User + +def award_timers(v, bot=False): + now = time.time() + dirty = False + + def notify_if_not_bot(msg): + if not bot: + send_repeatable_notification(v.id,msg) + + if v.patron_utc and v.patron_utc < now: + v.patron = 0 + v.patron_utc = 0 + notify_if_not_bot("Your paypig status has expired!") + if not bot and v.discord_id: remove_role(v, "1") + dirty = True + if v.unban_utc and v.unban_utc < now: + v.is_banned = 0 + v.unban_utc = 0 + v.ban_evade = 0 + notify_if_not_bot("You have been unbanned!") + dirty = True + if v.agendaposter and v.agendaposter < now: + v.agendaposter = 0 + notify_if_not_bot("Your chud theme has expired!") + badge = v.has_badge(28) + if badge: g.db.delete(badge) + dirty = True + if v.flairchanged and v.flairchanged < now: + v.flairchanged = None + notify_if_not_bot("Your flair lock has expired. You can now change your flair!") + badge = v.has_badge(96) + if badge: g.db.delete(badge) + dirty = True + if v.marseyawarded and v.marseyawarded < now: + v.marseyawarded = None + notify_if_not_bot("Your marsey award has expired!") + badge = v.has_badge(98) + if badge: g.db.delete(badge) + dirty = True + if v.longpost and v.longpost < now: + v.longpost = None + notify_if_not_bot("Your pizzashill award has expired!") + badge = v.has_badge(97) + if badge: g.db.delete(badge) + dirty = True + if v.bird and v.bird < now: + v.bird = None + notify_if_not_bot("Your bird site award has expired!") + badge = v.has_badge(95) + if badge: g.db.delete(badge) + dirty = True + if v.progressivestack and v.progressivestack < now: + v.progressivestack = None + notify_if_not_bot("Your progressive stack has expired!") + badge = v.has_badge(94) + if badge: g.db.delete(badge) + dirty = True + if v.rehab and v.rehab < now: + v.rehab = None + notify_if_not_bot("Your rehab has finished!") + badge = v.has_badge(109) + if badge: g.db.delete(badge) + dirty = True + if v.deflector and v.deflector < now: + v.deflector = None + notify_if_not_bot("Your deflector has expired!") + dirty = True + + if dirty: + g.db.add(v) + g.db.commit() + +def award_timers_bots_task(): + accs = g.db.query(User).filter(User.id.in_(bots)) + for u in accs: + award_timers(u, bot=True) diff --git a/files/helpers/const.py b/files/helpers/const.py index 40b859f61..89f484119 100644 --- a/files/helpers/const.py +++ b/files/helpers/const.py @@ -132,9 +132,11 @@ PIN_LIMIT = 3 POST_RATE_LIMIT = "1/second;2/minute;10/hour;50/day" USER_TITLE_COST = 0 +HOLE_INACTIVITY_DELETION = False if SITE in {'rdrama.net','devrama.xyz'}: HOLE_COST = 50000 + HOLE_INACTIVITY_DELETION = True USER_TITLE_COST = 25 NOTIFICATIONS_ID = 1046 AUTOJANNY_ID = 2360 diff --git a/files/helpers/cron.py b/files/helpers/cron.py index a0546a063..9012ecdde 100644 --- a/files/helpers/cron.py +++ b/files/helpers/cron.py @@ -5,6 +5,9 @@ import files.helpers.const as const import files.helpers.lottery as lottery import files.helpers.offsitementions as offsitementions import files.helpers.stats as stats +import files.helpers.awards as awards +import files.routes.static as route_static +from files.routes.subs import sub_inactive_purge_task @app.cli.command('cron', help='Run scheduled tasks.') @click.option('--every-5m', is_flag=True, help='Call every 5 minutes.') @@ -22,4 +25,8 @@ def cron(every_5m, every_1h, every_1d): if every_1d: stats.generate_charts_task(const.SITE) + route_static.stats_cached() + awards.award_timers_bots_task() + sub_inactive_purge_task() + diff --git a/files/helpers/stats.py b/files/helpers/stats.py index 8e354599e..db5e1f719 100644 --- a/files/helpers/stats.py +++ b/files/helpers/stats.py @@ -2,10 +2,14 @@ from flask import g import time import calendar import matplotlib.pyplot as plt +from sqlalchemy import * from files.classes.user import User from files.classes.submission import Submission from files.classes.comment import Comment +from files.classes.votes import Vote, CommentVote +from files.classes.marsey import Marsey +from files.classes.award import AwardRelationship from files.helpers.const import * def generate_charts_task(site): @@ -85,3 +89,58 @@ def chart(kind, site): def chart_path(kind, site): return f'/{site}_{kind}.png' + +def stats(site=None): + day = int(time.time()) - 86400 + week = int(time.time()) - 604800 + posters = g.db.query(Submission.author_id).distinct(Submission.author_id).filter(Submission.created_utc > week).all() + commenters = g.db.query(Comment.author_id).distinct(Comment.author_id).filter(Comment.created_utc > week).all() + voters = g.db.query(Vote.user_id).distinct(Vote.user_id).filter(Vote.created_utc > week).all() + commentvoters = g.db.query(CommentVote.user_id).distinct(CommentVote.user_id).filter(CommentVote.created_utc > week).all() + active_users = set(posters) | set(commenters) | set(voters) | set(commentvoters) + + stats = { + "marseys": g.db.query(Marsey).count(), + "users": g.db.query(User).count(), + "private users": g.db.query(User).filter_by(is_private=True).count(), + "banned users": g.db.query(User).filter(User.is_banned > 0).count(), + "verified email users": g.db.query(User).filter_by(is_activated=True).count(), + "coins in circulation": g.db.query(func.sum(User.coins)).scalar(), + "total shop sales": g.db.query(func.sum(User.coins_spent)).scalar(), + "signups last 24h": g.db.query(User).filter(User.created_utc > day).count(), + "total posts": g.db.query(Submission).count(), + "posting users": g.db.query(Submission.author_id).distinct().count(), + "listed posts": g.db.query(Submission).filter_by(is_banned=False).filter(Submission.deleted_utc == 0).count(), + "removed posts (by admins)": g.db.query(Submission).filter_by(is_banned=True).count(), + "deleted posts (by author)": g.db.query(Submission).filter(Submission.deleted_utc > 0).count(), + "posts last 24h": g.db.query(Submission).filter(Submission.created_utc > day).count(), + "total comments": g.db.query(Comment).filter(Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(), + "commenting users": g.db.query(Comment.author_id).distinct().count(), + "removed comments (by admins)": g.db.query(Comment).filter_by(is_banned=True).count(), + "deleted comments (by author)": g.db.query(Comment).filter(Comment.deleted_utc > 0).count(), + "comments last_24h": g.db.query(Comment).filter(Comment.created_utc > day, Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(), + "post votes": g.db.query(Vote).count(), + "post voting users": g.db.query(Vote.user_id).distinct().count(), + "comment votes": g.db.query(CommentVote).count(), + "comment voting users": g.db.query(CommentVote.user_id).distinct().count(), + "total upvotes": g.db.query(Vote).filter_by(vote_type=1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=1).count(), + "total downvotes": g.db.query(Vote).filter_by(vote_type=-1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=-1).count(), + "total awards": g.db.query(AwardRelationship).count(), + "awards given": g.db.query(AwardRelationship).filter(or_(AwardRelationship.submission_id != None, AwardRelationship.comment_id != None)).count(), + "users who posted, commented, or voted in the past 7 days": len(active_users), + } + + if site == 'rDrama': + stats2 = { + "House furry members": g.db.query(User).filter(User.house.like('Furry%')).count(), + "House femboy members": g.db.query(User).filter(User.house.like('Femboy%')).count(), + "House vampire members": g.db.query(User).filter(User.house.like('Vampire%')).count(), + "House racist members": g.db.query(User).filter(User.house.like('Racist%')).count(), + "House furry total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Furry%')).scalar(), + "House femboy total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Femboy%')).scalar(), + "House vampire total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Vampire%')).scalar(), + "House racist total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Racist%')).scalar(), + } + stats.update(stats2) + + return stats diff --git a/files/routes/awards.py b/files/routes/awards.py index bd04392e0..5006f6382 100644 --- a/files/routes/awards.py +++ b/files/routes/awards.py @@ -443,3 +443,4 @@ def admin_userawards_post(v): if v.admin_level != 3: return render_template("admin/awards.html", awards=list(AWARDS3.values()), v=v) return render_template("admin/awards.html", awards=list(AWARDS.values()), v=v) + diff --git a/files/routes/front.py b/files/routes/front.py index 41e5076e6..091aa2cbc 100644 --- a/files/routes/front.py +++ b/files/routes/front.py @@ -3,6 +3,7 @@ from files.helpers.get import * from files.helpers.discord import * from files.__main__ import app, cache, limiter from files.classes.submission import Submission +from files.helpers.awards import award_timers defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip() @@ -219,85 +220,7 @@ def front_all(v, sub=None, subdomain=None): if v: if v.hidevotedon: posts = [x for x in posts if not hasattr(x, 'voted') or not x.voted] - - - if v.patron_utc and v.patron_utc < time.time(): - v.patron = 0 - v.patron_utc = 0 - send_repeatable_notification(v.id, "Your paypig status has expired!") - if v.discord_id: remove_role(v, "1") - g.db.add(v) - g.db.commit() - - if v.unban_utc and v.unban_utc < time.time(): - v.is_banned = 0 - v.unban_utc = 0 - v.ban_evade = 0 - send_repeatable_notification(v.id, "You have been unbanned!") - g.db.add(v) - g.db.commit() - - if v.agendaposter and v.agendaposter < time.time(): - v.agendaposter = 0 - send_repeatable_notification(v.id, "Your chud theme has expired!") - g.db.add(v) - badge = v.has_badge(28) - if badge: g.db.delete(badge) - g.db.commit() - - if v.flairchanged and v.flairchanged < time.time(): - v.flairchanged = None - send_repeatable_notification(v.id, "Your flair lock has expired. You can now change your flair!") - g.db.add(v) - badge = v.has_badge(96) - if badge: g.db.delete(badge) - g.db.commit() - - if v.marseyawarded and v.marseyawarded < time.time(): - v.marseyawarded = None - send_repeatable_notification(v.id, "Your marsey award has expired!") - g.db.add(v) - badge = v.has_badge(98) - if badge: g.db.delete(badge) - g.db.commit() - - if v.longpost and v.longpost < time.time(): - v.longpost = None - send_repeatable_notification(v.id, "Your pizzashill award has expired!") - g.db.add(v) - badge = v.has_badge(97) - if badge: g.db.delete(badge) - g.db.commit() - - if v.bird and v.bird < time.time(): - v.bird = None - send_repeatable_notification(v.id, "Your bird site award has expired!") - g.db.add(v) - badge = v.has_badge(95) - if badge: g.db.delete(badge) - g.db.commit() - - if v.progressivestack and v.progressivestack < time.time(): - v.progressivestack = None - send_repeatable_notification(v.id, "Your progressive stack has expired!") - g.db.add(v) - badge = v.has_badge(94) - if badge: g.db.delete(badge) - g.db.commit() - - if v.rehab and v.rehab < time.time(): - v.rehab = None - send_repeatable_notification(v.id, "Your rehab has finished!") - g.db.add(v) - badge = v.has_badge(109) - if badge: g.db.delete(badge) - g.db.commit() - - if v.deflector and v.deflector < time.time(): - v.deflector = None - send_repeatable_notification(v.id, "Your deflector has expired!") - g.db.add(v) - g.db.commit() + award_timers(v) if request.headers.get("Authorization"): return {"data": [x.json for x in posts], "next_exists": next_exists} return render_template("home.html", v=v, listing=posts, next_exists=next_exists, sort=sort, t=t, page=page, ccmode=ccmode, sub=sub, home=True) diff --git a/files/routes/static.py b/files/routes/static.py index 64ea64a5c..f415489c5 100644 --- a/files/routes/static.py +++ b/files/routes/static.py @@ -69,151 +69,12 @@ def sidebar(v): @app.get("/stats") @auth_required def participation_stats(v): - - return render_template("admin/content_stats.html", v=v, title="Content Statistics", data=stats(site=SITE)) - + return render_template("admin/content_stats.html", + v=v, title="Content Statistics", data=stats_cached()) @cache.memoize(timeout=86400) -def stats(site=None): - day = int(time.time()) - 86400 - - week = int(time.time()) - 604800 - posters = g.db.query(Submission.author_id).distinct(Submission.author_id).filter(Submission.created_utc > week).all() - commenters = g.db.query(Comment.author_id).distinct(Comment.author_id).filter(Comment.created_utc > week).all() - voters = g.db.query(Vote.user_id).distinct(Vote.user_id).filter(Vote.created_utc > week).all() - commentvoters = g.db.query(CommentVote.user_id).distinct(CommentVote.user_id).filter(CommentVote.created_utc > week).all() - - active_users = set(posters) | set(commenters) | set(voters) | set(commentvoters) - - stats = {"marseys": g.db.query(Marsey).count(), - "users": g.db.query(User).count(), - "private users": g.db.query(User).filter_by(is_private=True).count(), - "banned users": g.db.query(User).filter(User.is_banned > 0).count(), - "verified email users": g.db.query(User).filter_by(is_activated=True).count(), - "coins in circulation": g.db.query(func.sum(User.coins)).scalar(), - "total shop sales": g.db.query(func.sum(User.coins_spent)).scalar(), - "signups last 24h": g.db.query(User).filter(User.created_utc > day).count(), - "total posts": g.db.query(Submission).count(), - "posting users": g.db.query(Submission.author_id).distinct().count(), - "listed posts": g.db.query(Submission).filter_by(is_banned=False).filter(Submission.deleted_utc == 0).count(), - "removed posts (by admins)": g.db.query(Submission).filter_by(is_banned=True).count(), - "deleted posts (by author)": g.db.query(Submission).filter(Submission.deleted_utc > 0).count(), - "posts last 24h": g.db.query(Submission).filter(Submission.created_utc > day).count(), - "total comments": g.db.query(Comment).filter(Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(), - "commenting users": g.db.query(Comment.author_id).distinct().count(), - "removed comments (by admins)": g.db.query(Comment).filter_by(is_banned=True).count(), - "deleted comments (by author)": g.db.query(Comment).filter(Comment.deleted_utc > 0).count(), - "comments last_24h": g.db.query(Comment).filter(Comment.created_utc > day, Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(), - "post votes": g.db.query(Vote).count(), - "post voting users": g.db.query(Vote.user_id).distinct().count(), - "comment votes": g.db.query(CommentVote).count(), - "comment voting users": g.db.query(CommentVote.user_id).distinct().count(), - "total upvotes": g.db.query(Vote).filter_by(vote_type=1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=1).count(), - "total downvotes": g.db.query(Vote).filter_by(vote_type=-1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=-1).count(), - "total awards": g.db.query(AwardRelationship).count(), - "awards given": g.db.query(AwardRelationship).filter(or_(AwardRelationship.submission_id != None, AwardRelationship.comment_id != None)).count(), - "users who posted, commented, or voted in the past 7 days": len(active_users), - } - - - if SITE_NAME == 'rDrama': - furries1 = g.db.query(User).filter(User.house.like('Furry%')).count() - femboys1 = g.db.query(User).filter(User.house.like('Femboy%')).count() - vampires1 = g.db.query(User).filter(User.house.like('Vampire%')).count() - racists1 = g.db.query(User).filter(User.house.like('Racist%')).count() - - furries2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Furry%')).scalar() - femboys2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Femboy%')).scalar() - vampires2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Vampire%')).scalar() - racists2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Racist%')).scalar() - - stats2 = {"House furry members": furries1, - "House femboy members": femboys1, - "House vampire members": vampires1, - "House racist members": racists1, - "House furry total truescore": furries2, - "House femboy total truescore": femboys2, - "House vampire total truescore": vampires2, - "House racist total truescore": racists2, - } - - stats.update(stats2) - - accs = g.db.query(User).filter(User.id.in_(bots)) - - for u in accs: - g.db.add(u) - - if u.patron_utc and u.patron_utc < time.time(): - u.patron = 0 - u.patron_utc = 0 - - if u.unban_utc and u.unban_utc < time.time(): - u.is_banned = 0 - u.unban_utc = 0 - u.ban_evade = 0 - - if u.agendaposter and u.agendaposter < time.time(): - u.agendaposter = 0 - badge = u.has_badge(28) - if badge: g.db.delete(badge) - - if u.flairchanged and u.flairchanged < time.time(): - u.flairchanged = None - badge = u.has_badge(96) - if badge: g.db.delete(badge) - - if u.marseyawarded and u.marseyawarded < time.time(): - u.marseyawarded = None - badge = u.has_badge(98) - if badge: g.db.delete(badge) - - if u.longpost and u.longpost < time.time(): - u.longpost = None - badge = u.has_badge(97) - if badge: g.db.delete(badge) - - if u.bird and u.bird < time.time(): - u.bird = None - badge = u.has_badge(95) - if badge: g.db.delete(badge) - - if u.progressivestack and u.progressivestack < time.time(): - u.progressivestack = None - badge = u.has_badge(94) - if badge: g.db.delete(badge) - - if u.rehab and u.rehab < time.time(): - u.rehab = None - badge = u.has_badge(109) - if badge: g.db.delete(badge) - - if u.deflector and u.deflector < time.time(): - u.deflector = None - - one_week_ago = time.time() - 604800 - active_holes = [x[0] for x in g.db.query(Submission.sub).distinct().filter(Submission.sub != None, Submission.created_utc > one_week_ago).all()] - - dead_holes = g.db.query(Sub).filter(Sub.name.notin_(active_holes)).all() - names = [x.name for x in dead_holes] - - posts = g.db.query(Submission).filter(Submission.sub.in_(names)).all() - for post in posts: - post.sub = None - g.db.add(post) - - to_delete = g.db.query(Mod).filter(Mod.sub.in_(names)).all() + g.db.query(Exile).filter(Exile.sub.in_(names)).all() + g.db.query(SubBlock).filter(SubBlock.sub.in_(names)).all() + g.db.query(SubSubscription).filter(SubSubscription.sub.in_(names)).all() - for x in to_delete: - g.db.delete(x) - g.db.flush() - - for x in dead_holes: - g.db.delete(x) - - g.db.commit() - - return stats - +def stats_cached(): + return statshelper.stats(SITE_NAME) @app.get("/chart") def chart(): diff --git a/files/routes/subs.py b/files/routes/subs.py index ff7d327f4..22c4e13ac 100644 --- a/files/routes/subs.py +++ b/files/routes/subs.py @@ -458,4 +458,35 @@ def sub_sidebar(v, sub): @auth_desired def subs(v): subs = g.db.query(Sub, func.count(Submission.sub)).outerjoin(Submission, Sub.name == Submission.sub).group_by(Sub.name).order_by(func.count(Submission.sub).desc()).all() - return render_template('sub/subs.html', v=v, subs=subs) \ No newline at end of file + return render_template('sub/subs.html', v=v, subs=subs) + +def sub_inactive_purge_task(): + if not HOLE_INACTIVITY_DELETION: + return False + + one_week_ago = time.time() - 604800 + active_holes = [x[0] for x in g.db.query(Submission.sub).distinct() \ + .filter(Submission.sub != None, Submission.created_utc > one_week_ago).all()] + + dead_holes = g.db.query(Sub).filter(Sub.name.notin_(active_holes)).all() + names = [x.name for x in dead_holes] + + posts = g.db.query(Submission).filter(Submission.sub.in_(names)).all() + for post in posts: + post.sub = None + g.db.add(post) + + to_delete = g.db.query(Mod).filter(Mod.sub.in_(names)).all() \ + + g.db.query(Exile).filter(Exile.sub.in_(names)).all() \ + + g.db.query(SubBlock).filter(SubBlock.sub.in_(names)).all() \ + + g.db.query(SubSubscription).filter(SubSubscription.sub.in_(names)).all() + + for x in to_delete: + g.db.delete(x) + g.db.flush() + + for x in dead_holes: + g.db.delete(x) + + g.db.commit() + return True