Refactor stat timers, award timers.

Prior to this comment, the every-1d cron.py command was broken due
to lack of proper stats import. However, while refactoring this, it
was convenient to move other recurring tasks that had been stuffed in
odd places--not least `stats(...)`--into the new cron system. This
entailed a number of refactorings of other things.

1. Move stats(...) from static.py to helpers/stats.py.
2. Move hole inactivity purge task from stats(...) to routes/subs.py.
3. Move bot award timer checks from stats(...) to helpers/awards.py.
4. Unify award timer logic formerly in routes/front.py into the new
   helpers/awards.py.

files/cli.py

@@ -1,4 +1,4 @@
-from .__main__ import app, db_session
+from .__main__ import app, db_session, cache
 from flask import g
 import files.helpers.cron
 

files/helpers/awards.py

@@ -0,0 +1,83 @@
+from flask import g
+import time
+from files.helpers.alerts import send_repeatable_notification
+from files.helpers.const import bots
+from files.helpers.discord import remove_role
+from files.classes.badges import Badge
+from files.classes.user import User
+
+def award_timers(v, bot=False):
+	now = time.time()
+	dirty = False
+
+	def notify_if_not_bot(msg):
+		if not bot:
+			send_repeatable_notification(v.id,msg)
+
+	if v.patron_utc and v.patron_utc < now:
+		v.patron = 0
+		v.patron_utc = 0
+		notify_if_not_bot("Your paypig status has expired!")
+		if not bot and v.discord_id: remove_role(v, "1")
+		dirty = True
+	if v.unban_utc and v.unban_utc < now:
+		v.is_banned = 0
+		v.unban_utc = 0
+		v.ban_evade = 0
+		notify_if_not_bot("You have been unbanned!")
+		dirty = True
+	if v.agendaposter and v.agendaposter < now:
+		v.agendaposter = 0
+		notify_if_not_bot("Your chud theme has expired!")
+		badge = v.has_badge(28)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.flairchanged and v.flairchanged < now:
+		v.flairchanged = None
+		notify_if_not_bot("Your flair lock has expired. You can now change your flair!")
+		badge = v.has_badge(96)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.marseyawarded and v.marseyawarded < now:
+		v.marseyawarded = None
+		notify_if_not_bot("Your marsey award has expired!")
+		badge = v.has_badge(98)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.longpost and v.longpost < now:
+		v.longpost = None
+		notify_if_not_bot("Your pizzashill award has expired!")
+		badge = v.has_badge(97)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.bird and v.bird < now:
+		v.bird = None
+		notify_if_not_bot("Your bird site award has expired!")
+		badge = v.has_badge(95)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.progressivestack and v.progressivestack < now:
+		v.progressivestack = None
+		notify_if_not_bot("Your progressive stack has expired!")
+		badge = v.has_badge(94)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.rehab and v.rehab < now:
+		v.rehab = None
+		notify_if_not_bot("Your rehab has finished!")
+		badge = v.has_badge(109)
+		if badge: g.db.delete(badge)
+		dirty = True
+	if v.deflector and v.deflector < now:
+		v.deflector = None
+		notify_if_not_bot("Your deflector has expired!")
+		dirty = True
+
+	if dirty:
+		g.db.add(v)
+		g.db.commit()
+
+def award_timers_bots_task():
+	accs = g.db.query(User).filter(User.id.in_(bots))
+	for u in accs:
+		award_timers(u, bot=True)

files/helpers/const.py

@@ -132,9 +132,11 @@ PIN_LIMIT = 3
 POST_RATE_LIMIT = "1/second;2/minute;10/hour;50/day"
 
 USER_TITLE_COST = 0
+HOLE_INACTIVITY_DELETION = False
 
 if SITE in {'rdrama.net','devrama.xyz'}:
 	HOLE_COST = 50000
+	HOLE_INACTIVITY_DELETION = True
 	USER_TITLE_COST = 25
 	NOTIFICATIONS_ID = 1046
 	AUTOJANNY_ID = 2360

files/helpers/cron.py

@@ -5,6 +5,9 @@ import files.helpers.const as const
 import files.helpers.lottery as lottery
 import files.helpers.offsitementions as offsitementions
 import files.helpers.stats as stats
+import files.helpers.awards as awards
+import files.routes.static as route_static
+from files.routes.subs import sub_inactive_purge_task
 
 @app.cli.command('cron', help='Run scheduled tasks.')
 @click.option('--every-5m', is_flag=True, help='Call every 5 minutes.')
@@ -22,4 +25,8 @@ def cron(every_5m, every_1h, every_1d):
 
 	if every_1d:
 		stats.generate_charts_task(const.SITE)
+		route_static.stats_cached()
+		awards.award_timers_bots_task()
+		sub_inactive_purge_task()
+
 

files/helpers/stats.py

@@ -2,10 +2,14 @@ from flask import g
 import time
 import calendar
 import matplotlib.pyplot as plt
+from sqlalchemy import *
 
 from files.classes.user import User
 from files.classes.submission import Submission
 from files.classes.comment import Comment
+from files.classes.votes import Vote, CommentVote
+from files.classes.marsey import Marsey
+from files.classes.award import AwardRelationship
 from files.helpers.const import *
 
 def generate_charts_task(site):
@@ -85,3 +89,58 @@ def chart(kind, site):
 
 def chart_path(kind, site):
 	return f'/{site}_{kind}.png'
+
+def stats(site=None):
+	day = int(time.time()) - 86400
+	week = int(time.time()) - 604800
+	posters = g.db.query(Submission.author_id).distinct(Submission.author_id).filter(Submission.created_utc > week).all()
+	commenters = g.db.query(Comment.author_id).distinct(Comment.author_id).filter(Comment.created_utc > week).all()
+	voters = g.db.query(Vote.user_id).distinct(Vote.user_id).filter(Vote.created_utc > week).all()
+	commentvoters = g.db.query(CommentVote.user_id).distinct(CommentVote.user_id).filter(CommentVote.created_utc > week).all()
+	active_users = set(posters) | set(commenters) | set(voters) | set(commentvoters)
+
+	stats = {
+			"marseys": g.db.query(Marsey).count(),
+			"users": g.db.query(User).count(),
+			"private users": g.db.query(User).filter_by(is_private=True).count(),
+			"banned users": g.db.query(User).filter(User.is_banned > 0).count(),
+			"verified email users": g.db.query(User).filter_by(is_activated=True).count(),
+			"coins in circulation": g.db.query(func.sum(User.coins)).scalar(),
+			"total shop sales": g.db.query(func.sum(User.coins_spent)).scalar(),
+			"signups last 24h": g.db.query(User).filter(User.created_utc > day).count(),
+			"total posts": g.db.query(Submission).count(),
+			"posting users": g.db.query(Submission.author_id).distinct().count(),
+			"listed posts": g.db.query(Submission).filter_by(is_banned=False).filter(Submission.deleted_utc == 0).count(),
+			"removed posts (by admins)": g.db.query(Submission).filter_by(is_banned=True).count(),
+			"deleted posts (by author)": g.db.query(Submission).filter(Submission.deleted_utc > 0).count(),
+			"posts last 24h": g.db.query(Submission).filter(Submission.created_utc > day).count(),
+			"total comments": g.db.query(Comment).filter(Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(),
+			"commenting users": g.db.query(Comment.author_id).distinct().count(),
+			"removed comments (by admins)": g.db.query(Comment).filter_by(is_banned=True).count(),
+			"deleted comments (by author)": g.db.query(Comment).filter(Comment.deleted_utc > 0).count(),
+			"comments last_24h": g.db.query(Comment).filter(Comment.created_utc > day, Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(),
+			"post votes": g.db.query(Vote).count(),
+			"post voting users": g.db.query(Vote.user_id).distinct().count(),
+			"comment votes": g.db.query(CommentVote).count(),
+			"comment voting users": g.db.query(CommentVote.user_id).distinct().count(),
+			"total upvotes": g.db.query(Vote).filter_by(vote_type=1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=1).count(),
+			"total downvotes": g.db.query(Vote).filter_by(vote_type=-1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=-1).count(),
+			"total awards": g.db.query(AwardRelationship).count(),
+			"awards given": g.db.query(AwardRelationship).filter(or_(AwardRelationship.submission_id != None, AwardRelationship.comment_id != None)).count(),
+			"users who posted, commented, or voted in the past 7 days": len(active_users),
+			}
+
+	if site == 'rDrama':
+		stats2 = {
+			"House furry members": g.db.query(User).filter(User.house.like('Furry%')).count(),
+			"House femboy members": g.db.query(User).filter(User.house.like('Femboy%')).count(),
+			"House vampire members": g.db.query(User).filter(User.house.like('Vampire%')).count(),
+			"House racist members": g.db.query(User).filter(User.house.like('Racist%')).count(),
+			"House furry total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Furry%')).scalar(),
+			"House femboy total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Femboy%')).scalar(),
+			"House vampire total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Vampire%')).scalar(),
+			"House racist total truescore": g.db.query(func.sum(User.truecoins)).filter(User.house.like('Racist%')).scalar(),
+			}
+		stats.update(stats2)
+
+	return stats

files/routes/awards.py

@@ -443,3 +443,4 @@ def admin_userawards_post(v):
 
 	if v.admin_level != 3: return render_template("admin/awards.html", awards=list(AWARDS3.values()), v=v)
 	return render_template("admin/awards.html", awards=list(AWARDS.values()), v=v) 
+

files/routes/front.py

@@ -3,6 +3,7 @@ from files.helpers.get import *
 from files.helpers.discord import *
 from files.__main__ import app, cache, limiter
 from files.classes.submission import Submission
+from files.helpers.awards import award_timers
 
 defaulttimefilter = environ.get("DEFAULT_TIME_FILTER", "all").strip()
 
@@ -219,85 +220,7 @@ def front_all(v, sub=None, subdomain=None):
 	
 	if v:
 		if v.hidevotedon: posts = [x for x in posts if not hasattr(x, 'voted') or not x.voted]
-
+		award_timers(v)
-	
-		if v.patron_utc and v.patron_utc < time.time():
-			v.patron = 0
-			v.patron_utc = 0
-			send_repeatable_notification(v.id, "Your paypig status has expired!")
-			if v.discord_id: remove_role(v, "1")
-			g.db.add(v)
-			g.db.commit()
-
-		if v.unban_utc and v.unban_utc < time.time():
-			v.is_banned = 0
-			v.unban_utc = 0
-			v.ban_evade = 0
-			send_repeatable_notification(v.id, "You have been unbanned!")
-			g.db.add(v)
-			g.db.commit()
-
-		if v.agendaposter and v.agendaposter < time.time():
-			v.agendaposter = 0
-			send_repeatable_notification(v.id, "Your chud theme has expired!")
-			g.db.add(v)
-			badge = v.has_badge(28)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.flairchanged and v.flairchanged < time.time():
-			v.flairchanged = None
-			send_repeatable_notification(v.id, "Your flair lock has expired. You can now change your flair!")
-			g.db.add(v)
-			badge = v.has_badge(96)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.marseyawarded and v.marseyawarded < time.time():
-			v.marseyawarded = None
-			send_repeatable_notification(v.id, "Your marsey award has expired!")
-			g.db.add(v)
-			badge = v.has_badge(98)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.longpost and v.longpost < time.time():
-			v.longpost = None
-			send_repeatable_notification(v.id, "Your pizzashill award has expired!")
-			g.db.add(v)
-			badge = v.has_badge(97)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.bird and v.bird < time.time():
-			v.bird = None
-			send_repeatable_notification(v.id, "Your bird site award has expired!")
-			g.db.add(v)
-			badge = v.has_badge(95)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.progressivestack and v.progressivestack < time.time():
-			v.progressivestack = None
-			send_repeatable_notification(v.id, "Your progressive stack has expired!")
-			g.db.add(v)
-			badge = v.has_badge(94)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.rehab and v.rehab < time.time():
-			v.rehab = None
-			send_repeatable_notification(v.id, "Your rehab has finished!")
-			g.db.add(v)
-			badge = v.has_badge(109)
-			if badge: g.db.delete(badge)
-			g.db.commit()
-
-		if v.deflector and v.deflector < time.time():
-			v.deflector = None
-			send_repeatable_notification(v.id, "Your deflector has expired!")
-			g.db.add(v)
-			g.db.commit()
 
 	if request.headers.get("Authorization"): return {"data": [x.json for x in posts], "next_exists": next_exists}
 	return render_template("home.html", v=v, listing=posts, next_exists=next_exists, sort=sort, t=t, page=page, ccmode=ccmode, sub=sub, home=True)

files/routes/static.py

@@ -69,151 +69,12 @@ def sidebar(v):
 @app.get("/stats")
 @auth_required
 def participation_stats(v):
-
+	return render_template("admin/content_stats.html", 
-	return render_template("admin/content_stats.html", v=v, title="Content Statistics", data=stats(site=SITE))
+		v=v, title="Content Statistics", data=stats_cached())
-
 
 @cache.memoize(timeout=86400)
-def stats(site=None):
+def stats_cached():
-	day = int(time.time()) - 86400
+	return statshelper.stats(SITE_NAME)
-
-	week = int(time.time()) - 604800
-	posters = g.db.query(Submission.author_id).distinct(Submission.author_id).filter(Submission.created_utc > week).all()
-	commenters = g.db.query(Comment.author_id).distinct(Comment.author_id).filter(Comment.created_utc > week).all()
-	voters = g.db.query(Vote.user_id).distinct(Vote.user_id).filter(Vote.created_utc > week).all()
-	commentvoters = g.db.query(CommentVote.user_id).distinct(CommentVote.user_id).filter(CommentVote.created_utc > week).all()
-
-	active_users = set(posters) | set(commenters) | set(voters) | set(commentvoters)
-
-	stats = {"marseys": g.db.query(Marsey).count(),
-			"users": g.db.query(User).count(),
-			"private users": g.db.query(User).filter_by(is_private=True).count(),
-			"banned users": g.db.query(User).filter(User.is_banned > 0).count(),
-			"verified email users": g.db.query(User).filter_by(is_activated=True).count(),
-			"coins in circulation": g.db.query(func.sum(User.coins)).scalar(),
-			"total shop sales": g.db.query(func.sum(User.coins_spent)).scalar(),
-			"signups last 24h": g.db.query(User).filter(User.created_utc > day).count(),
-			"total posts": g.db.query(Submission).count(),
-			"posting users": g.db.query(Submission.author_id).distinct().count(),
-			"listed posts": g.db.query(Submission).filter_by(is_banned=False).filter(Submission.deleted_utc == 0).count(),
-			"removed posts (by admins)": g.db.query(Submission).filter_by(is_banned=True).count(),
-			"deleted posts (by author)": g.db.query(Submission).filter(Submission.deleted_utc > 0).count(),
-			"posts last 24h": g.db.query(Submission).filter(Submission.created_utc > day).count(),
-			"total comments": g.db.query(Comment).filter(Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(),
-			"commenting users": g.db.query(Comment.author_id).distinct().count(),
-			"removed comments (by admins)": g.db.query(Comment).filter_by(is_banned=True).count(),
-			"deleted comments (by author)": g.db.query(Comment).filter(Comment.deleted_utc > 0).count(),
-			"comments last_24h": g.db.query(Comment).filter(Comment.created_utc > day, Comment.author_id.notin_((AUTOJANNY_ID,NOTIFICATIONS_ID))).count(),
-			"post votes": g.db.query(Vote).count(),
-			"post voting users": g.db.query(Vote.user_id).distinct().count(),
-			"comment votes": g.db.query(CommentVote).count(),
-			"comment voting users": g.db.query(CommentVote.user_id).distinct().count(),
-			"total upvotes": g.db.query(Vote).filter_by(vote_type=1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=1).count(),
-			"total downvotes": g.db.query(Vote).filter_by(vote_type=-1).count() + g.db.query(CommentVote.comment_id).filter_by(vote_type=-1).count(),
-			"total awards": g.db.query(AwardRelationship).count(),
-			"awards given": g.db.query(AwardRelationship).filter(or_(AwardRelationship.submission_id != None, AwardRelationship.comment_id != None)).count(),
-			"users who posted, commented, or voted in the past 7 days": len(active_users),
-			}
-
-
-	if SITE_NAME == 'rDrama':
-		furries1 = g.db.query(User).filter(User.house.like('Furry%')).count()
-		femboys1 = g.db.query(User).filter(User.house.like('Femboy%')).count()
-		vampires1 = g.db.query(User).filter(User.house.like('Vampire%')).count()
-		racists1 = g.db.query(User).filter(User.house.like('Racist%')).count()
-
-		furries2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Furry%')).scalar()
-		femboys2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Femboy%')).scalar()
-		vampires2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Vampire%')).scalar()
-		racists2 = g.db.query(func.sum(User.truecoins)).filter(User.house.like('Racist%')).scalar()
-
-		stats2 = {"House furry members": furries1,
-			"House femboy members": femboys1,
-			"House vampire members": vampires1,
-			"House racist members": racists1,
-			"House furry total truescore": furries2,
-			"House femboy total truescore": femboys2,
-			"House vampire total truescore": vampires2,
-			"House racist total truescore": racists2,
-			}
-
-		stats.update(stats2)
-
-		accs = g.db.query(User).filter(User.id.in_(bots))
-
-		for u in accs:
-			g.db.add(u)
-
-			if u.patron_utc and u.patron_utc < time.time():
-				u.patron = 0
-				u.patron_utc = 0
-
-			if u.unban_utc and u.unban_utc < time.time():
-				u.is_banned = 0
-				u.unban_utc = 0
-				u.ban_evade = 0
-
-			if u.agendaposter and u.agendaposter < time.time():
-				u.agendaposter = 0
-				badge = u.has_badge(28)
-				if badge: g.db.delete(badge)
-
-			if u.flairchanged and u.flairchanged < time.time():
-				u.flairchanged = None
-				badge = u.has_badge(96)
-				if badge: g.db.delete(badge)
-
-			if u.marseyawarded and u.marseyawarded < time.time():
-				u.marseyawarded = None
-				badge = u.has_badge(98)
-				if badge: g.db.delete(badge)
-
-			if u.longpost and u.longpost < time.time():
-				u.longpost = None
-				badge = u.has_badge(97)
-				if badge: g.db.delete(badge)
-
-			if u.bird and u.bird < time.time():
-				u.bird = None
-				badge = u.has_badge(95)
-				if badge: g.db.delete(badge)
-
-			if u.progressivestack and u.progressivestack < time.time():
-				u.progressivestack = None
-				badge = u.has_badge(94)
-				if badge: g.db.delete(badge)
-
-			if u.rehab and u.rehab < time.time():
-				u.rehab = None
-				badge = u.has_badge(109)
-				if badge: g.db.delete(badge)
-
-			if u.deflector and u.deflector < time.time():
-				u.deflector = None
-
-		one_week_ago = time.time() - 604800
-		active_holes = [x[0] for x in g.db.query(Submission.sub).distinct().filter(Submission.sub != None, Submission.created_utc > one_week_ago).all()]
-
-		dead_holes = g.db.query(Sub).filter(Sub.name.notin_(active_holes)).all()
-		names = [x.name for x in dead_holes]
-
-		posts = g.db.query(Submission).filter(Submission.sub.in_(names)).all()
-		for post in posts:
-			post.sub = None
-			g.db.add(post)
-
-		to_delete = g.db.query(Mod).filter(Mod.sub.in_(names)).all() + g.db.query(Exile).filter(Exile.sub.in_(names)).all() + g.db.query(SubBlock).filter(SubBlock.sub.in_(names)).all() + g.db.query(SubSubscription).filter(SubSubscription.sub.in_(names)).all()
-		for x in to_delete:
-			g.db.delete(x)
-		g.db.flush()
-
-		for x in dead_holes:
-			g.db.delete(x)
-
-	g.db.commit()
-
-	return stats
-
 
 @app.get("/chart")
 def chart():

files/routes/subs.py

@@ -458,4 +458,35 @@ def sub_sidebar(v, sub):
 @auth_desired
 def subs(v):
 	subs = g.db.query(Sub, func.count(Submission.sub)).outerjoin(Submission, Sub.name == Submission.sub).group_by(Sub.name).order_by(func.count(Submission.sub).desc()).all()
-	return render_template('sub/subs.html', v=v, subs=subs)
+	return render_template('sub/subs.html', v=v, subs=subs)
+
+def sub_inactive_purge_task():
+	if not HOLE_INACTIVITY_DELETION:
+		return False
+
+	one_week_ago = time.time() - 604800
+	active_holes = [x[0] for x in g.db.query(Submission.sub).distinct() \
+		.filter(Submission.sub != None, Submission.created_utc > one_week_ago).all()]
+
+	dead_holes = g.db.query(Sub).filter(Sub.name.notin_(active_holes)).all()
+	names = [x.name for x in dead_holes]
+
+	posts = g.db.query(Submission).filter(Submission.sub.in_(names)).all()
+	for post in posts:
+		post.sub = None
+		g.db.add(post)
+
+	to_delete = g.db.query(Mod).filter(Mod.sub.in_(names)).all() \
+		+ g.db.query(Exile).filter(Exile.sub.in_(names)).all() \
+		+ g.db.query(SubBlock).filter(SubBlock.sub.in_(names)).all() \
+		+ g.db.query(SubSubscription).filter(SubSubscription.sub.in_(names)).all()
+
+	for x in to_delete:
+		g.db.delete(x)
+	g.db.flush()
+
+	for x in dead_holes:
+		g.db.delete(x)
+
+	g.db.commit()
+	return True