2022-11-15 09:19:08 +00:00
import datetime
import time
2023-02-25 19:18:40 +00:00
import os
2022-11-15 09:19:08 +00:00
from sys import stdout
2023-02-24 23:14:18 +00:00
from shutil import make_archive
from hashlib import md5
2023-04-25 09:31:02 +00:00
from collections import Counter
2023-07-03 01:58:02 +00:00
from sqlalchemy . orm import load_only , InstrumentedAttribute
2023-02-25 19:18:40 +00:00
2022-06-07 10:57:26 +00:00
import click
2022-11-15 09:19:08 +00:00
import requests
2023-07-05 16:59:06 +00:00
import traceback
2022-06-07 14:42:24 +00:00
import files . helpers . offsitementions as offsitementions
2022-06-07 12:31:24 +00:00
import files . helpers . stats as stats
2022-09-22 05:15:55 +00:00
import files . routes . static as route_static
2022-11-15 09:19:08 +00:00
from files . __main__ import cache
from files . classes import *
from files . helpers . alerts import send_repeatable_notification
2022-12-11 23:44:34 +00:00
from files . helpers . config . const import *
2022-11-15 09:19:08 +00:00
from files . helpers . get import *
from files . helpers . lottery import check_if_end_lottery_task
from files . helpers . roulette import spin_roulette_wheel
from files . helpers . useractions import *
2023-03-16 06:27:58 +00:00
from files . cli import app , db_session , g
2022-07-08 16:21:39 +00:00
2023-07-07 19:08:23 +00:00
CRON_CACHE_TIMEOUT = 172800
2022-07-21 18:11:04 +00:00
@app.cli.command ( ' cron ' , help = ' Run scheduled tasks. ' )
@click.option ( ' --every-5m ' , is_flag = True , help = ' Call every 5 minutes. ' )
@click.option ( ' --every-1h ' , is_flag = True , help = ' Call every 1 hour. ' )
@click.option ( ' --every-1d ' , is_flag = True , help = ' Call every 1 day. ' )
@click.option ( ' --every-1mo ' , is_flag = True , help = ' Call every 1 month. ' )
def cron ( every_5m , every_1h , every_1d , every_1mo ) :
2023-03-16 06:27:58 +00:00
g . db = db_session ( )
2023-03-11 07:36:41 +00:00
g . v = None
2022-07-21 18:11:04 +00:00
2023-07-05 16:59:06 +00:00
try :
if every_5m :
if FEATURES [ ' GAMBLING ' ] :
check_if_end_lottery_task ( )
2023-05-13 02:00:54 +00:00
2023-07-05 16:59:06 +00:00
spin_roulette_wheel ( )
#offsitementions.offsite_mentions_task(cache)
2023-07-08 13:07:06 +00:00
_award_timers_task ( )
2022-07-21 18:11:04 +00:00
2023-07-05 16:59:06 +00:00
if every_1d :
stats . generate_charts_task ( SITE )
2023-05-13 02:00:54 +00:00
2023-07-05 16:59:06 +00:00
_sub_inactive_purge_task ( )
2023-05-13 02:00:54 +00:00
2023-07-07 19:08:23 +00:00
cache . set ( ' stats ' , stats . stats ( ) , timeout = CRON_CACHE_TIMEOUT )
2022-07-21 18:11:04 +00:00
2023-07-05 16:59:06 +00:00
_generate_emojis_zip ( )
2023-05-14 16:27:33 +00:00
2023-07-05 16:59:06 +00:00
_leaderboard_task ( )
g . db . commit ( )
except :
print ( traceback . format_exc ( ) , flush = True )
g . db . rollback ( )
2023-05-14 16:27:33 +00:00
2023-03-16 06:27:58 +00:00
g . db . close ( )
del g . db
2022-07-21 18:11:04 +00:00
stdout . flush ( )
2022-07-08 16:21:39 +00:00
2022-11-15 09:19:08 +00:00
def _sub_inactive_purge_task ( ) :
2022-07-08 16:21:39 +00:00
if not HOLE_INACTIVITY_DELETION :
return False
one_week_ago = time . time ( ) - 604800
2023-06-07 23:26:32 +00:00
active_holes = [ x [ 0 ] for x in g . db . query ( Post . sub ) . distinct ( ) \
. filter ( Post . sub != None , Post . created_utc > one_week_ago ,
Post . private == False , Post . is_banned == False ,
Post . deleted_utc == 0 ) . all ( ) ]
2023-04-23 02:53:12 +00:00
active_holes . extend ( [ ' changelog ' , ' countryclub ' , ' museumofrdrama ' ] ) # holes immune from deletion
2022-07-08 16:21:39 +00:00
2023-03-16 06:27:58 +00:00
dead_holes = g . db . query ( Sub ) . filter ( Sub . name . notin_ ( active_holes ) ) . all ( )
2022-07-08 16:21:39 +00:00
names = [ x . name for x in dead_holes ]
2023-03-16 06:27:58 +00:00
admins = [ x [ 0 ] for x in g . db . query ( User . id ) . filter ( User . admin_level > = PERMS [ ' NOTIFICATIONS_HOLE_INACTIVITY_DELETION ' ] ) . all ( ) ]
2022-08-14 06:12:34 +00:00
2023-03-16 06:27:58 +00:00
mods = g . db . query ( Mod ) . filter ( Mod . sub . in_ ( names ) ) . all ( )
2022-07-08 16:21:39 +00:00
for x in mods :
2022-08-14 06:12:34 +00:00
if x . user_id in admins : continue
2022-07-08 16:21:39 +00:00
send_repeatable_notification ( x . user_id , f " :marseyrave: /h/ { x . sub } has been deleted for inactivity after one week without new posts. All posts in it have been moved to the main feed :marseyrave: " )
for name in names :
2023-03-16 06:27:58 +00:00
first_mod_id = g . db . query ( Mod . user_id ) . filter_by ( sub = name ) . order_by ( Mod . created_utc ) . first ( )
2022-08-24 20:30:31 +00:00
if first_mod_id :
2022-08-24 20:38:44 +00:00
first_mod = get_account ( first_mod_id [ 0 ] )
2022-08-24 20:30:31 +00:00
badge_grant (
user = first_mod ,
badge_id = 156 ,
2022-12-24 18:55:26 +00:00
description = f ' Let a hole they owned die (/h/ { name } ) '
2022-08-24 20:30:31 +00:00
)
2022-08-24 20:26:41 +00:00
2022-07-08 16:21:39 +00:00
for admin in admins :
send_repeatable_notification ( admin , f " :marseyrave: /h/ { name } has been deleted for inactivity after one week without new posts. All posts in it have been moved to the main feed :marseyrave: " )
2023-06-07 23:26:32 +00:00
posts = g . db . query ( Post ) . filter ( Post . sub . in_ ( names ) ) . all ( )
2022-07-08 16:21:39 +00:00
for post in posts :
2022-12-10 14:00:58 +00:00
if post . sub == ' programming ' :
post . sub = ' slackernews '
else :
post . sub = None
2022-11-18 19:58:12 +00:00
post . hole_pinned = None
2023-03-16 06:27:58 +00:00
g . db . add ( post )
2022-07-08 16:21:39 +00:00
to_delete = mods \
2023-03-16 06:27:58 +00:00
+ g . db . query ( Exile ) . filter ( Exile . sub . in_ ( names ) ) . all ( ) \
+ g . db . query ( SubBlock ) . filter ( SubBlock . sub . in_ ( names ) ) . all ( ) \
+ g . db . query ( SubJoin ) . filter ( SubJoin . sub . in_ ( names ) ) . all ( ) \
+ g . db . query ( SubSubscription ) . filter ( SubSubscription . sub . in_ ( names ) ) . all ( ) \
+ g . db . query ( SubAction ) . filter ( SubAction . sub . in_ ( names ) ) . all ( )
2022-07-08 16:21:39 +00:00
for x in to_delete :
2023-03-16 06:27:58 +00:00
g . db . delete ( x )
g . db . flush ( )
2022-07-08 16:21:39 +00:00
for x in dead_holes :
2023-03-16 06:27:58 +00:00
g . db . delete ( x )
2022-07-08 16:21:39 +00:00
return True
2023-02-24 23:14:18 +00:00
def _generate_emojis_zip ( ) :
make_archive ( ' files/assets/emojis ' , ' zip ' , ' files/assets/images/emojis ' )
m = md5 ( )
with open ( ' files/assets/emojis.zip ' , " rb " ) as f :
data = f . read ( )
2023-05-05 21:45:25 +00:00
2023-02-24 23:14:18 +00:00
m . update ( data )
2023-07-07 19:08:23 +00:00
cache . set ( ' emojis_hash ' , m . hexdigest ( ) , timeout = CRON_CACHE_TIMEOUT )
2023-02-25 19:18:40 +00:00
count = str ( len ( os . listdir ( ' files/assets/images/emojis ' ) ) )
2023-07-07 19:08:23 +00:00
cache . set ( ' emojis_count ' , count , timeout = CRON_CACHE_TIMEOUT )
2023-02-25 19:18:40 +00:00
size = str ( int ( os . stat ( ' files/assets/emojis.zip ' ) . st_size / 1024 / 1024 ) ) + ' MB '
2023-07-07 19:08:23 +00:00
cache . set ( ' emojis_size ' , size , timeout = CRON_CACHE_TIMEOUT )
2023-04-25 09:31:02 +00:00
2023-04-25 09:50:47 +00:00
def _leaderboard_task ( ) :
2023-04-25 09:31:02 +00:00
votes1 = g . db . query ( Vote . user_id , func . count ( Vote . user_id ) ) . filter ( Vote . vote_type == 1 ) . group_by ( Vote . user_id ) . order_by ( func . count ( Vote . user_id ) . desc ( ) ) . all ( )
votes2 = g . db . query ( CommentVote . user_id , func . count ( CommentVote . user_id ) ) . filter ( CommentVote . vote_type == 1 ) . group_by ( CommentVote . user_id ) . order_by ( func . count ( CommentVote . user_id ) . desc ( ) ) . all ( )
votes3 = Counter ( dict ( votes1 ) ) + Counter ( dict ( votes2 ) )
users14 = g . db . query ( User ) . filter ( User . id . in_ ( votes3 . keys ( ) ) ) . all ( )
users13 = [ ]
for user in users14 :
users13 . append ( ( user . id , votes3 [ user . id ] - user . post_count - user . comment_count ) )
if not users13 : users13 = [ ( None , None ) ]
users13 = sorted ( users13 , key = lambda x : x [ 1 ] , reverse = True )
users13_1 , users13_2 = zip ( * users13 [ : 25 ] )
2023-07-07 19:08:23 +00:00
cache . set ( " users13 " , list ( users13 ) , timeout = CRON_CACHE_TIMEOUT )
cache . set ( " users13_1 " , list ( users13_1 ) , timeout = CRON_CACHE_TIMEOUT )
cache . set ( " users13_2 " , list ( users13_2 ) , timeout = CRON_CACHE_TIMEOUT )
2023-04-25 09:50:47 +00:00
2023-06-07 23:26:32 +00:00
votes1 = g . db . query ( Post . author_id , func . count ( Post . author_id ) ) . join ( Vote ) . filter ( Vote . vote_type == - 1 ) . group_by ( Post . author_id ) . order_by ( func . count ( Post . author_id ) . desc ( ) ) . all ( )
2023-04-25 09:50:47 +00:00
votes2 = g . db . query ( Comment . author_id , func . count ( Comment . author_id ) ) . join ( CommentVote ) . filter ( CommentVote . vote_type == - 1 ) . group_by ( Comment . author_id ) . order_by ( func . count ( Comment . author_id ) . desc ( ) ) . all ( )
votes3 = Counter ( dict ( votes1 ) ) + Counter ( dict ( votes2 ) )
users8 = g . db . query ( User . id ) . filter ( User . id . in_ ( votes3 . keys ( ) ) ) . all ( )
users9 = [ ]
for user in users8 :
users9 . append ( ( user . id , votes3 [ user . id ] ) )
if not users9 : users9 = [ ( None , None ) ]
users9 = sorted ( users9 , key = lambda x : x [ 1 ] , reverse = True )
users9_1 , users9_2 = zip ( * users9 [ : 25 ] )
2023-07-07 19:08:23 +00:00
cache . set ( " users9 " , list ( users9 ) , timeout = CRON_CACHE_TIMEOUT )
cache . set ( " users9_1 " , list ( users9_1 ) , timeout = CRON_CACHE_TIMEOUT )
cache . set ( " users9_2 " , list ( users9_2 ) , timeout = CRON_CACHE_TIMEOUT )
2023-05-13 02:00:54 +00:00
def _process_timer ( attr , badge_ids , text , extra_attrs = { } ) :
now = time . time ( )
users = g . db . query ( User ) . options ( load_only ( User . id ) ) . filter ( 1 < attr , attr < now )
uids = set ( [ x . id for x in users ] )
#set user attributes
2023-07-04 20:59:42 +00:00
attr = str ( attr ) . split ( ' . ' ) [ 1 ]
2023-07-02 21:42:34 +00:00
for user in users :
2023-07-04 20:44:28 +00:00
setattr ( user , attr , 0 )
for k , val in extra_attrs . items ( ) :
2023-07-02 21:42:34 +00:00
k = str ( k ) . split ( ' . ' ) [ 1 ]
2023-07-03 01:58:02 +00:00
if isinstance ( val , InstrumentedAttribute ) :
val = str ( val ) . split ( ' . ' ) [ 1 ]
val = getattr ( user , val )
2023-07-02 21:42:34 +00:00
setattr ( user , k , val )
2023-05-13 02:00:54 +00:00
#remove corresponding badges
if badge_ids :
g . db . query ( Badge ) . options ( load_only ( Badge . badge_id ) ) . filter ( Badge . badge_id . in_ ( badge_ids ) , Badge . user_id . in_ ( uids ) ) . delete ( )
#notify users
for uid in uids :
send_repeatable_notification ( uid , text )
def _award_timers_task ( ) :
#only awards
_process_timer ( User . deflector , [ ] , " The deflector award you received has expired! " )
_process_timer ( User . progressivestack , [ 94 ] , " The progressive stack award you received has expired! " )
_process_timer ( User . bird , [ 95 ] , " The bird site award you received has expired! " )
_process_timer ( User . longpost , [ 97 ] , " The pizzashill award you received has expired! " )
_process_timer ( User . marseyawarded , [ 98 ] , " The marsey award you received has expired! " )
_process_timer ( User . rehab , [ 109 ] , " The rehab award you received has expired! " )
_process_timer ( User . owoify , [ 167 ] , " The OwOify award you received has expired! " )
2023-06-26 14:38:48 +00:00
_process_timer ( User . sharpen , [ 289 ] , " The Sharpen award you received has expired! " )
2023-05-19 14:19:53 +00:00
_process_timer ( User . bite , [ 168 ] , " The bite award you received has expired! You ' re now back in your original house! " , {
2023-07-03 01:58:02 +00:00
User . house : User . old_house ,
2023-05-13 02:00:54 +00:00
User . old_house : ' ' ,
} )
_process_timer ( User . earlylife , [ 169 ] , " The earlylife award you received has expired! " )
_process_timer ( User . marsify , [ 170 ] , " The marsify award you received has expired! " )
_process_timer ( User . rainbow , [ 171 ] , " The rainbow award you received has expired! " )
2023-07-08 16:11:29 +00:00
_process_timer ( User . queen , [ 285 ] , " The queen award you received has expired! " )
2023-07-08 15:58:16 +00:00
_process_timer ( User . spider , [ 179 ] , " The spider award you received has expired! " )
_process_timer ( User . namechanged , [ 281 ] , " The namelock award you received has expired. You ' re now back to your old username! " , {
User . username : User . prelock_username ,
User . prelock_username : None ,
} )
2023-05-13 02:00:54 +00:00
#both awards and janny powers
_process_timer ( User . unban_utc , [ ] , " Your temporary ban has expired! " , {
User . is_banned : None ,
User . ban_reason : None ,
} )
_process_timer ( User . patron_utc , [ 22 , 23 , 24 , 25 , 26 , 27 , 28 ] , f " Your { patron } status has expired! " , {
User . patron : 0 ,
} )
2023-06-23 11:07:47 +00:00
_process_timer ( User . chud , [ 58 ] , " Your temporary chud status has expired! " , {
User . chud_phrase : None ,
2023-05-13 02:00:54 +00:00
User . chudded_by : None ,
} )
_process_timer ( User . flairchanged , [ 96 ] , " Your temporary flair-lock has expired. You can now change your flair! " )