2022-05-07 06:04:14 +00:00
import os
2022-11-15 09:19:08 +00:00
import subprocess
2022-05-22 16:13:19 +00:00
import time
2023-01-23 09:58:38 +00:00
import requests
2022-11-15 09:19:08 +00:00
from shutil import copyfile
from typing import Optional
2022-06-19 14:49:44 +00:00
import gevent
2022-09-27 01:24:20 +00:00
import imagehash
2023-01-23 09:58:38 +00:00
from flask import abort , g , has_request_context , request
2022-11-11 04:17:25 +00:00
from werkzeug . utils import secure_filename
2022-11-15 09:19:08 +00:00
from PIL import Image
from PIL import UnidentifiedImageError
from PIL . ImageSequence import Iterator
2022-11-15 11:24:17 +00:00
from sqlalchemy . orm . session import Session
2022-11-15 09:19:08 +00:00
2022-10-06 05:54:04 +00:00
from files . classes . media import *
2022-10-13 07:27:56 +00:00
from files . helpers . cloudflare import purge_files_in_cache
2023-01-23 09:58:38 +00:00
from files . helpers . settings import get_setting
2022-05-22 22:15:29 +00:00
2022-12-11 23:44:34 +00:00
from . config . const import *
2022-11-15 09:19:08 +00:00
2023-01-25 15:13:24 +00:00
def media_ratelimit ( v ) :
t = time . time ( ) - 86400
count = g . db . query ( Media ) . filter ( Media . user_id == v . id , Media . created_utc > t ) . count ( )
2023-02-27 16:25:58 +00:00
if count > 50 :
abort ( 500 )
print ( f ' \n \n ★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★ \n \n @ { v . username } hit the 50 files daily limit! \n \n ★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★ \n \n ' , flush = True )
2023-01-25 15:13:24 +00:00
2023-02-26 12:08:37 +00:00
def process_files ( files , v , body ) :
2022-11-15 09:19:08 +00:00
if g . is_tor or not files . get ( " file " ) : return body
2023-02-27 16:05:28 +00:00
files = files . getlist ( ' file ' ) [ : 20 ]
2023-02-22 17:27:33 +00:00
2023-01-25 15:13:24 +00:00
if files :
media_ratelimit ( v )
2023-02-26 12:46:38 +00:00
2022-11-15 09:19:08 +00:00
for file in files :
2023-02-27 15:02:35 +00:00
if f ' [ { file . filename } ] ' not in body :
body + = f ' \n [ { file . filename } ] '
2022-11-15 09:19:08 +00:00
if file . content_type . startswith ( ' image/ ' ) :
name = f ' /images/ { time . time ( ) } ' . replace ( ' . ' , ' ' ) + ' .webp '
file . save ( name )
url = process_image ( name , v )
2023-02-27 15:02:35 +00:00
body = body . replace ( f ' [ { file . filename } ] ' , f " ![]( { url } ) " , 1 )
2022-11-15 09:19:08 +00:00
elif file . content_type . startswith ( ' video/ ' ) :
2023-02-27 15:02:35 +00:00
body = body . replace ( f ' [ { file . filename } ] ' , f " { SITE_FULL } { process_video ( file , v ) } " , 1 )
2022-11-15 09:19:08 +00:00
elif file . content_type . startswith ( ' audio/ ' ) :
2023-02-27 15:02:35 +00:00
body = body . replace ( f ' [ { file . filename } ] ' , f " { SITE_FULL } { process_audio ( file , v ) } " , 1 )
2022-11-15 09:19:08 +00:00
else :
abort ( 415 )
2022-06-18 15:53:34 +00:00
return body
2022-11-15 09:19:08 +00:00
def process_audio ( file , v ) :
2022-06-22 17:44:43 +00:00
name = f ' /audio/ { time . time ( ) } ' . replace ( ' . ' , ' ' )
2022-11-11 04:17:25 +00:00
name_original = secure_filename ( file . filename )
extension = name_original . split ( ' . ' ) [ - 1 ] . lower ( )
2022-06-22 17:44:43 +00:00
name = name + ' . ' + extension
2022-05-22 22:15:29 +00:00
file . save ( name )
2022-05-23 18:03:59 +00:00
2022-10-06 04:31:08 +00:00
size = os . stat ( name ) . st_size
2022-11-15 09:19:08 +00:00
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not v . patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024 :
2022-10-06 04:31:08 +00:00
os . remove ( name )
2022-10-12 10:45:45 +00:00
abort ( 413 , f " Max image/audio size is { MAX_IMAGE_AUDIO_SIZE_MB } MB ( { MAX_IMAGE_AUDIO_SIZE_MB_PATRON } MB for { patron . lower ( ) } s) " )
2022-05-23 18:03:59 +00:00
2022-10-11 17:26:38 +00:00
media = g . db . query ( Media ) . filter_by ( filename = name , kind = ' audio ' ) . one_or_none ( )
if media : g . db . delete ( media )
2023-02-22 17:27:33 +00:00
2022-10-06 05:54:04 +00:00
media = Media (
kind = ' audio ' ,
2022-10-11 17:20:15 +00:00
filename = name ,
2022-11-15 09:19:08 +00:00
user_id = v . id ,
2022-10-06 05:54:04 +00:00
size = size
)
g . db . add ( media )
2022-10-18 11:09:53 +00:00
return name
2022-05-22 22:15:29 +00:00
2022-11-15 09:19:08 +00:00
def webm_to_mp4 ( old , new , vid , db ) :
2022-10-06 05:16:24 +00:00
tmp = new . replace ( ' .mp4 ' , ' -t.mp4 ' )
2022-10-06 19:17:36 +00:00
subprocess . run ( [ " ffmpeg " , " -y " , " -loglevel " , " warning " , " -nostats " , " -threads:v " , " 1 " , " -i " , old , " -map_metadata " , " -1 " , tmp ] , check = True , stderr = subprocess . STDOUT )
2022-10-06 05:07:24 +00:00
os . replace ( tmp , new )
2022-10-06 04:54:05 +00:00
os . remove ( old )
2022-10-11 17:26:38 +00:00
2022-10-11 17:28:08 +00:00
media = db . query ( Media ) . filter_by ( filename = new , kind = ' video ' ) . one_or_none ( )
2022-10-11 17:26:38 +00:00
if media : db . delete ( media )
2022-10-06 06:00:41 +00:00
media = Media (
kind = ' video ' ,
2022-10-11 17:20:15 +00:00
filename = new ,
2022-10-06 06:00:41 +00:00
user_id = vid ,
size = os . stat ( new ) . st_size
)
db . add ( media )
db . commit ( )
db . close ( )
2022-11-22 15:49:15 +00:00
purge_files_in_cache ( f " { SITE_FULL } { new } " )
2022-10-06 06:00:41 +00:00
2022-11-15 09:19:08 +00:00
def process_video ( file , v ) :
2022-05-24 20:07:04 +00:00
old = f ' /videos/ { time . time ( ) } ' . replace ( ' . ' , ' ' )
2022-06-19 13:03:14 +00:00
file . save ( old )
2022-06-22 17:44:43 +00:00
2022-10-06 04:31:08 +00:00
size = os . stat ( old ) . st_size
2022-10-19 02:15:57 +00:00
if ( SITE_NAME != ' WPD ' and
( size > MAX_VIDEO_SIZE_MB_PATRON * 1024 * 1024
2022-11-15 09:19:08 +00:00
or not v . patron and size > MAX_VIDEO_SIZE_MB * 1024 * 1024 ) ) :
2022-10-06 04:31:08 +00:00
os . remove ( old )
2022-10-12 08:53:23 +00:00
abort ( 413 , f " Max video size is { MAX_VIDEO_SIZE_MB } MB ( { MAX_VIDEO_SIZE_MB_PATRON } MB for paypigs) " )
2022-10-06 04:31:08 +00:00
2022-11-11 04:17:25 +00:00
name_original = secure_filename ( file . filename )
extension = name_original . split ( ' . ' ) [ - 1 ] . lower ( )
2022-06-22 17:44:43 +00:00
new = old + ' . ' + extension
2022-05-24 20:07:04 +00:00
2022-10-06 04:54:05 +00:00
if extension == ' webm ' :
new = new . replace ( ' .webm ' , ' .mp4 ' )
2022-10-06 05:23:05 +00:00
copyfile ( old , new )
2022-11-15 11:24:17 +00:00
db = Session ( bind = g . db . get_bind ( ) , autoflush = False )
2022-11-15 09:19:08 +00:00
gevent . spawn ( webm_to_mp4 , old , new , v . id , db )
2022-10-06 04:54:05 +00:00
else :
2022-10-06 19:17:36 +00:00
subprocess . run ( [ " ffmpeg " , " -y " , " -loglevel " , " warning " , " -nostats " , " -i " , old , " -map_metadata " , " -1 " , " -c:v " , " copy " , " -c:a " , " copy " , new ] , check = True )
2022-10-06 04:54:05 +00:00
os . remove ( old )
2022-10-11 17:28:08 +00:00
media = g . db . query ( Media ) . filter_by ( filename = new , kind = ' video ' ) . one_or_none ( )
2022-10-11 17:26:38 +00:00
if media : g . db . delete ( media )
2022-10-06 06:00:41 +00:00
media = Media (
kind = ' video ' ,
2022-10-11 17:20:15 +00:00
filename = new ,
2022-11-15 09:19:08 +00:00
user_id = v . id ,
2022-10-06 06:00:41 +00:00
size = os . stat ( new ) . st_size
)
g . db . add ( media )
2022-10-06 05:54:04 +00:00
2022-10-18 11:09:53 +00:00
return new
2022-05-23 19:00:14 +00:00
2022-11-15 09:19:08 +00:00
def process_image ( filename : str , v , resize = 0 , trim = False , uploader_id : Optional [ int ] = None , db = None ) :
# thumbnails are processed in a thread and not in the request context
# if an image is too large or webp conversion fails, it'll crash
# to avoid this, we'll simply return None instead
has_request = has_request_context ( )
2022-05-07 06:04:14 +00:00
size = os . stat ( filename ) . st_size
2022-11-15 09:19:08 +00:00
patron = bool ( v . patron )
2022-05-07 06:04:14 +00:00
2022-10-12 08:52:08 +00:00
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024 :
2022-09-10 00:38:06 +00:00
os . remove ( filename )
2022-11-15 09:19:08 +00:00
if has_request :
abort ( 413 , f " Max image/audio size is { MAX_IMAGE_AUDIO_SIZE_MB } MB ( { MAX_IMAGE_AUDIO_SIZE_MB_PATRON } MB for paypigs) " )
return None
2022-05-04 23:09:46 +00:00
2022-11-15 09:19:08 +00:00
try :
with Image . open ( filename ) as i :
2022-12-26 02:16:46 +00:00
params = [ " magick " ]
if resize == 99 : params . append ( f " { filename } [0] " )
else : params . append ( filename )
params . extend ( [ " -coalesce " , " -quality " , " 88 " , " -strip " , " -auto-orient " ] )
2022-11-15 09:19:08 +00:00
if trim and len ( list ( Iterator ( i ) ) ) == 1 :
params . append ( " -trim " )
if resize and i . width > resize :
params . extend ( [ " -resize " , f " { resize } > " ] )
except UnidentifiedImageError as e :
print ( f " Couldn ' t identify an image for { filename } ; deleting... (user { v . id if v else ' -no user- ' } ) " )
try :
os . remove ( filename )
except : pass
if has_request :
abort ( 415 )
return None
2022-10-25 15:41:18 +00:00
2022-12-21 17:05:45 +00:00
params . append ( filename )
2022-12-21 16:59:16 +00:00
try :
subprocess . run ( params , timeout = MAX_IMAGE_CONVERSION_TIMEOUT )
except subprocess . TimeoutExpired :
if has_request :
abort ( 413 , ( " An uploaded image took too long to convert to WEBP. "
" Please convert it to WEBP elsewhere then upload it again. " ) )
return None
2022-09-27 01:24:20 +00:00
2022-10-22 15:24:07 +00:00
if resize :
2022-12-07 20:53:52 +00:00
if os . stat ( filename ) . st_size > MAX_IMAGE_SIZE_BANNER_RESIZED_MB * 1024 * 1024 :
2022-09-27 01:24:20 +00:00
os . remove ( filename )
2022-11-15 09:19:08 +00:00
if has_request :
2022-12-07 20:53:52 +00:00
abort ( 413 , f " Max size for site assets is { MAX_IMAGE_SIZE_BANNER_RESIZED_MB } MB " )
2022-11-15 09:19:08 +00:00
return None
2022-09-27 01:24:20 +00:00
2022-10-22 15:24:07 +00:00
if filename . startswith ( ' files/assets/images/ ' ) :
path = filename . rsplit ( ' / ' , 1 ) [ 0 ]
2022-10-23 09:27:40 +00:00
kind = path . split ( ' / ' ) [ - 1 ]
2022-12-23 22:22:41 +00:00
if kind in { ' banners ' , ' sidebar ' } :
2022-10-23 09:27:40 +00:00
hashes = { }
for img in os . listdir ( path ) :
2022-12-23 22:22:41 +00:00
if resize == 400 and img in { ' 256.webp ' , ' 585.webp ' } : continue
2022-10-23 09:27:40 +00:00
img_path = f ' { path } / { img } '
if img_path == filename : continue
2022-10-25 15:39:57 +00:00
with Image . open ( img_path ) as i :
i_hash = str ( imagehash . phash ( i ) )
2022-10-23 09:27:40 +00:00
if i_hash in hashes . keys ( ) :
2023-03-05 10:11:50 +00:00
print ( f ' \n \n ★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★ \n \n Remove one of these existing duplicates please: \n \n ' , flush = True )
2022-10-23 09:27:40 +00:00
print ( hashes [ i_hash ] , flush = True )
print ( img_path , flush = True )
2023-03-05 10:11:50 +00:00
print ( f ' \n \n ★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★★ \n \n ' , flush = True )
2022-10-23 09:27:40 +00:00
else : hashes [ i_hash ] = img_path
2022-10-25 15:39:57 +00:00
with Image . open ( filename ) as i :
i_hash = str ( imagehash . phash ( i ) )
2022-10-22 15:24:07 +00:00
if i_hash in hashes . keys ( ) :
2022-10-23 09:27:40 +00:00
os . remove ( filename )
2022-11-15 09:19:08 +00:00
if has_request :
abort ( 409 , " Image already exists! " )
return None
2022-09-26 01:27:31 +00:00
2022-10-11 17:26:38 +00:00
db = db or g . db
2022-10-11 17:28:08 +00:00
media = db . query ( Media ) . filter_by ( filename = filename , kind = ' image ' ) . one_or_none ( )
2022-10-11 17:26:38 +00:00
if media : db . delete ( media )
2022-10-06 05:54:04 +00:00
media = Media (
kind = ' image ' ,
2022-10-11 17:20:15 +00:00
filename = filename ,
2022-11-15 09:19:08 +00:00
user_id = uploader_id or v . id ,
2022-10-06 05:54:04 +00:00
size = os . stat ( filename ) . st_size
)
2022-10-06 19:07:45 +00:00
db . add ( media )
2022-10-06 05:54:04 +00:00
2022-09-29 05:43:29 +00:00
return filename
2023-01-23 09:58:38 +00:00
2023-02-26 12:08:37 +00:00
def process_dm_images ( v , user , body ) :
2023-01-23 09:58:38 +00:00
if not request . files . get ( " file " ) or g . is_tor or not get_setting ( " dm_images " ) :
2023-02-26 12:54:36 +00:00
return body
2023-01-23 09:58:38 +00:00
2023-02-27 16:05:28 +00:00
files = request . files . getlist ( ' file ' ) [ : 20 ]
2023-02-26 12:08:37 +00:00
2023-02-26 12:46:38 +00:00
for file in files :
2023-02-27 15:02:35 +00:00
if f ' [ { file . filename } ] ' not in body :
body + = f ' \n [ { file . filename } ] '
2023-01-23 09:58:38 +00:00
if file . content_type . startswith ( ' image/ ' ) :
filename = f ' /dm_images/ { time . time ( ) } ' . replace ( ' . ' , ' ' ) + ' .webp '
file . save ( filename )
size = os . stat ( filename ) . st_size
patron = bool ( v . patron )
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024 :
os . remove ( filename )
abort ( 413 , f " Max image/audio size is { MAX_IMAGE_AUDIO_SIZE_MB } MB ( { MAX_IMAGE_AUDIO_SIZE_MB_PATRON } MB for paypigs) " )
with open ( filename , ' rb ' ) as f :
os . remove ( filename )
try :
req = requests . request (
" POST " ,
" https://pomf2.lain.la/upload.php " ,
files = { ' files[] ' : f } ,
timeout = 20 ,
proxies = proxies
) . json ( )
except requests . Timeout :
abort ( 400 , " Image upload timed out, please try again! " )
2023-02-22 17:27:33 +00:00
2023-01-23 09:58:38 +00:00
try : url = req [ ' files ' ] [ 0 ] [ ' url ' ]
except : abort ( 400 , req [ ' description ' ] )
2023-02-22 17:27:33 +00:00
2023-02-27 15:02:35 +00:00
body = body . replace ( f ' [ { file . filename } ] ' , url , 1 )
2023-02-26 12:08:37 +00:00
with open ( f " { LOG_DIRECTORY } /dm_images.log " , " a+ " , encoding = " utf-8 " ) as f :
if user :
f . write ( f ' { url } , { v . username } , { v . id } , { user . username } , { user . id } , { int ( time . time ( ) ) } \n ' )
else :
f . write ( f ' { url } , { v . username } , { v . id } , Modmail, Modmail, { int ( time . time ( ) ) } \n ' )
2023-02-22 17:27:33 +00:00
2023-02-26 12:08:37 +00:00
return body . strip ( )