347 lines
10 KiB
Python
347 lines
10 KiB
Python
import os
|
|
import subprocess
|
|
import time
|
|
from shutil import copyfile
|
|
import json
|
|
|
|
import requests
|
|
import ffmpeg
|
|
import gevent
|
|
import imagehash
|
|
from flask import abort, g, has_request_context, request
|
|
from mimetypes import guess_extension
|
|
from PIL import Image
|
|
from PIL import UnidentifiedImageError
|
|
from PIL.ImageSequence import Iterator
|
|
|
|
from files.classes.media import *
|
|
from files.classes.badges import BadgeDef
|
|
from files.helpers.cloudflare import purge_files_in_cloudflare_cache
|
|
from files.helpers.settings import get_setting
|
|
|
|
from .config.const import *
|
|
from .regex import badge_name_regex
|
|
|
|
if SITE == 'watchpeopledie.tv':
|
|
from rclone_python import rclone
|
|
|
|
def remove_media_using_link(path):
|
|
if SITE in path:
|
|
path = path.split(SITE, 1)[1]
|
|
os.remove(path)
|
|
|
|
def media_ratelimit(v):
|
|
if v.id in {15014,1718156}: # Marseygen exception
|
|
return
|
|
t = time.time() - 86400
|
|
count = g.db.query(Media).filter(Media.user_id == v.id, Media.created_utc > t).count()
|
|
|
|
if v.patron or (SITE == 'rdrama.net' and v.id == 2158):
|
|
limit = 300
|
|
else:
|
|
limit = 100
|
|
|
|
if count > limit and v.admin_level < PERMS['USE_ADMIGGER_THREADS']:
|
|
print(STARS, flush=True)
|
|
print(f'@{v.username} hit the {limit} file daily limit!')
|
|
print(STARS, flush=True)
|
|
abort(500)
|
|
|
|
def process_files(files, v, body, is_dm=False, dm_user=None, admigger_thread=None, comment_body=None):
|
|
if g.is_tor or not files.get("file"): return body
|
|
files = files.getlist('file')[:20]
|
|
|
|
if files:
|
|
media_ratelimit(v)
|
|
|
|
|
|
for file in files:
|
|
if f'[{file.filename}]' not in body:
|
|
continue
|
|
|
|
if file.content_type.startswith('image/'):
|
|
name = f'/images/{time.time()}'.replace('.','') + '.webp'
|
|
file.save(name)
|
|
url = process_image(name, v)
|
|
if admigger_thread:
|
|
process_admigger_entry(name, v, admigger_thread, comment_body)
|
|
elif file.content_type.startswith('video/'):
|
|
url = process_video(file, v)
|
|
elif file.content_type.startswith('audio/'):
|
|
url = f'{SITE_FULL}{process_audio(file, v)}'
|
|
elif has_request_context():
|
|
abort(415)
|
|
else:
|
|
return None
|
|
|
|
body = body.replace(f'[{file.filename}]', f' {url} ', 1)
|
|
|
|
if is_dm:
|
|
with open(f"{LOG_DIRECTORY}/dm_media.log", "a+") as f:
|
|
if dm_user:
|
|
f.write(f'{url}, {v.username}, {v.id}, {dm_user.username}, {dm_user.id}, {int(time.time())}\n')
|
|
else:
|
|
f.write(f'{url}, {v.username}, {v.id}, Modmail, Modmail, {int(time.time())}\n')
|
|
|
|
return body.replace('\n ', '\n').strip()
|
|
|
|
|
|
def process_audio(file, v, old=None):
|
|
if not old:
|
|
old = f'/audio/{time.time()}'.replace('.','')
|
|
|
|
file.save(old)
|
|
|
|
size = os.stat(old).st_size
|
|
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not v.patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024:
|
|
os.remove(old)
|
|
abort(413, f"Max image/audio size is {MAX_IMAGE_AUDIO_SIZE_MB} MB ({MAX_IMAGE_AUDIO_SIZE_MB_PATRON} MB for {patron}s)")
|
|
|
|
extension = guess_extension(file.content_type)
|
|
if not extension:
|
|
os.remove(old)
|
|
abort(400, "Unsupported audio format.")
|
|
new = old + extension
|
|
|
|
try:
|
|
ffmpeg.input(old).output(new, loglevel="quiet", map_metadata=-1).run()
|
|
except:
|
|
os.remove(old)
|
|
if os.path.isfile(new):
|
|
os.remove(new)
|
|
abort(400, "Something went wrong processing your audio on our end. Please try uploading it to https://pomf2.lain.la and post the link instead.")
|
|
|
|
os.remove(old)
|
|
|
|
media = Media(
|
|
kind='audio',
|
|
filename=new,
|
|
user_id=v.id,
|
|
size=size
|
|
)
|
|
g.db.add(media)
|
|
|
|
return new
|
|
|
|
|
|
def reencode_video(old, new, check_sizes=False):
|
|
tmp = new.replace('.mp4', '-t.mp4')
|
|
try:
|
|
ffmpeg.input(old).output(tmp, loglevel="quiet", map_metadata=-1).run()
|
|
except:
|
|
os.remove(old)
|
|
if os.path.isfile(tmp):
|
|
os.remove(tmp)
|
|
return
|
|
|
|
if check_sizes:
|
|
old_size = os.stat(old).st_size
|
|
new_size = os.stat(tmp).st_size
|
|
if new_size > old_size:
|
|
os.remove(tmp)
|
|
return
|
|
|
|
os.replace(tmp, new)
|
|
os.remove(old)
|
|
|
|
if SITE == 'watchpeopledie.tv':
|
|
url = f'https://videos.{SITE}' + new.split('/videos')[1]
|
|
else:
|
|
url = f"{SITE_FULL}{new}"
|
|
|
|
purge_files_in_cloudflare_cache(url)
|
|
|
|
|
|
|
|
def process_video(file, v):
|
|
old = f'/videos/{time.time()}'.replace('.','')
|
|
file.save(old)
|
|
|
|
size = os.stat(old).st_size
|
|
if size > MAX_VIDEO_SIZE_MB_PATRON * 1024 * 1024 or (not v.patron and size > MAX_VIDEO_SIZE_MB * 1024 * 1024):
|
|
os.remove(old)
|
|
abort(413, f"Max video size is {MAX_VIDEO_SIZE_MB} MB ({MAX_VIDEO_SIZE_MB_PATRON} MB for {patron}s)")
|
|
|
|
new = f'{old}.mp4'
|
|
|
|
try:
|
|
video_info = ffmpeg.probe(old)['streams'][0]
|
|
codec = video_info['codec_name']
|
|
bitrate = int(video_info.get('bit_rate', 3000000))
|
|
except:
|
|
os.remove(old)
|
|
abort(400, "Something went wrong processing your video on our end. Please try uploading it to https://pomf2.lain.la and post the link instead.")
|
|
|
|
if codec != 'h264':
|
|
copyfile(old, new)
|
|
gevent.spawn(reencode_video, old, new)
|
|
elif bitrate >= 3000000:
|
|
copyfile(old, new)
|
|
gevent.spawn(reencode_video, old, new, True)
|
|
else:
|
|
try:
|
|
ffmpeg.input(old).output(new, loglevel="quiet", map_metadata=-1, acodec="copy", vcodec="copy").run()
|
|
except:
|
|
os.remove(old)
|
|
if os.path.isfile(new):
|
|
os.remove(new)
|
|
abort(400, "Something went wrong processing your video on our end. Please try uploading it to https://pomf2.lain.la and post the link instead.")
|
|
|
|
os.remove(old)
|
|
|
|
media = Media(
|
|
kind='video',
|
|
filename=new,
|
|
user_id=v.id,
|
|
size=os.stat(new).st_size
|
|
)
|
|
g.db.add(media)
|
|
|
|
if SITE == 'watchpeopledie.tv' and v and v.username.lower().startswith("icosaka"):
|
|
gevent.spawn(delete_file, new, f'https://videos.{SITE}' + new.split('/videos')[1])
|
|
|
|
if SITE == 'watchpeopledie.tv':
|
|
gevent.spawn(send_file, new)
|
|
return f'https://videos.{SITE}' + new.split('/videos')[1]
|
|
else:
|
|
return f"{SITE_FULL}{new}"
|
|
|
|
def process_image(filename, v, resize=0, trim=False, uploader_id=None):
|
|
# thumbnails are processed in a thread and not in the request context
|
|
# if an image is too large or webp conversion fails, it'll crash
|
|
# to avoid this, we'll simply return None instead
|
|
original_resize = resize
|
|
has_request = has_request_context()
|
|
size = os.stat(filename).st_size
|
|
if v and v.patron:
|
|
max_size = MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024
|
|
else:
|
|
max_size = MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024
|
|
|
|
try:
|
|
with Image.open(filename) as i:
|
|
if not resize and size > max_size:
|
|
ratio = max_size / size
|
|
resize = i.width * ratio
|
|
|
|
oldformat = i.format
|
|
params = ["magick"]
|
|
if resize == 99: params.append(f"{filename}[0]")
|
|
else: params.append(filename)
|
|
params.extend(["-coalesce", "-quality", "88", "-strip", "-auto-orient"])
|
|
if trim and len(list(Iterator(i))) == 1:
|
|
params.append("-trim")
|
|
if resize and i.width > resize:
|
|
params.extend(["-resize", f"{resize}>"])
|
|
except:
|
|
os.remove(filename)
|
|
if has_request and not filename.startswith('/chat_images/'):
|
|
abort(400, "Something went wrong processing your image on our end. Please try uploading it to https://pomf2.lain.la and post the link instead.")
|
|
return None
|
|
|
|
params.append(filename)
|
|
try:
|
|
subprocess.run(params, check=True, timeout=30)
|
|
except:
|
|
os.remove(filename)
|
|
if has_request:
|
|
abort(400, "An uploaded image couldn't be converted to WEBP. Please convert it to WEBP elsewhere then upload it again.")
|
|
return None
|
|
|
|
size_after_conversion = os.stat(filename).st_size
|
|
|
|
if original_resize:
|
|
if size_after_conversion > MAX_IMAGE_SIZE_BANNER_RESIZED_MB * 1024 * 1024:
|
|
os.remove(filename)
|
|
if has_request:
|
|
abort(413, f"Max size for site assets is {MAX_IMAGE_SIZE_BANNER_RESIZED_MB} MB")
|
|
return None
|
|
|
|
if filename.startswith('files/assets/images/'):
|
|
path = filename.rsplit('/', 1)[0]
|
|
kind = path.split('/')[-1]
|
|
|
|
if kind in {'banners','sidebar'}:
|
|
hashes = {}
|
|
|
|
for img in os.listdir(path):
|
|
img_path = f'{path}/{img}'
|
|
if img_path == filename: continue
|
|
|
|
with Image.open(img_path) as i:
|
|
i_hash = str(imagehash.phash(i))
|
|
|
|
if i_hash not in hashes.keys():
|
|
hashes[i_hash] = img_path
|
|
|
|
with Image.open(filename) as i:
|
|
i_hash = str(imagehash.phash(i))
|
|
|
|
if i_hash in hashes.keys():
|
|
os.remove(filename)
|
|
return None
|
|
|
|
media = g.db.query(Media).filter_by(filename=filename, kind='image').one_or_none()
|
|
if media: g.db.delete(media)
|
|
|
|
media = Media(
|
|
kind='image',
|
|
filename=filename,
|
|
user_id=uploader_id or v.id,
|
|
size=os.stat(filename).st_size
|
|
)
|
|
g.db.add(media)
|
|
|
|
if SITE == 'watchpeopledie.tv' and v and "dylan" in v.username.lower() and "hewitt" in v.username.lower():
|
|
gevent.spawn(delete_file, filename, f'{SITE_FULL_IMAGES}{filename}')
|
|
|
|
return f'{SITE_FULL_IMAGES}{filename}'
|
|
|
|
def delete_file(filename, url):
|
|
time.sleep(60)
|
|
os.remove(filename)
|
|
purge_files_in_cloudflare_cache(url)
|
|
|
|
def send_file(filename):
|
|
rclone.copy(filename, 'no:/videos', ignore_existing=True, show_progress=False)
|
|
|
|
|
|
def process_sidebar_or_banner(oldname, v, type, resize):
|
|
li = sorted(os.listdir(f'files/assets/images/{SITE_NAME}/{type}'),
|
|
key=lambda e: int(e.split('.webp')[0]))[-1]
|
|
num = int(li.split('.webp')[0]) + 1
|
|
filename = f'files/assets/images/{SITE_NAME}/{type}/{num}.webp'
|
|
copyfile(oldname, filename)
|
|
process_image(filename, v, resize=resize)
|
|
|
|
def process_admigger_entry(oldname, v, admigger_thread, comment_body):
|
|
if admigger_thread == SIDEBAR_THREAD:
|
|
process_sidebar_or_banner(oldname, v, 'sidebar', 600)
|
|
elif admigger_thread == BANNER_THREAD:
|
|
banner_width = 1600
|
|
process_sidebar_or_banner(oldname, v, 'banners', banner_width)
|
|
elif admigger_thread == BADGE_THREAD:
|
|
try:
|
|
json_body = '{' + comment_body.split('{')[1].split('}')[0] + '}'
|
|
badge_def = json.loads(json_body)
|
|
name = badge_def["name"]
|
|
|
|
if len(name) > 50:
|
|
abort(400, "Badge name is too long (max 50 characters)")
|
|
|
|
if not badge_name_regex.fullmatch(name):
|
|
abort(400, "Invalid badge name!")
|
|
|
|
existing = g.db.query(BadgeDef).filter_by(name=name).one_or_none()
|
|
if existing: abort(409, "A badge with this name already exists!")
|
|
|
|
badge = BadgeDef(name=name, description=badge_def["description"])
|
|
g.db.add(badge)
|
|
g.db.flush()
|
|
filename = f'files/assets/images/{SITE_NAME}/badges/{badge.id}.webp'
|
|
copyfile(oldname, filename)
|
|
process_image(filename, v, resize=300, trim=True)
|
|
purge_files_in_cloudflare_cache(f"{SITE_FULL_IMAGES}/i/{SITE_NAME}/badges/{badge.id}.webp")
|
|
except Exception as e:
|
|
abort(400, str(e))
|