2022-05-07 06:04:14 +00:00
|
|
|
import os
|
2022-11-15 09:19:08 +00:00
|
|
|
import subprocess
|
2022-05-22 16:13:19 +00:00
|
|
|
import time
|
2023-01-23 09:58:38 +00:00
|
|
|
import requests
|
2022-11-15 09:19:08 +00:00
|
|
|
from shutil import copyfile
|
|
|
|
from typing import Optional
|
|
|
|
|
2022-06-19 14:49:44 +00:00
|
|
|
import gevent
|
2022-09-27 01:24:20 +00:00
|
|
|
import imagehash
|
2023-01-23 09:58:38 +00:00
|
|
|
from flask import abort, g, has_request_context, request
|
2022-11-11 04:17:25 +00:00
|
|
|
from werkzeug.utils import secure_filename
|
2022-11-15 09:19:08 +00:00
|
|
|
from PIL import Image
|
|
|
|
from PIL import UnidentifiedImageError
|
|
|
|
from PIL.ImageSequence import Iterator
|
2022-11-15 11:24:17 +00:00
|
|
|
from sqlalchemy.orm.session import Session
|
2022-11-15 09:19:08 +00:00
|
|
|
|
2022-10-06 05:54:04 +00:00
|
|
|
from files.classes.media import *
|
2022-10-13 07:27:56 +00:00
|
|
|
from files.helpers.cloudflare import purge_files_in_cache
|
2023-01-23 09:58:38 +00:00
|
|
|
from files.helpers.settings import get_setting
|
2022-05-22 22:15:29 +00:00
|
|
|
|
2022-12-11 23:44:34 +00:00
|
|
|
from .config.const import *
|
2022-11-15 09:19:08 +00:00
|
|
|
|
2023-01-25 15:13:24 +00:00
|
|
|
def media_ratelimit(v):
|
|
|
|
t = time.time() - 86400
|
|
|
|
count = g.db.query(Media).filter(Media.user_id == v.id, Media.created_utc > t).count()
|
|
|
|
if count > 50: abort(500)
|
|
|
|
|
2023-02-26 12:08:37 +00:00
|
|
|
def process_files(files, v, body):
|
2022-11-15 09:19:08 +00:00
|
|
|
if g.is_tor or not files.get("file"): return body
|
2023-02-26 12:19:30 +00:00
|
|
|
files = files.getlist('file')[:8]
|
2023-02-22 17:27:33 +00:00
|
|
|
|
2023-01-25 15:13:24 +00:00
|
|
|
if files:
|
|
|
|
media_ratelimit(v)
|
|
|
|
|
2023-02-27 05:24:26 +00:00
|
|
|
while body.count('[file]') < len(files):
|
|
|
|
body += '\n[file]'
|
2023-02-26 12:46:38 +00:00
|
|
|
|
2022-11-15 09:19:08 +00:00
|
|
|
for file in files:
|
|
|
|
if file.content_type.startswith('image/'):
|
|
|
|
name = f'/images/{time.time()}'.replace('.','') + '.webp'
|
|
|
|
file.save(name)
|
|
|
|
url = process_image(name, v)
|
2023-02-27 05:24:26 +00:00
|
|
|
body = body.replace('[file]', f"![]({url})", 1)
|
2022-11-15 09:19:08 +00:00
|
|
|
elif file.content_type.startswith('video/'):
|
2023-02-27 05:24:26 +00:00
|
|
|
body = body.replace('[file]', f"{SITE_FULL}{process_video(file, v)}", 1)
|
2022-11-15 09:19:08 +00:00
|
|
|
elif file.content_type.startswith('audio/'):
|
2023-02-27 05:24:26 +00:00
|
|
|
body = body.replace('[file]', f"{SITE_FULL}{process_audio(file, v)}", 1)
|
2022-11-15 09:19:08 +00:00
|
|
|
else:
|
|
|
|
abort(415)
|
2022-06-18 15:53:34 +00:00
|
|
|
return body
|
|
|
|
|
|
|
|
|
2022-11-15 09:19:08 +00:00
|
|
|
def process_audio(file, v):
|
2022-06-22 17:44:43 +00:00
|
|
|
name = f'/audio/{time.time()}'.replace('.','')
|
|
|
|
|
2022-11-11 04:17:25 +00:00
|
|
|
name_original = secure_filename(file.filename)
|
|
|
|
extension = name_original.split('.')[-1].lower()
|
2022-06-22 17:44:43 +00:00
|
|
|
name = name + '.' + extension
|
2022-05-22 22:15:29 +00:00
|
|
|
file.save(name)
|
2022-05-23 18:03:59 +00:00
|
|
|
|
2022-10-06 04:31:08 +00:00
|
|
|
size = os.stat(name).st_size
|
2022-11-15 09:19:08 +00:00
|
|
|
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not v.patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024:
|
2022-10-06 04:31:08 +00:00
|
|
|
os.remove(name)
|
2022-10-12 10:45:45 +00:00
|
|
|
abort(413, f"Max image/audio size is {MAX_IMAGE_AUDIO_SIZE_MB} MB ({MAX_IMAGE_AUDIO_SIZE_MB_PATRON} MB for {patron.lower()}s)")
|
2022-05-23 18:03:59 +00:00
|
|
|
|
2022-10-11 17:26:38 +00:00
|
|
|
media = g.db.query(Media).filter_by(filename=name, kind='audio').one_or_none()
|
|
|
|
if media: g.db.delete(media)
|
2023-02-22 17:27:33 +00:00
|
|
|
|
2022-10-06 05:54:04 +00:00
|
|
|
media = Media(
|
|
|
|
kind='audio',
|
2022-10-11 17:20:15 +00:00
|
|
|
filename=name,
|
2022-11-15 09:19:08 +00:00
|
|
|
user_id=v.id,
|
2022-10-06 05:54:04 +00:00
|
|
|
size=size
|
|
|
|
)
|
|
|
|
g.db.add(media)
|
|
|
|
|
2022-10-18 11:09:53 +00:00
|
|
|
return name
|
2022-05-22 22:15:29 +00:00
|
|
|
|
|
|
|
|
2022-11-15 09:19:08 +00:00
|
|
|
def webm_to_mp4(old, new, vid, db):
|
2022-10-06 05:16:24 +00:00
|
|
|
tmp = new.replace('.mp4', '-t.mp4')
|
2022-10-06 19:17:36 +00:00
|
|
|
subprocess.run(["ffmpeg", "-y", "-loglevel", "warning", "-nostats", "-threads:v", "1", "-i", old, "-map_metadata", "-1", tmp], check=True, stderr=subprocess.STDOUT)
|
2022-10-06 05:07:24 +00:00
|
|
|
os.replace(tmp, new)
|
2022-10-06 04:54:05 +00:00
|
|
|
os.remove(old)
|
2022-10-11 17:26:38 +00:00
|
|
|
|
2022-10-11 17:28:08 +00:00
|
|
|
media = db.query(Media).filter_by(filename=new, kind='video').one_or_none()
|
2022-10-11 17:26:38 +00:00
|
|
|
if media: db.delete(media)
|
|
|
|
|
2022-10-06 06:00:41 +00:00
|
|
|
media = Media(
|
|
|
|
kind='video',
|
2022-10-11 17:20:15 +00:00
|
|
|
filename=new,
|
2022-10-06 06:00:41 +00:00
|
|
|
user_id=vid,
|
|
|
|
size=os.stat(new).st_size
|
|
|
|
)
|
|
|
|
db.add(media)
|
|
|
|
db.commit()
|
|
|
|
db.close()
|
|
|
|
|
2022-11-22 15:49:15 +00:00
|
|
|
purge_files_in_cache(f"{SITE_FULL}{new}")
|
|
|
|
|
|
|
|
|
2022-10-06 06:00:41 +00:00
|
|
|
|
2022-11-15 09:19:08 +00:00
|
|
|
def process_video(file, v):
|
2022-05-24 20:07:04 +00:00
|
|
|
old = f'/videos/{time.time()}'.replace('.','')
|
2022-06-19 13:03:14 +00:00
|
|
|
file.save(old)
|
2022-06-22 17:44:43 +00:00
|
|
|
|
2022-10-06 04:31:08 +00:00
|
|
|
size = os.stat(old).st_size
|
2022-10-19 02:15:57 +00:00
|
|
|
if (SITE_NAME != 'WPD' and
|
|
|
|
(size > MAX_VIDEO_SIZE_MB_PATRON * 1024 * 1024
|
2022-11-15 09:19:08 +00:00
|
|
|
or not v.patron and size > MAX_VIDEO_SIZE_MB * 1024 * 1024)):
|
2022-10-06 04:31:08 +00:00
|
|
|
os.remove(old)
|
2022-10-12 08:53:23 +00:00
|
|
|
abort(413, f"Max video size is {MAX_VIDEO_SIZE_MB} MB ({MAX_VIDEO_SIZE_MB_PATRON} MB for paypigs)")
|
2022-10-06 04:31:08 +00:00
|
|
|
|
2022-11-11 04:17:25 +00:00
|
|
|
name_original = secure_filename(file.filename)
|
|
|
|
extension = name_original.split('.')[-1].lower()
|
2022-06-22 17:44:43 +00:00
|
|
|
new = old + '.' + extension
|
2022-05-24 20:07:04 +00:00
|
|
|
|
2022-10-06 04:54:05 +00:00
|
|
|
if extension == 'webm':
|
|
|
|
new = new.replace('.webm', '.mp4')
|
2022-10-06 05:23:05 +00:00
|
|
|
copyfile(old, new)
|
2022-11-15 11:24:17 +00:00
|
|
|
db = Session(bind=g.db.get_bind(), autoflush=False)
|
2022-11-15 09:19:08 +00:00
|
|
|
gevent.spawn(webm_to_mp4, old, new, v.id, db)
|
2022-10-06 04:54:05 +00:00
|
|
|
else:
|
2022-10-06 19:17:36 +00:00
|
|
|
subprocess.run(["ffmpeg", "-y", "-loglevel", "warning", "-nostats", "-i", old, "-map_metadata", "-1", "-c:v", "copy", "-c:a", "copy", new], check=True)
|
2022-10-06 04:54:05 +00:00
|
|
|
os.remove(old)
|
|
|
|
|
2022-10-11 17:28:08 +00:00
|
|
|
media = g.db.query(Media).filter_by(filename=new, kind='video').one_or_none()
|
2022-10-11 17:26:38 +00:00
|
|
|
if media: g.db.delete(media)
|
|
|
|
|
2022-10-06 06:00:41 +00:00
|
|
|
media = Media(
|
|
|
|
kind='video',
|
2022-10-11 17:20:15 +00:00
|
|
|
filename=new,
|
2022-11-15 09:19:08 +00:00
|
|
|
user_id=v.id,
|
2022-10-06 06:00:41 +00:00
|
|
|
size=os.stat(new).st_size
|
|
|
|
)
|
|
|
|
g.db.add(media)
|
2022-10-06 05:54:04 +00:00
|
|
|
|
2022-10-18 11:09:53 +00:00
|
|
|
return new
|
2022-05-23 19:00:14 +00:00
|
|
|
|
2022-11-15 09:19:08 +00:00
|
|
|
def process_image(filename:str, v, resize=0, trim=False, uploader_id:Optional[int]=None, db=None):
|
|
|
|
# thumbnails are processed in a thread and not in the request context
|
|
|
|
# if an image is too large or webp conversion fails, it'll crash
|
|
|
|
# to avoid this, we'll simply return None instead
|
|
|
|
has_request = has_request_context()
|
2022-05-07 06:04:14 +00:00
|
|
|
size = os.stat(filename).st_size
|
2022-11-15 09:19:08 +00:00
|
|
|
patron = bool(v.patron)
|
2022-05-07 06:04:14 +00:00
|
|
|
|
2022-10-12 08:52:08 +00:00
|
|
|
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024:
|
2022-09-10 00:38:06 +00:00
|
|
|
os.remove(filename)
|
2022-11-15 09:19:08 +00:00
|
|
|
if has_request:
|
|
|
|
abort(413, f"Max image/audio size is {MAX_IMAGE_AUDIO_SIZE_MB} MB ({MAX_IMAGE_AUDIO_SIZE_MB_PATRON} MB for paypigs)")
|
|
|
|
return None
|
2022-05-04 23:09:46 +00:00
|
|
|
|
2022-11-15 09:19:08 +00:00
|
|
|
try:
|
|
|
|
with Image.open(filename) as i:
|
2022-12-26 02:16:46 +00:00
|
|
|
params = ["magick"]
|
|
|
|
if resize == 99: params.append(f"{filename}[0]")
|
|
|
|
else: params.append(filename)
|
|
|
|
params.extend(["-coalesce", "-quality", "88", "-strip", "-auto-orient"])
|
2022-11-15 09:19:08 +00:00
|
|
|
if trim and len(list(Iterator(i))) == 1:
|
|
|
|
params.append("-trim")
|
|
|
|
if resize and i.width > resize:
|
|
|
|
params.extend(["-resize", f"{resize}>"])
|
|
|
|
except UnidentifiedImageError as e:
|
|
|
|
print(f"Couldn't identify an image for {filename}; deleting... (user {v.id if v else '-no user-'})")
|
|
|
|
try:
|
|
|
|
os.remove(filename)
|
|
|
|
except: pass
|
|
|
|
if has_request:
|
|
|
|
abort(415)
|
|
|
|
return None
|
2022-10-25 15:41:18 +00:00
|
|
|
|
2022-12-21 17:05:45 +00:00
|
|
|
params.append(filename)
|
2022-12-21 16:59:16 +00:00
|
|
|
try:
|
|
|
|
subprocess.run(params, timeout=MAX_IMAGE_CONVERSION_TIMEOUT)
|
|
|
|
except subprocess.TimeoutExpired:
|
|
|
|
if has_request:
|
|
|
|
abort(413, ("An uploaded image took too long to convert to WEBP. "
|
|
|
|
"Please convert it to WEBP elsewhere then upload it again."))
|
|
|
|
return None
|
2022-09-27 01:24:20 +00:00
|
|
|
|
2022-10-22 15:24:07 +00:00
|
|
|
if resize:
|
2022-12-07 20:53:52 +00:00
|
|
|
if os.stat(filename).st_size > MAX_IMAGE_SIZE_BANNER_RESIZED_MB * 1024 * 1024:
|
2022-09-27 01:24:20 +00:00
|
|
|
os.remove(filename)
|
2022-11-15 09:19:08 +00:00
|
|
|
if has_request:
|
2022-12-07 20:53:52 +00:00
|
|
|
abort(413, f"Max size for site assets is {MAX_IMAGE_SIZE_BANNER_RESIZED_MB} MB")
|
2022-11-15 09:19:08 +00:00
|
|
|
return None
|
2022-09-27 01:24:20 +00:00
|
|
|
|
2022-10-22 15:24:07 +00:00
|
|
|
if filename.startswith('files/assets/images/'):
|
|
|
|
path = filename.rsplit('/', 1)[0]
|
2022-10-23 09:27:40 +00:00
|
|
|
kind = path.split('/')[-1]
|
|
|
|
|
2022-12-23 22:22:41 +00:00
|
|
|
if kind in {'banners','sidebar'}:
|
2022-10-23 09:27:40 +00:00
|
|
|
hashes = {}
|
|
|
|
|
|
|
|
for img in os.listdir(path):
|
2022-12-23 22:22:41 +00:00
|
|
|
if resize == 400 and img in {'256.webp','585.webp'}: continue
|
2022-10-23 09:27:40 +00:00
|
|
|
img_path = f'{path}/{img}'
|
|
|
|
if img_path == filename: continue
|
2022-10-25 15:39:57 +00:00
|
|
|
|
|
|
|
with Image.open(img_path) as i:
|
|
|
|
i_hash = str(imagehash.phash(i))
|
|
|
|
|
2022-10-23 09:27:40 +00:00
|
|
|
if i_hash in hashes.keys():
|
|
|
|
print(hashes[i_hash], flush=True)
|
|
|
|
print(img_path, flush=True)
|
|
|
|
else: hashes[i_hash] = img_path
|
|
|
|
|
2022-10-25 15:39:57 +00:00
|
|
|
with Image.open(filename) as i:
|
|
|
|
i_hash = str(imagehash.phash(i))
|
|
|
|
|
2022-10-22 15:24:07 +00:00
|
|
|
if i_hash in hashes.keys():
|
2022-10-23 09:27:40 +00:00
|
|
|
os.remove(filename)
|
2022-11-15 09:19:08 +00:00
|
|
|
if has_request:
|
|
|
|
abort(409, "Image already exists!")
|
|
|
|
return None
|
2022-09-26 01:27:31 +00:00
|
|
|
|
2022-10-11 17:26:38 +00:00
|
|
|
db = db or g.db
|
|
|
|
|
2022-10-11 17:28:08 +00:00
|
|
|
media = db.query(Media).filter_by(filename=filename, kind='image').one_or_none()
|
2022-10-11 17:26:38 +00:00
|
|
|
if media: db.delete(media)
|
|
|
|
|
2022-10-06 05:54:04 +00:00
|
|
|
media = Media(
|
|
|
|
kind='image',
|
2022-10-11 17:20:15 +00:00
|
|
|
filename=filename,
|
2022-11-15 09:19:08 +00:00
|
|
|
user_id=uploader_id or v.id,
|
2022-10-06 05:54:04 +00:00
|
|
|
size=os.stat(filename).st_size
|
|
|
|
)
|
2022-10-06 19:07:45 +00:00
|
|
|
db.add(media)
|
2022-10-06 05:54:04 +00:00
|
|
|
|
2022-09-29 05:43:29 +00:00
|
|
|
return filename
|
2023-01-23 09:58:38 +00:00
|
|
|
|
|
|
|
|
2023-02-26 12:08:37 +00:00
|
|
|
def process_dm_images(v, user, body):
|
2023-01-23 09:58:38 +00:00
|
|
|
if not request.files.get("file") or g.is_tor or not get_setting("dm_images"):
|
2023-02-26 12:54:36 +00:00
|
|
|
return body
|
2023-01-23 09:58:38 +00:00
|
|
|
|
2023-02-26 12:19:30 +00:00
|
|
|
files = request.files.getlist('file')[:8]
|
2023-02-26 12:08:37 +00:00
|
|
|
|
2023-02-27 05:24:26 +00:00
|
|
|
while body.count('[file]') < len(files):
|
|
|
|
body += '\n[file]'
|
2023-02-26 12:46:38 +00:00
|
|
|
|
|
|
|
for file in files:
|
2023-01-23 09:58:38 +00:00
|
|
|
if file.content_type.startswith('image/'):
|
|
|
|
filename = f'/dm_images/{time.time()}'.replace('.','') + '.webp'
|
|
|
|
file.save(filename)
|
|
|
|
|
|
|
|
size = os.stat(filename).st_size
|
|
|
|
patron = bool(v.patron)
|
|
|
|
|
|
|
|
if size > MAX_IMAGE_AUDIO_SIZE_MB_PATRON * 1024 * 1024 or not patron and size > MAX_IMAGE_AUDIO_SIZE_MB * 1024 * 1024:
|
|
|
|
os.remove(filename)
|
|
|
|
abort(413, f"Max image/audio size is {MAX_IMAGE_AUDIO_SIZE_MB} MB ({MAX_IMAGE_AUDIO_SIZE_MB_PATRON} MB for paypigs)")
|
|
|
|
|
|
|
|
with open(filename, 'rb') as f:
|
|
|
|
os.remove(filename)
|
|
|
|
try:
|
|
|
|
req = requests.request(
|
|
|
|
"POST",
|
|
|
|
"https://pomf2.lain.la/upload.php",
|
|
|
|
files={'files[]': f},
|
|
|
|
timeout=20,
|
|
|
|
proxies=proxies
|
|
|
|
).json()
|
|
|
|
except requests.Timeout:
|
|
|
|
abort(400, "Image upload timed out, please try again!")
|
2023-02-22 17:27:33 +00:00
|
|
|
|
2023-01-23 09:58:38 +00:00
|
|
|
try: url = req['files'][0]['url']
|
|
|
|
except: abort(400, req['description'])
|
2023-02-22 17:27:33 +00:00
|
|
|
|
2023-02-27 05:24:26 +00:00
|
|
|
body = body.replace('[file]', url, 1)
|
2023-02-26 12:08:37 +00:00
|
|
|
|
|
|
|
with open(f"{LOG_DIRECTORY}/dm_images.log", "a+", encoding="utf-8") as f:
|
|
|
|
if user:
|
|
|
|
f.write(f'{url}, {v.username}, {v.id}, {user.username}, {user.id}, {int(time.time())}\n')
|
|
|
|
else:
|
|
|
|
f.write(f'{url}, {v.username}, {v.id}, Modmail, Modmail, {int(time.time())}\n')
|
2023-02-22 17:27:33 +00:00
|
|
|
|
2023-02-26 12:08:37 +00:00
|
|
|
return body.strip()
|