konvert/server.py

import base64, uuid
import datetime
import hashlib
import os
import subprocess
from subprocess import Popen, PIPE
import tornado.ioloop
import tornado.web
from tornado.escape import url_escape
from tornado.options import define, options
from cgi import escape as htmlspecialchars
import xattr

import logging
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

class KonvertHandler(tornado.web.RequestHandler):
    def writeln(self, *args, **kwargs):
        super().write(*args, **kwargs)
        self.write("\n")

class DownloadHandler(tornado.web.StaticFileHandler):
    def get(self, *args, head=False, **kwargs):
        with open(path, 'rb') as f:
            self.set_header('Expires', datetime.datetime.utcnow() + datetime.timedelta(1000000))

            try:
                mimetype = xattr.get(f, 'user.mime_type').decode('utf-8')
                log.debug('Found mime_type in xattrs')
                self.set_header('Content-Type', mimetype)
            except OSError:
                pass
            try:
                orig_filename = xattr.get(f, 'user.filename').decode('utf-8')
                self.set_header('Content-Disposition',' inline; filename="{}"'.format(url_escape(orig_filename, plus=False)))
            except OSError:
                pass
            self.set_header('Content-Length',os.stat(f.fileno()).st_size)
            if head:
               self.finish()
               return
        return super().get(*args, **kwargs)

    def head(self, *args, **kwargs):
        return self.get(*args, head=True, **kwargs)

class IndexHandler(KonvertHandler):
    def get(self):
        log.debug

@tornado.web.stream_request_body
class MainHandler(KonvertHandler):
    expect_hash = None

    def sha512_existed(self):
        self.load_uuid_by_hash('sha512', self.sha512sum)
        with open(os.path.join(self._metadatadir, 'filenames'), 'a') as f:
            ''' Append the new filename (maybe check for dupes in the future? '''
            f.write(self._filename + "\n")
        self.report_back()
        self.finish()

    def initialize(self):
        log.debug('Initializing variables')
        self.recv_bytes = 0
        self.sha512 = hashlib.sha512()
        self._sha512dir = os.path.join('uploads', 'by-hash', 'sha512')
        os.makedirs(self._sha512dir, exist_ok=True)

    def prepare(self):
        supplied_hash = self.request.headers.get('X-Body-Hash')
        if supplied_hash:
            ''' should be able to handle something like: hexdigest; algo=sha512'''
            hexdigest = None
            algo = None # default
            for part in supplised_hash.split(';'):
                if hexdigest is None:
                    hexdigest = part.strip()
                    continue
                try:
                    (var_name, var_value) = list(x.strip() for x in part.split('='))
                    if var_name == 'algo':
                        if not var_value in supported_algos:
                            algo = False
                            raise KeyError('Algorithm "{}" not supported, try sha512 instead.'.format(var_value))
                        algo = var_value

                except ValueError as e:
                    log.info('Malformed X-Body-Hash part could not be parsed: {}'.format(part))
                except KeyError as e:
                    log.info(e)
            
            if algo or algo is None:
                self.expecthash = (algo, hexdigest)
                log.debug('Expected hash value presented: '.self.expecthash)
                if os.path.exists(os.path.join(self._sha512dir, self.expecthash[1].lower())):
                    self.sha512sum = self.expect_hash
                    ''' this runs finish() '''
                    self.sha512_existed()

        log.debug('Preparing to receive file data')
        self._uuid = uuid.uuid4().hex
        log.debug('Using uuid={}'.format(self._uuid))
        self._metadatadir = os.path.join('metadata', self._uuid)
        self._uploaddir = os.path.join('receiving', self._uuid)

        os.makedirs(self._metadatadir)
        os.makedirs(self._uploaddir)

        self._recvpath = os.path.join(self._uploaddir, 'receiving')

    def data_received(self, data):
        if self.recv_bytes == 0:
            self._file = open(self._recvpath, 'wb')

        log.debug('receiving %d bytes' % len(data))
        self.recv_bytes += len(data)
        self.sha512.update(data)
        self._file.write(data)
        log.debug('%d bytes received in total' % self.recv_bytes)

    def load_uuid_by_hash(self, hash_algo, hexdigest):
        with open(os.path.join('uploads', 'by-hash', hash_algo, hexdigest + '.uuid'), 'r') as f:
            uuid = f.read(127).strip()
        log.debug('read uuid=%s' % uuid)
        self._uuid = uuid
        return uuid

    def put(self, filename):
        if len(filename) > 127:
            (filename, ext) = os.path.splitext(filename)
            extlen = len(ext)+1 # with a dot
            filename = '%s.%s' % (filename[:(127-extlen)], ext,)
        self._file.close()

        self.sha512sum = self.sha512.hexdigest()
        if self.expect_hash and self.expect_hash != ('sha512', self.sha512sum,):
            raise ValueError('Actual hash did not match client X-Body-Hash header')

        sha512path = os.path.join(self._sha512dir, self.sha512sum)
        self._filename = os.path.basename(filename)

        if os.path.exists(sha512path):
            try:
                os.remove(self._recvpath)
            except FileNotFoundError as e:
                log.info('File already gone or never existed: {}'.format(self._recvpath))
            ''' FIXME: remove metadatadir with old uuid too '''
            self.sha512_existed()

        os.rename(self._recvpath, sha512path)

        xattr.setxattr(sha512path, 'user.filename', filename.encode())
        mimetype = Popen(['file', '-b','--mime-type', sha512path], stdout=PIPE).communicate()[0].decode('utf8').strip()
        xattr.setxattr(sha512path, 'user.mime_type', mimetype.encode())

        with open(os.path.join(self._metadatadir, 'filenames'), 'a') as f:
            f.write(filename + "\n")
        with open(os.path.join(sha512path + '.uuid'), 'w') as f:
            f.write(self._uuid)
        with open(os.path.join(self._metadatadir, 'mimetype'), 'w') as f:
            f.write(mimetype)
        with open(os.path.join(self._metadatadir, 'SHA512SUMS'), 'w') as f:
            f.write('{}  file/{}'.format(self.sha512sum, self._filename))

        return self.report_back()

    def report_back(self, url=None, redirect=False):
        self.write(url or 'http://{}/uploads/by-hash/sha512/{}\n'.format(self.request.host, self.sha512sum))
        if redirect:
            self.redirect('http://{}/metadata/by-hash/sha512/{}'.format(self.request.host, self.sha512sum), status=303)

supported_algos = ['sha512']

templates = {
    'ffmpeg__opus': {
                    'streams': ['audio'],
                    'command': 'ffmpeg',
                    'file_ext': 'opus',
                    'args': [
                        ('-loglevel', 'warning',),
                        ('-i', '%%FILENAME%%',),
                        ('-f', 'opus',),
                        ('-c:a', 'opus',),
                        ('-vn'),
                        ],
                    },
    'ffmpeg__flac': {
                    'streams': ['audio'],
                    'command': 'ffmpeg',
                    'file_ext': 'flac',
                    'args': [
                        ('-loglevel', 'warning',),
                        ('-f', 'flac',),
                        ('-c:a', 'libvorbis',),
                        ('-vn'),
                        ],
                    },
    'ffmpeg__ogg_vorbis': {
                    'streams': ['audio'],
                    'command': 'ffmpeg',
                    'file_ext': 'ogg',
                    'args': [
                        ('-loglevel', 'warning',),
                        ('-i', '%%FILENAME%%',),
                        ('-f', 'oga',),
                        ('-c:a', 'libvorbis',),
                        ('-vn'),
                        ],
                    },
    'ffmpeg__wvga_webm_vorbis@128k_vp8@600k': {
                    'streams': ['video', 'audio'],
                    'command': 'ffmpeg',
                    'file_ext': 'webm',
                    'args': [
                        ('-loglevel', 'warning',),
                        ('-progress', '%%FFMPEG_PROGRESS_URL%%',),
                        ('-i', '%%FILENAME%%',),
                        ('-s', 'wvga',),
                        ('-f', 'webm',),
                        ('-c:a', 'libvorbis',),
                        ('-b:a', '128k',),
                        ('-c:v', 'libvpx',),
                        ('-b:v', '600k',),
                        ],
                    },
    }

class ProcessHandler(KonvertHandler):
    def load_filename(self, uuid):
        with open(os.path.join('uploads', 'by-uuid', uuid, 'filename'), 'r') as f:
            filename = f.read(127)
        log.debug('read filename=%s' % filename)
        return filename

    def load_mimetype(self, uuid):
        with open(os.path.join('uploads', 'by-uuid', uuid, 'mimetype'), 'r') as f:
            mimetype = f.read(127)
        log.debug('read mimetype=%s' % mimetype)
        return mimetype

    def load_template(self, template_name):
        return templates[template_name]

    def get(self, uuid, template_name):
        filename = self.load_filename(uuid)
        metadata_dir = os.path.join('uploads', 'by-uuid', uuid)
        filepath = os.path.join(metadata_dir, 'file', filename)

        mimetype = self.load_mimetype(uuid)
        self.writeln('%s from %s to %s' % (uuid, mimetype, template_name,))

        template = self.load_template(template_name)

        outname = '%s.%s' % (os.path.splitext(filename)[0], template['file_ext'],)
        outdir = os.path.join('uploads', 'by-uuid', uuid, 'convert', template_name)
        os.makedirs(outdir)
        outtmp = os.path.join(outdir, 'converting')
        outpath = os.path.join(outdir, outname)

        progress_url = 'http://{}/uploads/{}/convert/{}/progress'.format(self.request.host, uuid, url_escape(template_name, plus=False))

        ffmpeg_cmd = [template['command']]
        for arg in template['args']:
            for part in arg:
                log.debug('replacing macros with values in: %s' % part)
                part = part.replace('%%FILENAME%%', filepath)
                part = part.replace('%%FFMPEG_PROGRESS_URL%%', progress_url)
                ffmpeg_cmd.append(part)
        ffmpeg_cmd += [outtmp]

        ffmpeg_log = open(os.path.join(outdir, 'ffmpeg.log'), 'wb')
        log.debug('Running %s' % ffmpeg_cmd)
        p_conv = Popen(ffmpeg_cmd, stdout=ffmpeg_log, stderr=subprocess.STDOUT)

        try:
            p_conv.wait(2)
        except subprocess.TimeoutExpired as e:
            self.writeln('Resulting file will be available at %s' % 'http://{}/uploads/{}/convert/{}/{}'.format(self.request.host, uuid, url_escape(template_name, plus=False), url_escape(filename, plus=False)))
            self.set_status(202, reason='Accepted, will konvert it. Please come back later.')
            return

        if p_conv.poll() is None:
            raise IOError('Process returncode None despite that it should have ended!')

        if p_conv.returncode < 0:
            raise ValueError('ffmpeg exited unnaturally with POSIX signal %d' % abs(p_conv.returncode))
        elif p_conv.returncode > 0:
            self.set_status(500, 'ffmpeg exited due to an error, please see the log output at {}'.format('http://{}/uploads/{}/convert/{}/ffmpeg.log'.format(self.request.host, uuid, template_name), status=500))
            return

        return self.report_back(uuid, template_name, outname)

    def report_back(self, uuid, template_name, filename):
        url = 'http://{}/uploads/{}/convert/{}'.format(self.request.host, uuid, url_escape(filename, plus=False))
        self.writeln(url)
        self.redirect(url)

@tornado.web.stream_request_body
class ProgressHandler(tornado.web.RequestHandler):
    def load_progress(self, uuid, template_name):
        template_name = os.path.basename(template_name)
        with open(os.path.join('uploads', 'by-uuid', uuid, 'convert', template_name, 'progress'), 'r') as f:
            progress = f.read(127)
        log.debug('read progress=%s' % progress)
        return progress

    def prepare(self):
        log.debug('new progress for %s template %s' % (self.path_kwargs['uuid'], self.path_kwargs['template_name']))
        self._uuid = self.path_kwargs['uuid']
        self._template_name = self.path_kwargs['template_name']

    def data_received(self, data):
        #self.save_progress(data)
        log.debug('progress ongoing: %s' % data)

    def get(self, uuid, template_name):
        self.write(self.load_progress(uuid, template_name))

    def post(self, uuid, template_name):
        log.debug('%s with %s' % (uuid, template_name,))

class HelpHandler(KonvertHandler):
    def get(self, section):
        self.writeln('<h1>Konvert Help</h1>')
        if section == 'convert':
            self.writeln('<h2>Convert</h2>')
            self.writeln('<p>Convert by appending the UUID URL with <code>/convert/%%TEMPLATE_NAME%%</code> where <code>%%TEMPLATE_NAME%%</code> is one of the templates listed below.</p>')
            self.writeln('<h3>Templates</h3>')
            self.writeln('<ul>')
            for template in templates:
                self.write('<li>{}</li>'.format(htmlspecialchars(template)))
            self.writeln('</ul>')

application = tornado.web.Application([
    (r'/uploads/(by-hash/sha512/[a-f0-9]{16,})', tornado.web.StaticFileHandler, {
            'path': os.path.join(os.path.dirname(__file__), 'uploads'),
        }),
    (r'/uploads/([a-f0-9]{16,}/file/.*)', tornado.web.StaticFileHandler, {
            'path': os.path.join(os.path.dirname(__file__), 'uploads'),
        }),
    (r'/uploads/([a-f0-9]{16,}/convert/[\w\d\-\_\.\@\%]+/(?!progress).*)', tornado.web.StaticFileHandler, {
            'path': os.path.join(os.path.dirname(__file__), 'uploads'),
        }),
    (r'/uploads/(?P<uuid>[a-f0-9]{16,})/convert/(?P<template_name>[\w\d\-\_\.\@\%]+)/progress', ProgressHandler),
    (r'/uploads/(?P<uuid>[a-f0-9]{16,})/convert/(?P<template_name>[\w\d\-\_\.\@\%]+)', ProcessHandler),
    (r'/help/(\w+)', HelpHandler),
    (r'/index.html', IndexHandler, {'path': os.path.join(os.path.dirname(__file__), 'uploads')}),
    (r'/(.*)', MainHandler),
], autoreload=True, debug=True)

if __name__ == '__main__':
    tornado.options.parse_command_line()
    path = os.path.join(os.path.join(os.path.dirname(__file__)), 'uploads')
    if not os.path.exists(path):
        os.makedirs(path)
    port = 8888
    log.info('Listening on %d' % port)
    application.listen(port)
    tornado.ioloop.IOLoop.instance().start()