From 0bbfb1ed0aa8c0b1fa66137e5271b02c8ceebe74 Mon Sep 17 00:00:00 2001 From: paranarimasu <33796518+paranarimasu@users.noreply.github.com> Date: Fri, 23 Oct 2020 00:36:09 -0500 Subject: [PATCH] Initial Commit --- .gitignore | 148 ++++++++++++++++++++++ LICENSE | 21 +++ README.md | 36 ++++++ batch-encoder.ini | 14 ++ batch_encoder/__init__.py | 0 batch_encoder/__main__.py | 131 +++++++++++++++++++ batch_encoder/_bitrate_mode.py | 24 ++++ batch_encoder/_colorspace.py | 56 ++++++++ batch_encoder/_encode_webm.py | 204 ++++++++++++++++++++++++++++++ batch_encoder/_encoding_config.py | 57 +++++++++ batch_encoder/_loudnorm_filter.py | 58 +++++++++ batch_encoder/_seek.py | 18 +++ batch_encoder/_seek_collector.py | 133 +++++++++++++++++++ batch_encoder/_source_file.py | 153 ++++++++++++++++++++++ batch_encoder/_utils.py | 37 ++++++ setup.py | 30 +++++ 16 files changed, 1120 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 batch-encoder.ini create mode 100644 batch_encoder/__init__.py create mode 100644 batch_encoder/__main__.py create mode 100644 batch_encoder/_bitrate_mode.py create mode 100644 batch_encoder/_colorspace.py create mode 100644 batch_encoder/_encode_webm.py create mode 100644 batch_encoder/_encoding_config.py create mode 100644 batch_encoder/_loudnorm_filter.py create mode 100644 batch_encoder/_seek.py create mode 100644 batch_encoder/_seek_collector.py create mode 100644 batch_encoder/_source_file.py create mode 100644 batch_encoder/_utils.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fd1e7c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,148 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +# JetBrains IDEs +.idea + +# End of https://www.toptal.com/developers/gitignore/api/python \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d51e884 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 paranarimasu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..931e2d2 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +### Description + +Generate and execute collection of FFmpeg commands sequentially from external file to produce WebMs that meet [AnimeThemes.moe](https://animethemes.moe/) encoding standards. + +Take advantage of sleep, work, or any other time that we cannot actively monitor the encoding process to produce a set of encodes for later quality checking and/or tweaking for additional encodes. + +Ideally we are iterating over a combination of filters and settings, picking the best one at the end. + +### Install + +**Requirements:** + +* FFmpeg +* Python >= 3.6 + +**Install:** + + pip install animethemes-batch-encoder + +### Usage + + python -m batch_encoder [-h] --mode [{1,2,3}] [--file [FILE]] [--configfile [CONFIGFILE]] --loglevel [{debug,info,error}] + +* `--mode 1`: Generates commands from input files in the current directory. User will be prompted for inclusion/exclusion of input file, start time, end time and output file name. +* `--mode 2`: Execute commands from file line-by-line in the current directory. +* `--mode 3`: Generate commands from input files in the current directory in the same manner as Mode 1 and then execute commands without writing to file. +* `[FILE]`: The file that commands are written to or read from. Default: commands.txt in the current directory. Unused in `--mode 3`. +* `[CONFIGFILE]`: The configuration file in which our properties are defined. Default: batch_encoder.ini written to the same directory as the batch_encoder.py script. +* `AllowedFileTypes`: Configuration property for file extensions that will be considered source file candidates +* `EncodingModes`: Configuration property for the inclusion and order of encoding modes `{CBR, VBR, CQ}` +* `CRFs`: Configuration property for the ordered list of CRF values to use with VBR and/or CQ encoding modes +* `IncludeUnfiltered`: Configuration property that sets the flag for including/excluding an encode without video filters for each EncodingMode +* `VideoFilters`: Configuration items used for named video filtergraphs +* `--loglevel error`: Only show error messages +* `--loglevel info`: Show error messages and script progression info messages +* `--loglevel debug`: Show all messages, including variable dumps \ No newline at end of file diff --git a/batch-encoder.ini b/batch-encoder.ini new file mode 100644 index 0000000..cf57171 --- /dev/null +++ b/batch-encoder.ini @@ -0,0 +1,14 @@ +[Encoding] +allowedfiletypes = .avi,.m2ts,.mkv,.mp4,.wmv +encodingmodes = VBR,CBR +crfs = 12,15,18,21,24 +includeunfiltered = True +defaultvideostream = +defaultaudiostream = + +[VideoFilters] +filtered = hqdn3d=0:0:3:3,gradfun,unsharp +lightdenoise = hqdn3d=0:0:3:3 +heavydenoise = hqdn3d=1.5:1.5:6:6 +unsharp = unsharp + diff --git a/batch_encoder/__init__.py b/batch_encoder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/batch_encoder/__main__.py b/batch_encoder/__main__.py new file mode 100644 index 0000000..0279493 --- /dev/null +++ b/batch_encoder/__main__.py @@ -0,0 +1,131 @@ +from ._encode_webm import EncodeWebM +from ._encoding_config import EncodingConfig +from ._seek_collector import SeekCollector +from ._source_file import SourceFile +from ._utils import commandfile_arg_type +from ._utils import configfile_arg_type + +import argparse +import configparser +import logging +import os +import shutil +import subprocess +import sys + + +def main(): + # Load/Validate Arguments + parser = argparse.ArgumentParser(prog='batch-encoder', + description='Generate/Execute FFmpeg commands for files in acting directory', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--mode', nargs='?', type=int, choices=[1, 2, 3], required=True, + help='1: Generate commands and write to file\n' + '2: Execute commands from file\n' + '3: Generate and execute commands') + parser.add_argument('--file', nargs='?', default='commands.txt', type=commandfile_arg_type, + help='1: Name of file commands are written to (default: commands.txt)\n' + '2: Name of file commands are executed from (default: commands.txt)\n' + '3: Unused') + parser.add_argument('--configfile', nargs='?', default='batch-encoder.ini', type=configfile_arg_type, + help='Name of config file (default: batch-encoder.ini)\n' + 'If the file does not exist, default configuration will be written\n' + 'The file is expected to exist in the same directory as this script') + parser.add_argument('--loglevel', nargs='?', default='info', choices=['debug', 'info', 'error'], + help='Set logging level') + args = parser.parse_args() + + # Logging Config + logging.basicConfig(stream=sys.stdout, level=logging.getLevelName(args.loglevel.upper()), + format='%(levelname)s: %(message)s') + + # Env Check: Check that dependencies are installed + if shutil.which('ffmpeg') is None: + logging.error('FFmpeg is required') + sys.exit() + + if shutil.which('ffprobe') is None: + logging.error('FFprobe is required') + sys.exit() + + # Write default config file if it doesn't exist + config = configparser.ConfigParser() + config_file = os.path.join(sys.path[0], args.configfile) + if not os.path.exists(config_file): + config['Encoding'] = {EncodingConfig.config_allowed_filetypes: EncodingConfig.default_allowed_filetypes, + EncodingConfig.config_encoding_modes: EncodingConfig.default_encoding_modes, + EncodingConfig.config_crfs: EncodingConfig.default_crfs, + EncodingConfig.config_include_unfiltered: EncodingConfig.default_include_unfiltered, + EncodingConfig.config_default_video_stream: '', + EncodingConfig.config_default_audio_stream: ''} + config['VideoFilters'] = EncodingConfig.default_video_filters + + with open(config_file, 'w', encoding='utf8') as f: + config.write(f) + + # Load config file + config.read(config_file) + encoding_config = EncodingConfig.from_config(config) + + commands = [] + + # Generate commands from source file candidates in current directory + if args.mode == 1 or args.mode == 3: + source_file_candidates = [f for f in os.listdir('.') if f.endswith(tuple(encoding_config.allowed_filetypes))] + + if not source_file_candidates: + logging.error('No source file candidates in current directory') + sys.exit() + + for source_file_candidate in source_file_candidates: + if SourceFile.yes_or_no(source_file_candidate): + try: + source_file = SourceFile.from_file(source_file_candidate, encoding_config) + + is_collector_valid = False + seek_collector = None + while not is_collector_valid: + seek_collector = SeekCollector(source_file) + is_collector_valid = seek_collector.is_valid() + + for seek in seek_collector.get_seek_list(): + logging.info(f'Generating commands with seek ss: \'{seek.ss}\', to: \'{seek.to}\'') + encode_webm = EncodeWebM(source_file, seek) + commands = commands + encode_webm.get_commands(encoding_config) + except KeyboardInterrupt: + logging.info(f'Exiting from inclusion of file \'{source_file_candidate}\' after keyboard interrupt') + + # Write commands to file + if args.mode == 1: + logging.info(f'Writing {len(commands)} commands to file \'{args.file}\'...') + with open(args.file, mode='w', encoding='utf8') as f: + for command in commands: + f.write(command + '\n') + + # Read and execute commands from file + if args.mode == 2: + if not os.path.isfile(args.file): + logging.error(f'File \'{args.file}\' does not exist') + sys.exit() + + with open(args.file, mode='r', encoding='utf8') as f: + for command in f: + commands.append(command) + + logging.info(f'Reading {len(commands)} commands from file \'{args.file}\'...') + + for command in commands: + subprocess.call(command, shell=True) + + # Execute commands in memory + if args.mode == 3: + logging.info(f'Executing {len(commands)} commands...') + for command in commands: + subprocess.call(command, shell=True) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + logging.error('Exiting after keyboard interrupt') diff --git a/batch_encoder/_bitrate_mode.py b/batch_encoder/_bitrate_mode.py new file mode 100644 index 0000000..700dea8 --- /dev/null +++ b/batch_encoder/_bitrate_mode.py @@ -0,0 +1,24 @@ +import enum + + +# The Bitrate Mode Enumerated List +# Bitrate Mode determines the rate control argument values for our commands +# Further Reading: https://developers.google.com/media/vp9/bitrate-modes +class BitrateMode(enum.Enum): + def __new__(cls, value, first_pass_rate_control, second_pass_rate_control): + obj = object.__new__(cls) + obj._value_ = value + obj.first_pass_rate_control = first_pass_rate_control + obj.second_pass_rate_control = second_pass_rate_control + return obj + + # Constant Bitrate Mode + CBR = (0, lambda cbr_bitrate, cbr_max_bitrate, crf: f'-b:v {cbr_bitrate} -maxrate {cbr_max_bitrate} -qcomp 0.3', + lambda cbr_bitrate, cbr_max_bitrate, + crf: f'-b:v {cbr_bitrate} -maxrate {cbr_max_bitrate} -bufsize 6000k -qcomp 0.3') + # Variable Bitrate Mode / Constant Quality Mode + VBR = (1, lambda cbr_bitrate, cbr_max_bitrate, crf: f'-crf {crf} -b:v 0 -qcomp 0.7', + lambda cbr_bitrate, cbr_max_bitrate, crf: f'-crf {crf} -b:v 0 -qcomp 0.7') + # Constrained Quality Mode + CQ = (2, lambda cbr_bitrate, cbr_max_bitrate, crf: f'-crf {crf} -b:v {cbr_bitrate} -qcomp 0.7', + lambda cbr_bitrate, cbr_max_bitrate, crf: f'-crf {crf} -b:v {cbr_bitrate} -qcomp 0.7') diff --git a/batch_encoder/_colorspace.py b/batch_encoder/_colorspace.py new file mode 100644 index 0000000..2538884 --- /dev/null +++ b/batch_encoder/_colorspace.py @@ -0,0 +1,56 @@ +import enum +import logging + + +# The Colorspace Enumerated List +# Parse color data from file to provide arguments for the encoded file +class Colorspace(enum.Enum): + def __new__(cls, colorspace, color_primaries, color_trc): + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + obj.colorspace = colorspace + obj.color_primaries = color_primaries + obj.color_trc = color_trc + return obj + + HD = ('bt709', 'bt709', 'bt709') + NTSC = ('smpte170m', 'smpte170m', 'smpte170m') + PAL = ('bt470bg', 'bt470bg', 'gamma28') + + # The color data arguments for our encode + def get_args(self): + return f'-colorspace {self.colorspace} -color_primaries {self.color_primaries} -color_trc {self.color_trc}' + + @staticmethod + def value_of(source_file): + # Method 1: Carry over color data from source if specified + source_colorspace = source_file.video_format['streams'][0].get('color_space', '') + source_color_primaries = source_file.video_format['streams'][0].get('color_primaries', '') + source_color_trc = source_file.video_format['streams'][0].get('color_transfer', '') + logging.debug( + f'[Colorspace.value_of] source_colorspace: {source_colorspace}, ' + f'source_color_primaries: {source_color_primaries}, ' + f'source_color_trc: {source_color_trc}') + + for colorspace_candidate in Colorspace: + if ( + source_colorspace == colorspace_candidate.colorspace + or source_color_primaries == colorspace_candidate.color_primaries + or source_color_trc == colorspace_candidate.color_trc + ): + logging.debug(f'[Colorspace.value_of] carryover colorspace \'{colorspace_candidate.name}\' from source') + return colorspace_candidate + + # Method 2: Infer color date from source file resolution + resolution = int(source_file.video_format['streams'][0]['height']) + + if resolution >= 720: + logging.debug(f'[Colorspace.value_of] colorspace: \'{Colorspace.HD.name}\', resolution: \'{resolution}\'') + return Colorspace.HD + elif resolution >= 576: + logging.debug(f'[Colorspace.value_of] colorspace: \'{Colorspace.PAL.name}\', resolution: \'{resolution}\'') + return Colorspace.PAL + else: + logging.debug(f'[Colorspace.value_of] colorspace: \'{Colorspace.NTSC.name}\', resolution: \'{resolution}\'') + return Colorspace.NTSC diff --git a/batch_encoder/_encode_webm.py b/batch_encoder/_encode_webm.py new file mode 100644 index 0000000..30b37a3 --- /dev/null +++ b/batch_encoder/_encode_webm.py @@ -0,0 +1,204 @@ +from ._bitrate_mode import BitrateMode +from ._colorspace import Colorspace +from ._loudnorm_filter import LoudnormFilter +from ._utils import string_to_seconds + +import logging + + +# The class that generates FFmpeg commands for the specific cut in the source file +# We generate common argument values that can be determined programmatically and then use our config to produce commands +class EncodeWebM: + def __init__(self, source_file, seek): + self.source_file = source_file + self.seek = seek + self.loudnorm_filter = LoudnormFilter.from_seek(self.seek) + self.g = self.get_keyframe_interval() + self.audio_bitrate = self.get_audio_bitrate() + self.cbr_bitrate = self.get_cbr_bitrate() + self.cbr_max_bitrate = self.get_cbr_max_bitrate() + self.colorspace = Colorspace.value_of(self.source_file) + + # We want at least 10 keyframes in our encode and consistency in our interval + def get_keyframe_interval(self): + source_file_duration = float(self.source_file.file_format['format']['duration']) + + start_time = string_to_seconds(self.seek.ss) if self.seek.ss else 0 + end_time = string_to_seconds(self.seek.to) if self.seek.to else source_file_duration + duration = end_time - start_time + + logging.debug( + f'[EncodeWebm.get_keyframe_interval] ' + f'duration: \'{duration}\', ' + f'end_time: \'{end_time}\', ' + f'start_time: \'{start_time}\'') + + if duration < 60: + return 96 + elif duration < 120: + return 120 + else: + return 240 + + # Audio must use a default bitrate of 192 kbps + # Audio must use a bitrate of 320 kbps if the source bitrate is > 320 kbps + def get_audio_bitrate(self): + audio_bitrate = int(self.source_file.audio_format['format']['bit_rate']) + + logging.debug(f'[EncodeWebm.get_audio_bitrate] audio_bitrate: \'{audio_bitrate}\'') + + if audio_bitrate > 320000: + return '320k' + else: + return '192k' + + # Approximation of target average bitrate near file size limit + def get_cbr_bitrate(self): + resolution = int(self.source_file.video_format['streams'][0]['height']) + + logging.debug(f'[EncodeWebm.get_cbr_bitrate] resolution: \'{resolution}\'') + + if resolution >= 1080: + return '5600k' + elif resolution >= 720: + return '3700k' + elif resolution >= 576: + return '3200k' + else: + return '2400k' + + # Approximation of max overall bitrate near file size limit + def get_cbr_max_bitrate(self): + resolution = int(self.source_file.video_format['streams'][0]['height']) + + logging.debug(f'[EncodeWebm.get_cbr_max_bitrate] resolution: \'{resolution}\'') + + if resolution >= 1080: + return '6400k' + elif resolution >= 720: + return '4200k' + elif resolution >= 576: + return '3700k' + else: + return '3200k' + + # First-pass encode + def get_first_pass(self, encoding_mode, crf=None): + return f'ffmpeg {self.seek.get_seek_string()} ' \ + f'-pass 1 -passlogfile {self.seek.output_name} ' \ + f'-map 0:v:{self.source_file.selected_video_stream} ' \ + f'-map 0:a:{self.source_file.selected_audio_stream} ' \ + f'-c:v libvpx-vp9 ' \ + f'{encoding_mode.first_pass_rate_control(self.cbr_bitrate, self.cbr_max_bitrate, crf)} ' \ + f'-cpu-used 4 -g {self.g} -threads 4 -tile-columns 6 -frame-parallel 0 -auto-alt-ref 1 ' \ + f'-lag-in-frames 25 -row-mt 1 -pix_fmt yuv420p {self.colorspace.get_args()} -an -sn -f webm -y NUL' + + # Second-pass encode + def get_second_pass(self, encoding_mode, crf=None, video_filters='', webm_filename=''): + return f'ffmpeg {self.seek.get_seek_string()} ' \ + f'-pass 2 -passlogfile {self.seek.output_name} ' \ + f'-map 0:v:{self.source_file.selected_video_stream} ' \ + f'-map 0:a:{self.source_file.selected_audio_stream} ' \ + f'-c:v libvpx-vp9 ' \ + f'{encoding_mode.second_pass_rate_control(self.cbr_bitrate, self.cbr_max_bitrate, crf)} ' \ + f'-cpu-used 0 -g {self.g} -threads 4 {self.get_audio_filters()}{video_filters} -tile-columns 6 ' \ + f'-frame-parallel 0 -auto-alt-ref 1 -lag-in-frames 25 -row-mt 1 -pix_fmt yuv420p ' \ + f'{self.colorspace.get_args()} ' \ + f'-c:a libopus -b:a {self.audio_bitrate} -ar 48k ' \ + f'-map_metadata -1 -map_chapters -1 -sn -f webm -y {webm_filename}.webm' + + # Build audio filtergraph for encodes + def get_audio_filters(self): + audio_filters = [] + self.source_file.apply_audio_resampling(audio_filters) + audio_filters.append(self.loudnorm_filter.get_normalization_filter()) + return '-af ' + ','.join(audio_filters) + + # Build video filtergraph for encodes + @staticmethod + def get_video_filters(config_filter=None): + video_filters = [] + + if config_filter is not None: + video_filters.append(config_filter) + + if not video_filters: + return '' + + return ' -vf ' + ','.join(video_filters) + + # Build unique WebM filename for encodes + def get_webm_filename(self, crf=None, cbr_bitrate=None, filter_name=None): + webm_filename = self.seek.output_name + + if crf is not None: + webm_filename += f'-{crf}' + + if cbr_bitrate is not None: + webm_filename += f'-{self.cbr_bitrate}' + + if filter_name is not None: + webm_filename += f'-{filter_name}' + + return webm_filename + + # Get list of commands in sequence specified by configuration file + # Sequencing - 1. Encoding Mode, 2: CRF, 3: Filter mapping + def get_commands(self, encoding_config): + file_commands = [] + + logging.debug( + f'[EncodeWebm.get_commands] encoding_modes: \'{encoding_config.encoding_modes}\', ' + f'crfs: \'{encoding_config.crfs}\', ' + f'include_unfiltered: \'{encoding_config.include_unfiltered}\', ' + f'video_filters: \'{encoding_config.video_filters}\'') + + for encoding_mode in encoding_config.encoding_modes: + if BitrateMode.CBR.name == encoding_mode.upper(): + file_commands.append(self.get_first_pass(BitrateMode.CBR)) + if encoding_config.include_unfiltered: + file_commands.append(self.get_second_pass(BitrateMode.CBR, + video_filters=EncodeWebM.get_video_filters(), + webm_filename=self.get_webm_filename( + cbr_bitrate=self.cbr_bitrate))) + for filter_name, filter_value in encoding_config.video_filters: + file_commands.append(self.get_second_pass(BitrateMode.CBR, + video_filters=EncodeWebM.get_video_filters( + config_filter=filter_value), + webm_filename=self.get_webm_filename( + cbr_bitrate=self.cbr_bitrate, + filter_name=filter_name))) + elif BitrateMode.VBR.name == encoding_mode.upper(): + for crf in encoding_config.crfs: + file_commands.append(self.get_first_pass(BitrateMode.VBR, crf=crf)) + if encoding_config.include_unfiltered: + file_commands.append( + self.get_second_pass(BitrateMode.VBR, crf=crf, video_filters=EncodeWebM.get_video_filters(), + webm_filename=self.get_webm_filename(crf=crf))) + for filter_name, filter_value in encoding_config.video_filters: + file_commands.append(self.get_second_pass(BitrateMode.VBR, crf=crf, + video_filters=EncodeWebM.get_video_filters( + config_filter=filter_value), + webm_filename=self.get_webm_filename( + crf=crf, + filter_name=filter_name))) + elif BitrateMode.CQ.name == encoding_mode.upper(): + for crf in encoding_config.crfs: + file_commands.append(self.get_first_pass(BitrateMode.CQ, crf=crf)) + if encoding_config.include_unfiltered: + file_commands.append( + self.get_second_pass(BitrateMode.CQ, crf=crf, video_filters=self.get_video_filters(), + webm_filename=self.get_webm_filename(crf=crf, + cbr_bitrate=self.cbr_bitrate))) + for filter_name, filter_value in encoding_config.video_filters: + file_commands.append(self.get_second_pass(BitrateMode.CQ, crf=crf, + video_filters=EncodeWebM.get_video_filters( + config_filter=filter_value), + webm_filename=self.get_webm_filename( + crf=crf, + cbr_bitrate=self.cbr_bitrate, + filter_name=filter_name))) + + logging.debug(f'[EncodeWebm.get_commands] # of file_commands: \'{len(file_commands)}\'') + + return file_commands diff --git a/batch_encoder/_encoding_config.py b/batch_encoder/_encoding_config.py new file mode 100644 index 0000000..4059c5d --- /dev/null +++ b/batch_encoder/_encoding_config.py @@ -0,0 +1,57 @@ +from ._bitrate_mode import BitrateMode + + +class EncodingConfig: + # Config keys + config_allowed_filetypes = 'AllowedFileTypes' + config_encoding_modes = 'EncodingModes' + config_crfs = 'CRFs' + config_include_unfiltered = 'IncludeUnfiltered' + + # Default Config keys + config_default_video_stream = 'DefaultVideoStream' + config_default_audio_stream = 'DefaultAudioStream' + + # Default config values + default_allowed_filetypes = '.avi,.m2ts,.mkv,.mp4,.wmv' + default_encoding_modes = f'{BitrateMode.VBR.name},{BitrateMode.CBR.name}' + default_crfs = '12,15,18,21,24' + default_include_unfiltered = True + default_video_filters = {'filtered': 'hqdn3d=0:0:3:3,gradfun,unsharp', + 'lightdenoise': 'hqdn3d=0:0:3:3', + 'heavydenoise': 'hqdn3d=1.5:1.5:6:6', + 'unsharp': 'unsharp'} + + def __init__(self, allowed_filetypes, encoding_modes, crfs, include_unfiltered, video_filters, default_video_stream, + default_audio_stream): + self.allowed_filetypes = allowed_filetypes + self.encoding_modes = encoding_modes + self.crfs = crfs + self.include_unfiltered = include_unfiltered + self.video_filters = video_filters + self.default_video_stream = default_video_stream + self.default_audio_stream = default_audio_stream + + @classmethod + def from_config(cls, config): + allowed_filetypes = config['Encoding'].get(EncodingConfig.config_allowed_filetypes, + EncodingConfig.default_allowed_filetypes).split(',') + encoding_modes = config['Encoding'].get(EncodingConfig.config_encoding_modes, + EncodingConfig.default_encoding_modes).split(',') + crfs = config['Encoding'].get(EncodingConfig.config_crfs, EncodingConfig.default_crfs).split(',') + include_unfiltered = config.getboolean('Encoding', EncodingConfig.config_include_unfiltered, + fallback=EncodingConfig.default_include_unfiltered) + video_filters = config.items('VideoFilters', EncodingConfig.default_video_filters) + + default_video_stream = config['Encoding'].get(EncodingConfig.config_default_video_stream) + default_audio_stream = config['Encoding'].get(EncodingConfig.config_default_audio_stream) + + return cls(allowed_filetypes, encoding_modes, crfs, include_unfiltered, video_filters, default_video_stream, + default_audio_stream) + + def get_default_stream(self, stream_type): + if stream_type == 'video': + return self.default_video_stream + elif stream_type == 'audio': + return self.default_audio_stream + return None \ No newline at end of file diff --git a/batch_encoder/_loudnorm_filter.py b/batch_encoder/_loudnorm_filter.py new file mode 100644 index 0000000..bdca06c --- /dev/null +++ b/batch_encoder/_loudnorm_filter.py @@ -0,0 +1,58 @@ +import json +import logging +import re +import shlex +import subprocess + + +# The audio normalization filter for our encode +class LoudnormFilter: + first_pass_filter = 'loudnorm=I=-16:LRA=20:TP=-1:dual_mono=true:linear=true:print_format=json' + + def __init__(self, input_i, input_lra, input_tp, input_thresh, target_offset): + self.input_i = input_i + self.input_lra = input_lra + self.input_tp = input_tp + self.input_thresh = input_thresh + self.target_offset = target_offset + + @classmethod + def from_seek(cls, seek): + logging.info('Retrieving loudness data...') + loudnorm_cmd = f'ffmpeg {seek.get_seek_string()} ' \ + f'-map 0:a:{seek.source_file.selected_audio_stream} ' \ + f'-af {LoudnormFilter.get_first_pass_filters(seek)} ' \ + f'-vn -sn -dn -f null /dev/null' + loudnorm_args = shlex.split(loudnorm_cmd) + loudnorm_output = subprocess.check_output(loudnorm_args, stderr=subprocess.STDOUT).decode('utf-8').strip() + loudnorm_stats = re.search('^{[^}]*}$', loudnorm_output, re.MULTILINE) + loudnorm_stats = json.loads(loudnorm_stats.group(0)) + + logging.debug( + f'[Loudnorm.__init__] input_i: \'{loudnorm_stats["input_i"]}\', ' + f'input_lra: \'{loudnorm_stats["input_lra"]}\', ' + f'input_tp: \'{loudnorm_stats["input_tp"]}\', ' + f'input_thresh: \'{loudnorm_stats["input_thresh"]}\', ' + f'target_offset: \'{loudnorm_stats["target_offset"]}\'') + + return cls(loudnorm_stats['input_i'], + loudnorm_stats['input_lra'], + loudnorm_stats['input_tp'], + loudnorm_stats['input_thresh'], + loudnorm_stats['target_offset']) + + # The audio normalization filter argument for our encode + def get_normalization_filter(self): + return f'loudnorm=I=-16:LRA=20:TP=-1:dual_mono=true:linear=true:' \ + f'measured_I={self.input_i}:' \ + f'measured_LRA={self.input_lra}:' \ + f'measured_TP={self.input_tp}:' \ + f'measured_thresh={self.input_thresh}:' \ + f'offset={self.target_offset}' + + @staticmethod + def get_first_pass_filters(seek): + audio_filters = [] + seek.source_file.apply_audio_resampling(audio_filters) + audio_filters.append(LoudnormFilter.first_pass_filter) + return ','.join(audio_filters) diff --git a/batch_encoder/_seek.py b/batch_encoder/_seek.py new file mode 100644 index 0000000..b6e5a37 --- /dev/null +++ b/batch_encoder/_seek.py @@ -0,0 +1,18 @@ +# The seek information for our encode +class Seek: + def __init__(self, source_file, ss, to, output_name): + self.source_file = source_file + self.ss = ss + self.to = to + self.output_name = output_name + + # The seek string arguments for our encode + def get_seek_string(self): + if len(self.ss) > 0 and len(self.to) > 0: + return f'-ss {self.ss} -to {self.to} -i "{self.source_file.file}"' + elif len(self.ss) > 0: + return f'-ss {self.ss} -i "{self.source_file.file}"' + elif len(self.to) > 0: + return f'-i "{self.source_file.file}" -to {self.to}' + else: + return f'-i "{self.source_file.file}"' diff --git a/batch_encoder/_seek_collector.py b/batch_encoder/_seek_collector.py new file mode 100644 index 0000000..461f076 --- /dev/null +++ b/batch_encoder/_seek_collector.py @@ -0,0 +1,133 @@ +from ._seek import Seek +from ._utils import string_to_seconds + +import logging +import re + + +# The collection of positions that we will use to seek within the source file +# These are the validated sets of cuts for our encodes +class SeekCollector: + # Time Duration Specification: https://ffmpeg.org/ffmpeg-utils.html#time-duration-syntax + time_pattern = re.compile('^([0-5]?\d:){1,2}[0-5]?\d(?=\.\d+$|$)|\d+(?=\.\d+$|$)') + + # AnimeThemes File Name Convention: '[Title]-{OP|ED}#v#' + # Examples: Tamayura-OP1, PlanetWith-ED1v2 + filename_pattern = re.compile('^[a-zA-Z0-9\-]+$') + + def __init__(self, source_file): + self.source_file = source_file + self.start_positions = SeekCollector.prompt_time('Start time(s): ') + self.end_positions = SeekCollector.prompt_time('End time(s): ') + self.output_names = SeekCollector.prompt_output_name() + + # Prompt the user for our list of starting/ending positions of our WebMs + # For starting positions, a blank input value is the 0 position of the source file + # For ending positions, a blank input value is the end position of the source file + @staticmethod + def prompt_time(prompt_text): + while True: + invalid_time = False + positions = input(prompt_text).split(',') + for position in positions: + if len(position) > 0 and not SeekCollector.time_pattern.match(position): + logging.error(f'\'{position}\' is not a valid time duration') + invalid_time = True + break + if not invalid_time: + return positions + + # Prompt the user for our list of name for our passlog/WebMs + @staticmethod + def prompt_output_name(): + while True: + invalid_name = False + filenames = input('Output file name(s): ').split(',') + for filename in filenames: + if not SeekCollector.filename_pattern.match(filename): + logging.error(f'\'{filename}\' is not a valid output file name') + invalid_name = True + break + if not invalid_name: + return filenames + + # Integrity Test 1: Our lists should be of equal length + def is_length_consistent(self): + length = len(self.start_positions) + return all(len(lst) == length for lst in [self.end_positions, self.output_names]) + + # Integrity Test 2: Positions should be within source file duration + def is_within_source_duration(self): + source_file_duration = float(self.source_file.file_format['format']['duration']) + + for start_position, end_position in zip(self.start_positions, self.end_positions): + start_time = string_to_seconds(start_position) if start_position else 0 + end_time = string_to_seconds(end_position) if end_position else source_file_duration + + logging.debug( + f'[SeekCollector.is_within_source_duration] start_time: \'{start_time}\', ' + f'end_time: \'{end_time}\', ' + f'source_file_duration: \'{source_file_duration}\'') + + if start_time > source_file_duration or end_time > source_file_duration: + return False + + return True + + # Integrity Test 3: Start position is before end position + def is_start_before_end(self): + source_file_duration = float(self.source_file.file_format['format']['duration']) + for start_position, end_position in zip(self.start_positions, self.end_positions): + start_time = string_to_seconds(start_position) if start_position else 0 + end_time = string_to_seconds(end_position) if end_position else source_file_duration + + logging.debug( + f'[SeekCollector.is_start_before_end] start_time: \'{start_time}\', ' + f'end_time: \'{end_time}\', ' + f'source_file_duration: \'{source_file_duration}\'') + + if start_time >= end_time: + return False + + return True + + # Integrity Test 4: Unique output names + def is_unique_output_names(self): + logging.debug( + f'[SeekCollector.is_unique_output_names] len(set): \'{len(set(self.output_names))}\', ' + f'len: \'{len(self.output_names)}\'') + + return len(set(self.output_names)) == len(self.output_names) + + # Integrity Tests with feedback + def is_valid(self): + is_valid = True + + if not self.is_length_consistent(): + is_valid = False + logging.error('Collection not of equal length') + + if not self.is_within_source_duration(): + is_valid = False + logging.error('Position greater than file duration') + + if not self.is_start_before_end(): + is_valid = False + logging.error('Start Position is not before End Position') + + if not self.is_unique_output_names(): + is_valid = False + logging.error('Output Names are not unique') + + return is_valid + + # Our list of positions, validated if called after is_valid + def get_seek_list(self): + seek_list = [] + + for start_position, end_position, output_name in zip(self.start_positions, self.end_positions, + self.output_names): + seek = Seek(self.source_file, start_position, end_position, output_name) + seek_list.append(seek) + + return seek_list diff --git a/batch_encoder/_source_file.py b/batch_encoder/_source_file.py new file mode 100644 index 0000000..9e1df34 --- /dev/null +++ b/batch_encoder/_source_file.py @@ -0,0 +1,153 @@ +import json +import logging +import os +import subprocess + + +# Abstraction of the source file from which we are producing our encodes +# We are prefetching properties of the source file audio/video streams to help determine encoding argument values +class SourceFile: + format_args = ['ffprobe', '-v', 'quiet', '-print_format', 'json', '-show_streams', '-show_format', '-show_chapters'] + + def __init__(self, file, file_format, selected_video_stream, selected_audio_stream, video_format, audio_format): + self.file = file + self.file_format = file_format + self.selected_video_stream = selected_video_stream + self.selected_audio_stream = selected_audio_stream + self.video_format = video_format + self.audio_format = audio_format + + @classmethod + def from_file(cls, file, encoding_config): + video_file = None + audio_file = None + try: + file_format = SourceFile.get_file_format(file) + + selected_video_stream = SourceFile.get_default_stream(file_format, 'video', encoding_config) + if selected_video_stream is None: + selected_video_stream = SourceFile.get_selected_stream(file_format, 'video') + + selected_audio_stream = SourceFile.get_default_stream(file_format, 'audio', encoding_config) + if selected_audio_stream is None: + selected_audio_stream = SourceFile.get_selected_stream(file_format, 'audio') + + logging.info('Retrieving extracted audio/video stream/format data...') + + video_file = '[Video]' + file + logging.debug(f'[SourceFile.from_file] video_file: \'{video_file}\'') + audio_file = '[Audio]' + file + logging.debug(f'[SourceFile.from_file] audio_file: \'{audio_file}\'') + + demux_args = ['ffmpeg', '-i', file, '-v', 'quiet', '-y', '-sn', '-dn', '-map', + f'0:v:{selected_video_stream}', '-vcodec', 'copy', video_file, '-map', + f'0:a:{selected_audio_stream}', '-acodec', 'copy', audio_file] + subprocess.call(demux_args) + + video_args = SourceFile.format_args + [video_file] + video_format = subprocess.check_output(video_args).decode('utf-8') + video_format = json.loads(video_format) + + os.remove(video_file) + logging.debug(f'[SourceFile.from_file] video_file deleted: {not os.path.isfile(video_file)}') + + audio_args = SourceFile.format_args + [audio_file] + audio_format = subprocess.check_output(audio_args).decode('utf-8') + audio_format = json.loads(audio_format) + + os.remove(audio_file) + logging.debug(f'[SourceFile.from_file] audio_file deleted: {not os.path.isfile(audio_file)}') + + return cls(file, file_format, selected_video_stream, selected_audio_stream, video_format, audio_format) + except KeyboardInterrupt: + logging.info('Attempting to delete temp files after keyboard interrupt') + + if os.path.isfile(video_file): + os.remove(video_file) + + if os.path.isfile(audio_file): + os.remove(audio_file) + + raise + + # Source file streams/formats + @staticmethod + def get_file_format(file): + logging.info('Retrieving source file stream/format data...') + format_args = SourceFile.format_args + [file] + + file_format = subprocess.check_output(format_args).decode('utf-8') + + return json.loads(file_format) + + # Get the number of streams of the codec type (audio/video) + @staticmethod + def get_stream_count(file_format, target_codec_type): + count = 0 + + for stream in file_format['streams']: + if stream['codec_type'] == target_codec_type: + count += 1 + + logging.debug(f'[SourceFile.get_stream_count] target_codec_type: \'{target_codec_type}\', count: \'{count}\'') + + return count + + # Validate default stream selection before prompting the user to specify which stream to use + @staticmethod + def get_default_stream(file_format, stream_type, encoding_config): + # Exit early if default stream is not set + default_stream = encoding_config.get_default_stream(stream_type) + if not default_stream: + return None + + stream_count = SourceFile.get_stream_count(file_format, stream_type) + try: + default_stream = int(default_stream) + if default_stream in range(stream_count): + return default_stream + logging.error(f'Default stream selection \'{default_stream}\' is invalid') + except ValueError: + logging.error(f'Default stream selection \'{default_stream}\' must be an integer') + + return None + + # If there exists more than one stream for a codec type (audio/video), + # we want the user to specify which stream to use + @staticmethod + def get_selected_stream(file_format, stream_type): + stream_count = SourceFile.get_stream_count(file_format, stream_type) + if stream_count <= 1: + return 0 + + streams = range(stream_count) + prompt_text = f'Select {stream_type} stream [0-{stream_count - 1}]: ' + while True: + try: + selected_stream = int(input(prompt_text)) + if selected_stream in streams: + return selected_stream + logging.error('Stream selection is invalid') + except ValueError: + logging.error('Stream selection must be an integer') + + # Include the source file candidate? + @staticmethod + def yes_or_no(file): + yes = {'yes', 'y', ''} + no = {'no', 'n'} + while True: + choice = input(f'Include file \'{file}\': ').lower() + if choice in yes: + return True + elif choice in no: + return False + else: + logging.error('Please respond with \'y\' or \'n\'') + + # If our source file audio stream is not a 2-channel stereo layout, we need to resample it before normalization + def apply_audio_resampling(self, audio_filters): + channels = int(self.audio_format['streams'][0].get('channels', 2)) + channel_layout = self.audio_format['streams'][0].get('channel_layout', 'stereo') + if channels != 2 or channel_layout != 'stereo': + audio_filters.append('aresample=ocl=stereo') diff --git a/batch_encoder/_utils.py b/batch_encoder/_utils.py new file mode 100644 index 0000000..0a48f0b --- /dev/null +++ b/batch_encoder/_utils.py @@ -0,0 +1,37 @@ +import argparse +import os +import sys + + +# Convert position to seconds, needed for integrity tests +def string_to_seconds(time): + return sum(x * float(t) for x, t in zip([1, 60, 3600], reversed(time.split(':')))) + + +# Validate Arguments: check that file can be written to +def file_arg_type(arg_value): + if not os.access(arg_value, os.W_OK): + try: + open(arg_value, 'w').close() + os.remove(arg_value) + except OSError: + raise argparse.ArgumentTypeError(f'File \'{arg_value}\' cannot be created') + return arg_value + + +# Validate Arguments: check that command file can be written to and is a TXT file type +# New users were providing source files for this argument and overwriting them +def commandfile_arg_type(arg_value): + file_arg_type(arg_value) + if not arg_value.endswith('.txt'): + raise argparse.ArgumentTypeError(f'Command File \'{arg_value}\' must use \'.txt\' file extension') + return arg_value + + +# Validate Arguments: check that config file can be written to and is INI file type +def configfile_arg_type(arg_value): + config_file = os.path.join(sys.path[0], arg_value) + file_arg_type(config_file) + if not arg_value.endswith('.ini'): + raise argparse.ArgumentTypeError(f'Config File \'{arg_value}\' must use \'.ini\' file extension') + return arg_value diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..311fb97 --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +from setuptools import setup, find_packages + +with open('README.md') as f: + long_description = f.read() + +setup( + name='animethemes-batch-encoder', + version='1.0', + author='paranarimasu', + author_email='paranarimasu@gmail.com', + url='https://github.com/AnimeThemes/animethemes-batch-encoder', + description='Generate/Execute FFmpeg commands for files in acting directory', + long_description=long_description, + long_description_content_type='text/markdown', + packages=find_packages(), + classifiers=[ + 'Intended Audience :: Developers', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + ], + python_requires='>=3.6', +)