From a94827698c92b04aa71b58a765c32b6d3d3babc1 Mon Sep 17 00:00:00 2001 From: skelly37 Date: Sun, 2 Mar 2025 21:47:08 +0100 Subject: [PATCH] Video converter part done --- Preprocessing/ErrorHandlingLogger.py | 57 -------- Preprocessing/VideoConverter.py | 152 -------------------- bot/handlers/bot_message_handler.py | 4 +- bot/utils/functions.py | 47 +++--- bot/utils/resolution.py | 18 +++ preprocessor/__main__.py | 112 +++++++++++++-- preprocessor/utils/__init__.py | 0 preprocessor/utils/args.py | 22 +++ preprocessor/utils/error_handling_logger.py | 50 +++++++ preprocessor/video_transcoder.py | 109 +++++++++++--- 10 files changed, 299 insertions(+), 272 deletions(-) delete mode 100644 Preprocessing/ErrorHandlingLogger.py delete mode 100644 Preprocessing/VideoConverter.py create mode 100644 bot/utils/resolution.py create mode 100644 preprocessor/utils/__init__.py create mode 100644 preprocessor/utils/args.py create mode 100644 preprocessor/utils/error_handling_logger.py diff --git a/Preprocessing/ErrorHandlingLogger.py b/Preprocessing/ErrorHandlingLogger.py deleted file mode 100644 index cb8788e0..00000000 --- a/Preprocessing/ErrorHandlingLogger.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -from typing import ( - Dict, - List, -) - - -class ErrorHandlingLogger: - class LoggerNotFinalizedException(Exception): - def __init__(self): - super().__init__("Logger destroyed without finalize() being called.") - - - CLASS_EXIT_CODES: Dict[str, int] = { - "AudioNormalizer": 1, - "AudioProcessor": 2, - "JSONProcessor": 3, - "EpisodeInfoProcessor": 4, - "VideoConverter": 5, - } - - def __init__(self, logger: logging.Logger, class_name: str) -> None: - self.logger: logger = logger - self.class_name: str = class_name - - if class_name not in self.CLASS_EXIT_CODES: - raise ValueError(f"Class name '{class_name}' not found in CLASS_EXIT_CODES mapping.") - - self.error_exit_code: int = self.CLASS_EXIT_CODES[class_name] - self.errors: List[str] = [] - self.__is_finalized: bool = False - - def __del__(self) -> None: - if not self.__is_finalized: - self.logger.error(f"ErrorHandlingLogger for '{self.class_name}' destroyed without finalize().") - if self.errors: - self.logger.error("Logged errors:") - for error in self.errors: - self.logger.error(f"- {error}") - raise self.LoggerNotFinalizedException - - def info(self, message: str) -> None: - self.logger.info(message) - - def error(self, message: str) -> None: - self.logger.error(message) - self.errors.append(message) - - def finalize(self) -> int: - self.__is_finalized = True - if self.errors: - self.logger.error(f"Processing for '{self.class_name}' completed with errors:") - for error in self.errors: - self.logger.error(f"- {error}") - return self.error_exit_code - self.logger.info(f"Processing for '{self.class_name}' completed successfully.") - return 0 diff --git a/Preprocessing/VideoConverter.py b/Preprocessing/VideoConverter.py deleted file mode 100644 index 5c6cd069..00000000 --- a/Preprocessing/VideoConverter.py +++ /dev/null @@ -1,152 +0,0 @@ -import argparse -import json -from pathlib import Path -import subprocess -import sys -from typing import Any - -from Preprocessing.ErrorHandlingLogger import ErrorHandlingLogger -from Preprocessing.utils import setup_logger -from bot.utils.functions import RESOLUTIONS - - -class VideoConverter: - DEFAULT_CODEC: str = "h264_nvenc" - DEFAULT_PRESET: str = "slow" - DEFAULT_CRF: int = 31 - DEFAULT_GOP_SIZE: float = 0.5 - - def __init__(self): - self.logger: ErrorHandlingLogger = ErrorHandlingLogger( - class_name=self.__class__.__name__, - logger=setup_logger(self.__class__.__name__), - ) - - def run(self, args: argparse.Namespace) -> int: - try: - input_dir = Path(args.input_directory) - output_dir = Path(args.output_directory) - target_resolution = self.parse_resolution(args.resolution) - - self.logger.info("Starting video conversion...") - self.convert_videos(input_dir, output_dir, target_resolution, args.codec, args.preset, args.crf, args.gop_size) - except Exception as e: # pylint: disable=broad-exception-caught - self.logger.error(f"Critical error during run: {e}") - return self.logger.finalize() - - def convert_videos( - self, - input_dir: Path, - output_dir: Path, - target_resolution: Any, - codec: str, - preset: str, - crf: int, - gop_size: float, - ) -> None: - if not input_dir.is_dir(): - self.logger.error(f"Invalid input directory: {input_dir}") - return - - for video_file in input_dir.rglob("*.mp4"): - relative_path = video_file.relative_to(input_dir) - output_path = output_dir / relative_path - output_path.parent.mkdir(parents=True, exist_ok=True) - - try: - self.convert_video(video_file, output_path, target_resolution, codec, preset, crf, gop_size) - except Exception as e: # pylint: disable=broad-exception-caught - self.logger.error(f"Error processing video {video_file}: {e}") - - def convert_video( - self, - input_file: Path, - output_file: Path, - target_resolution: Any, - codec: str, - preset: str, - crf: int, - gop_size: float, - ) -> None: - fps = self.get_video_properties(input_file) - width, height = target_resolution - vf_filter = ( - f"scale={width}:{height}:force_original_aspect_ratio=decrease," - f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black" - ) - gop = int(fps * gop_size) - - command = [ - "ffmpeg", - "-y", - "-i", str(input_file), - "-c:v", codec, - "-preset", preset, - "-profile:v", "main", - "-cq:v", str(crf), - "-g", str(gop), - "-c:a", "aac", - "-b:a", "128k", - "-ac", "2", - "-vf", vf_filter, - "-movflags", "+faststart", - str(output_file), - ] - - self.logger.info(f"Processing: {input_file} -> {output_file} with resolution {width}x{height}") - subprocess.run(command, check=True) - - @staticmethod - def get_video_properties(video_path: Path) -> float: - cmd = [ - "ffprobe", "-v", "error", - "-select_streams", "v:0", - "-show_entries", "stream=r_frame_rate", - "-of", "json", - str(video_path), - ] - result = subprocess.run(cmd, capture_output=True, text=True, check=True) - probe_data = json.loads(result.stdout) - streams = probe_data.get("streams", []) - if not streams: - raise ValueError(f"No video streams found in {video_path}") - - r_frame_rate = streams[0].get("r_frame_rate") - if not r_frame_rate: - raise ValueError(f"Frame rate not found in {video_path}") - - num, denom = (int(x) for x in r_frame_rate.split('/')) - return num / denom - - @staticmethod - def parse_resolution(resolution: str) -> Any: - if resolution not in RESOLUTIONS: - raise ValueError(f"Invalid resolution {resolution}. Choose from: {list(RESOLUTIONS.keys())}") - return RESOLUTIONS[resolution] - - -def main() -> None: - parser = argparse.ArgumentParser(description="Convert .mp4 videos to a specific resolution with black bars.") - parser.add_argument("input_directory", type=Path, help="Path to the input directory containing videos.") - parser.add_argument("output_directory", type=Path, help="Path to the output directory for converted videos.") - parser.add_argument( - "--resolution", - type=str, - default="1080p", - choices=RESOLUTIONS.keys(), - help="Target resolution for all videos.", - ) - parser.add_argument("--codec", type=str, default=VideoConverter.DEFAULT_CODEC, help="Video codec.") - parser.add_argument("--preset", type=str, default=VideoConverter.DEFAULT_PRESET, help="FFmpeg preset.") - parser.add_argument("--crf", type=int, default=VideoConverter.DEFAULT_CRF, help="Quality (lower = better).") - parser.add_argument("--gop-size", type=float, default=VideoConverter.DEFAULT_GOP_SIZE, help="Keyframe interval in seconds.") - - args = parser.parse_args() - - converter = VideoConverter() - exit_code = converter.run(args) - sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/bot/handlers/bot_message_handler.py b/bot/handlers/bot_message_handler.py index c70b90e7..b8778013 100644 --- a/bot/handlers/bot_message_handler.py +++ b/bot/handlers/bot_message_handler.py @@ -34,7 +34,7 @@ get_video_sent_log_message, ) from bot.settings import settings -from bot.utils.functions import RESOLUTIONS +from bot.utils.resolution import Resolution from bot.utils.log import ( log_system_message, log_user_activity, @@ -126,7 +126,7 @@ async def _answer_video(self, message: Message, file_path: Path) -> None: ) await self._answer(message, await self.get_response(RK.CLIP_SIZE_EXCEEDED, as_parent=True)) else: - resolution = RESOLUTIONS.get(settings.DEFAULT_RESOLUTION_KEY) + resolution = Resolution.from_str(settings.DEFAULT_RESOLUTION_KEY) await message.answer_video( FSInputFile(file_path), diff --git a/bot/utils/functions.py b/bot/utils/functions.py index 4a927b35..eecf0b5e 100644 --- a/bot/utils/functions.py +++ b/bot/utils/functions.py @@ -12,17 +12,25 @@ logger = logging.getLogger(__name__) -@dataclass -class Resolution: - width: int - height: int - -RESOLUTIONS: Dict[str, Resolution] = { - "1080p": Resolution(1920, 1080), - "720p": Resolution(1280, 720), - "480p": Resolution(854, 480), + + +NUMBER_TO_EMOJI: Dict[str, str] = { + "0": "0️⃣", + "1": "1️⃣", + "2": "2️⃣", + "3": "3️⃣", + "4": "4️⃣", + "5": "5️⃣", + "6": "6️⃣", + "7": "7️⃣", + "8": "8️⃣", + "9": "9️⃣", } + +def convert_number_to_emoji(number: int) -> str: + return "".join(NUMBER_TO_EMOJI.get(digit, digit) for digit in str(number)) + class InvalidTimeStringException(Exception): def __init__(self, time: str) -> None: self.message = f"Invalid time string: '{time}'. Upewnij się, że używasz formatu MM:SS\u200B.ms, np. 20:30.11" @@ -102,25 +110,6 @@ def format_segment(segment: json, season_info: Dict[str, int]) -> FormattedSegme ) - -number_to_emoji: Dict[str, str] = { - "0": "0️⃣", - "1": "1️⃣", - "2": "2️⃣", - "3": "3️⃣", - "4": "4️⃣", - "5": "5️⃣", - "6": "6️⃣", - "7": "7️⃣", - "8": "8️⃣", - "9": "9️⃣", -} - - -def convert_number_to_emoji(number: int) -> str: - return "".join(number_to_emoji.get(digit, digit) for digit in str(number)) - - def format_user_list(users: List[UserProfile], title: str) -> str: user_lines = [] @@ -137,7 +126,7 @@ def format_user_list(users: List[UserProfile], title: str) -> str: response += "```\n" + "\n\n".join(user_lines) + "\n```" return response -def remove_diacritics_and_lowercase(text): +def remove_diacritics_and_lowercase(text: str) -> str: normalized_text = unicodedata.normalize('NFKD', text) cleaned_text = ''.join([char for char in normalized_text if not unicodedata.combining(char)]) return cleaned_text.lower() diff --git a/bot/utils/resolution.py b/bot/utils/resolution.py new file mode 100644 index 00000000..f3a052c1 --- /dev/null +++ b/bot/utils/resolution.py @@ -0,0 +1,18 @@ +from enum import Enum + + +class Resolution(Enum): + R1080P = (1920, 1080) + R720P = (1280, 720) + R480P = (854, 480) + + def __init__(self, width: int, height: int): + self.width = width + self.height = height + + def __str__(self): + return f"{self.height}p" + + @staticmethod + def from_str(init: str) -> "Resolution": + return Resolution(Resolution["R" + init.upper()]) diff --git a/preprocessor/__main__.py b/preprocessor/__main__.py index 366eaab2..45220d3d 100644 --- a/preprocessor/__main__.py +++ b/preprocessor/__main__.py @@ -1,27 +1,121 @@ -import argparse import logging from pathlib import Path from transciption_generator import TranscriptionGenerator from video_transcoder import VideoTranscoder +from preprocessor.utils.args import( + parse_multi_mode_args, + ParserModes, +) +from bot.utils.resolution import Resolution +def generate_parser_modes() -> ParserModes: + parser_modes = { + "transcribe": [ + ("videos", { + "type": Path, + "help": "Path to input videos for preprocessing"} + ), + + ("--transcription_jsons_dir", { + "type": Path, + "default": "", # todo + "help": "Path for output transcriptions JSONs" + }), + + # todo + + ], + + "transcode": [ + ("videos", { + "type": Path, + "help": "Path to input videos for preprocessing" + }), + + ("--transcoded_videos_dir", { + "type": Path, + "default": VideoTranscoder.DEFAULT_OUTPUT_DIR, + "help": "Path for output videos after transcoding"} + ), + ("--resolution", { + "type": lambda x: Resolution[x.upper()], + "choices": list(Resolution), + "default": Resolution.R1080P, + "help": "Target resolution for all videos." + }), + ("--codec", { + "type": str, + "default": VideoTranscoder.DEFAULT_CODEC, + "help": "Video codec." + }), + ("--preset", { + "type": str, + "default": VideoTranscoder.DEFAULT_PRESET, + "help": "FFmpeg preset." + }), + ("--crf", { + "type": int, + "default": VideoTranscoder.DEFAULT_CRF, + "help": "Quality (lower = better)." + }), + ("--gop-size", { + "type": float, + "default": VideoTranscoder.DEFAULT_GOP_SIZE, + "help": "Keyframe interval in seconds." + }), + ], + } + + unique_flag_names = set() + unique_flags = [] + + for flags in parser_modes.values(): + for name, flag in flags: + if name not in unique_flag_names: + unique_flag_names.add(name) + unique_flags.append((name, flag)) + + parser_modes["all"] = unique_flags + + return parser_modes + if __name__ == "__main__": logging.basicConfig(format="%(asctime)s | %(levelname)s | %(message)s", level=logging.DEBUG) - parser = argparse.ArgumentParser() - parser.add_argument("videos", type=Path, help="Path to input videos for preprocessing") - # 2 subparsers to split stuff + args = parse_multi_mode_args( + description="Generate JSON audio transcriptions or transcode videos to an acceptable resolution.", + modes = generate_parser_modes() + ) + + + mode_workers = { + "all": [ + VideoTranscoder, + TranscriptionGenerator + ], + + "transcode": [ + VideoTranscoder, + ], + + "transcribe": [ + TranscriptionGenerator, + ] + } + + print(args) + + # add defaults from classes here - parser.add_argument("--transcoded-videos-dir", "-v", type=Path, default="transcoded_videos", help="Path for output videos after transcoding") - parser.add_argument("--transcription-jsons-dir", "-j", type=Path, default="transcriptions", help="Path for output transcriptions JSONs") - args = parser.parse_args() + exit(0) - TranscriptionGenerator(args.videos, args.transcription_jsons_dir).transcribe() - VideoTranscoder(args.videos, args.transcoded_videos_dir).transcode() + #TranscriptionGenerator(args.videos, args.transcription_jsons_dir).transcribe() + #VideoTranscoder(args.videos, args.transcoded_videos_dir).transcode() # pass transcriptions to elastic # split two paths to be async diff --git a/preprocessor/utils/__init__.py b/preprocessor/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/preprocessor/utils/args.py b/preprocessor/utils/args.py new file mode 100644 index 00000000..b5554278 --- /dev/null +++ b/preprocessor/utils/args.py @@ -0,0 +1,22 @@ +import argparse +from typing import ( + Dict, + List, + Tuple, +) + +import json + + +Argument = Tuple[str, Dict[str, str]] +ParserModes = Dict[str, List[Argument]] +def parse_multi_mode_args(description: str, modes: ParserModes) -> json: + parser = argparse.ArgumentParser(description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + subparsers = parser.add_subparsers(dest="mode", required=True, help="Choose mode") + + for mode, flags in modes.items(): + subparser = subparsers.add_parser(mode, help=f"Mode: {mode}", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + for flag, options in flags: + subparser.add_argument(flag, **options) + + return vars(parser.parse_args()) diff --git a/preprocessor/utils/error_handling_logger.py b/preprocessor/utils/error_handling_logger.py new file mode 100644 index 00000000..0328ca3d --- /dev/null +++ b/preprocessor/utils/error_handling_logger.py @@ -0,0 +1,50 @@ +import logging +from typing import List + +class ErrorHandlingLogger: + class LoggerNotFinalizedException(Exception): + def __init__(self): + super().__init__("Logger destroyed without finalize() being called.") + + + def __init__(self, class_name: str, loglevel: int, error_exit_code: int) -> None: + self.__class_name: str = class_name + self.__error_exit_code: int = error_exit_code + + self.__errors: List[str] = [] + self.__is_finalized: bool = False + + self.__setup_logger(loglevel) + + def __del__(self) -> None: + if not self.__is_finalized: + self.__logger.error(f"ErrorHandlingLogger for '{self.__class_name}' destroyed without finalize().") + if self.__errors: + self.__logger.error("Logged errors:") + for error in self.__errors: + self.__logger.error(f"- {error}") + raise self.LoggerNotFinalizedException + + def __setup_logger(self, level: int) -> None: + logging.basicConfig( + format="%(asctime)s | %(name)s | %(levelname)s | %(message)s", + level=level, + ) + self.__logger: logging.Logger = logging.getLogger(self.__class_name) + + def info(self, message: str) -> None: + self.__logger.info(message) + + def error(self, message: str) -> None: + self.__logger.error(message) + self.__errors.append(message) + + def finalize(self) -> int: + self.__is_finalized = True + if self.__errors: + self.__logger.error(f"Processing for '{self.__class_name}' completed with errors:") + for error in self.__errors: + self.__logger.error(f"- {error}") + return self.__error_exit_code + self.__logger.info(f"Processing for '{self.__class_name}' completed successfully.") + return 0 \ No newline at end of file diff --git a/preprocessor/video_transcoder.py b/preprocessor/video_transcoder.py index 6f0149ea..d63698d3 100644 --- a/preprocessor/video_transcoder.py +++ b/preprocessor/video_transcoder.py @@ -1,37 +1,100 @@ from pathlib import Path +import subprocess +import json +import logging + +from bot.utils.resolution import Resolution +from preprocessor.utils.error_handling_logger import ErrorHandlingLogger + class VideoTranscoder: + DEFAULT_OUTPUT_DIR: Path = "transcoded_videos" + DEFAULT_RESOLUTION: Resolution = Resolution.R1080P DEFAULT_CODEC: str = "h264_nvenc" DEFAULT_PRESET: str = "slow" DEFAULT_CRF: int = 31 DEFAULT_GOP_SIZE: float = 0.5 - def __init__( - self, - input_videos: Path, - output_videos: Path, - codec: str = DEFAULT_CODEC, - preset: str = DEFAULT_PRESET, - crf: int = DEFAULT_CRF, - gop_size: float = DEFAULT_GOP_SIZE, - ): - self.__input_videos = input_videos - self.__output_videos = output_videos - self.__codec = codec - self.__preset = preset - self.__crf = crf - self.__gop_size = gop_size + def __init__(self, args: json): + self.__input_videos: Path = Path(args["input_videos"]) + self.__output_videos: Path = Path(args["output_videos"]) + self.__resolution: Resolution = Resolution.from_str(args["resolution"]) + + self.__codec: str = str(args["codec"]) + self.__preset: str = str(args["preset"]) + self.__crf: int = int(args["crf"]) + self.__gop_size: float = float(args["gop_size"]) + + if not self.__input_videos.is_dir(): + raise NotADirectoryError(f"Input videos is not a directory: '{self.__input_videos}'") + + self.logger: ErrorHandlingLogger = ErrorHandlingLogger( + class_name=self.__class__.__name__, + loglevel=logging.DEBUG, + error_exit_code=1, + ) + + + def transcode(self) -> int: + for video_file in self.__input_videos.rglob("*.mp4"): + output_path = self.__output_videos / video_file.relative_to(self.__input_videos) + output_path.parent.mkdir(parents=True, exist_ok=True) + + try: + self.__process_video(video_file, output_path) + except Exception as e: # pylint: disable=broad-exception-caught + self.logger.error(f"Error processing video {video_file}: {e}") + + return self.logger.finalize() + + def __process_video(self, video: Path, output: Path) -> None: + fps = self.__get_framerate(video) + + vf_filter = ( + f"scale={self.__resolution.width}:{self.__resolution.height}:force_original_aspect_ratio=decrease," + f"pad={self.__resolution.width}:{self.__resolution.height}:(ow-iw)/2:(oh-ih)/2:black" + ) + + command = [ + "ffmpeg", + "-y", + "-i", str(video), + "-c:v", self.__codec, + "-preset", self.__preset, + "-profile:v", "main", + "-cq:v", str(self.__crf), + "-g", str(int(fps * self.__gop_size)), + "-c:a", "aac", + "-b:a", "128k", + "-ac", "2", + "-vf", vf_filter, + "-movflags", "+faststart", + str(output), + ] + self.logger.info(f"Processing [{self.__resolution}]: {video} -> {output}") + subprocess.run(command, check=True) - # video converter + @staticmethod + def __get_framerate(video: Path) -> float: + cmd = [ + "ffprobe", "-v", "error", + "-select_streams", "v:0", + "-show_entries", "stream=r_frame_rate", + "-of", "json", + str(video), + ] + result = subprocess.run(cmd, capture_output=True, text=True, check=True) - def transcode(self): - self.__prepare_videos() - self.__do_transcoding() + probe_data = json.loads(result.stdout) + streams = probe_data.get("streams") + if not streams: + raise ValueError(f"No video streams found in {video}") - def __prepare_videos(self) -> None: - pass + r_frame_rate = streams[0].get("r_frame_rate") + if not r_frame_rate: + raise ValueError(f"Frame rate not found in {video}") - def __do_transcoding(self) -> None: - pass \ No newline at end of file + num, denom = (int(x) for x in r_frame_rate.split('/')) + return num / denom \ No newline at end of file