From 8b6355a3cf3d333f8456d2be460bfab0957c7c95 Mon Sep 17 00:00:00 2001 From: BuriedInCode <6057651+Buried-In-Code@users.noreply.github.com> Date: Mon, 5 Aug 2024 16:44:29 +1200 Subject: [PATCH] Metron updates (#26) - Update with new MetronInfo format statuses - Tidy up main - Unlink THEN move when converting archives --- .pre-commit-config.yaml | 2 +- perdoo/__init__.py | 2 +- perdoo/__main__.py | 187 +++++++++++++++-------------------- perdoo/archives/cb7.py | 2 +- perdoo/archives/cbt.py | 2 +- perdoo/archives/cbz.py | 2 +- perdoo/models/comic_info.py | 10 +- perdoo/models/metadata.py | 11 ++- perdoo/models/metron_info.py | 10 +- perdoo/services/marvel.py | 4 +- perdoo/services/metron.py | 2 +- pyproject.toml | 2 +- requirements-dev.lock | 4 +- requirements.lock | 4 +- 14 files changed, 109 insertions(+), 135 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a483f03..6747345 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.2 + rev: v0.5.6 hooks: - id: ruff-format - id: ruff diff --git a/perdoo/__init__.py b/perdoo/__init__.py index 9e32e98..b7a5abf 100644 --- a/perdoo/__init__.py +++ b/perdoo/__init__.py @@ -7,7 +7,7 @@ "get_data_root", "setup_logging", ] -__version__ = "0.2.1" +__version__ = "0.3.0" import logging import os diff --git a/perdoo/__main__.py b/perdoo/__main__.py index e17a3e9..cc1e13f 100644 --- a/perdoo/__main__.py +++ b/perdoo/__main__.py @@ -7,6 +7,7 @@ from pathlib import Path from platform import python_version from tempfile import TemporaryDirectory +from typing import cast from pydantic import ValidationError from rich.prompt import Prompt @@ -39,8 +40,7 @@ def convert_collection(path: Path, output: OutputFormat) -> None: OutputFormat.CB7: (".cb7", CB7Archive), OutputFormat.CBT: (".cbt", CBTArchive), }.get(output, (".cbz", CBZArchive)) - formats = list(ARCHIVE_EXTENSIONS) - formats.remove(format_) + formats = [ext for ext in ARCHIVE_EXTENSIONS if ext != format_] for file in list_files(path, *formats): with CONSOLE.status( f"Converting {file.name} to {output.name}", spinner="simpleDotsScrolling" @@ -57,102 +57,85 @@ def read_meta_file(cls: type[InfoModel], filename: str) -> InfoModel | None: return cls.from_bytes(content=archive.read_file(filename=filename)) return None - # region Read Metadata try: - metadata = read_meta_file(cls=Metadata, filename="/Metadata.xml") or read_meta_file( - cls=Metadata, filename="Metadata.xml" + metadata = read_meta_file(Metadata, "/Metadata.xml") or read_meta_file( + Metadata, "Metadata.xml" ) if metadata: + metadata = cast(Metadata, metadata) meta = metadata.meta details = Details( series=Identifications( search=metadata.issue.series.title, - comicvine=get_metadata_id( - resources=metadata.issue.series.resources, source=Source.COMICVINE - ), + comicvine=get_metadata_id(metadata.issue.series.resources, Source.COMICVINE), league=get_metadata_id( - resources=metadata.issue.series.resources, - source=Source.LEAGUE_OF_COMIC_GEEKS, - ), - marvel=get_metadata_id( - resources=metadata.issue.series.resources, source=Source.MARVEL - ), - metron=get_metadata_id( - resources=metadata.issue.series.resources, source=Source.METRON + metadata.issue.series.resources, Source.LEAGUE_OF_COMIC_GEEKS ), + marvel=get_metadata_id(metadata.issue.series.resources, Source.MARVEL), + metron=get_metadata_id(metadata.issue.series.resources, Source.METRON), ), issue=Identifications( search=metadata.issue.number, - comicvine=get_metadata_id( - resources=metadata.issue.resources, source=Source.COMICVINE - ), - league=get_metadata_id( - resources=metadata.issue.resources, source=Source.LEAGUE_OF_COMIC_GEEKS - ), - marvel=get_metadata_id( - resources=metadata.issue.resources, source=Source.MARVEL - ), - metron=get_metadata_id( - resources=metadata.issue.resources, source=Source.METRON - ), + comicvine=get_metadata_id(metadata.issue.resources, Source.COMICVINE), + league=get_metadata_id(metadata.issue.resources, Source.LEAGUE_OF_COMIC_GEEKS), + marvel=get_metadata_id(metadata.issue.resources, Source.MARVEL), + metron=get_metadata_id(metadata.issue.resources, Source.METRON), ), ) return meta, details except ValidationError: LOGGER.error("%s contains an invalid Metadata file", archive.path.name) # noqa: TRY400 - # endregion - # region Read MetronInfo try: - metron_info = read_meta_file(cls=MetronInfo, filename="/MetronInfo.xml") or read_meta_file( - cls=MetronInfo, filename="MetronInfo.xml" + metron_info = read_meta_file(MetronInfo, "/MetronInfo.xml") or read_meta_file( + MetronInfo, "MetronInfo.xml" ) if metron_info: + metron_info = cast(MetronInfo, metron_info) + series_id = metron_info.series.id if metron_info.id else None + issue_id = metron_info.id.primary.value if metron_info.id else None details = Details( series=Identifications( search=metron_info.series.name, - comicvine=metron_info.series.id - if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE + comicvine=series_id + if metron_info.id.primary.source == InformationSource.COMIC_VINE else None, - league=metron_info.series.id - if metron_info.id - and metron_info.id.source == InformationSource.LEAGUE_OF_COMIC_GEEKS + league=series_id + if metron_info.id.primary.source == InformationSource.LEAGUE_OF_COMIC_GEEKS else None, - marvel=metron_info.series.id - if metron_info.id and metron_info.id.source == InformationSource.MARVEL + marvel=series_id + if metron_info.id.primary.source == InformationSource.MARVEL else None, - metron=metron_info.series.id - if metron_info.id and metron_info.id.source == InformationSource.METRON + metron=series_id + if metron_info.id.primary.source == InformationSource.METRON else None, ), issue=Identifications( search=metron_info.number, - comicvine=metron_info.id.value - if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE + comicvine=issue_id + if metron_info.id.primary.source == InformationSource.COMIC_VINE else None, - league=metron_info.id.value - if metron_info.id - and metron_info.id.source == InformationSource.LEAGUE_OF_COMIC_GEEKS + league=issue_id + if metron_info.id.primary.source == InformationSource.LEAGUE_OF_COMIC_GEEKS else None, - marvel=metron_info.id.value - if metron_info.id and metron_info.id.source == InformationSource.MARVEL + marvel=issue_id + if metron_info.id.primary.source == InformationSource.MARVEL else None, - metron=metron_info.id.value - if metron_info.id and metron_info.id.source == InformationSource.METRON + metron=issue_id + if metron_info.id.primary.source == InformationSource.METRON else None, ), ) return Meta(date_=date.today(), tool=Tool(value="MetronInfo")), details except ValidationError: LOGGER.error("%s contains an invalid MetronInfo file", archive.path.name) # noqa: TRY400 - # endregion - # region Read ComicInfo try: - comic_info = read_meta_file(cls=ComicInfo, filename="/ComicInfo.xml") or read_meta_file( - cls=ComicInfo, filename="ComicInfo.xml" + comic_info = read_meta_file(ComicInfo, "/ComicInfo.xml") or read_meta_file( + ComicInfo, "ComicInfo.xml" ) if comic_info: + comic_info = cast(ComicInfo, comic_info) details = Details( series=Identifications(search=comic_info.series), issue=Identifications(search=comic_info.number), @@ -160,7 +143,6 @@ def read_meta_file(cls: type[InfoModel], filename: str) -> InfoModel | None: return Meta(date_=date.today(), tool=Tool(value="ComicInfo")), details except ValidationError: LOGGER.error("%s contains an invalid ComicInfo file", archive.path.name) # noqa: TRY400 - # endregion return None, None @@ -173,67 +155,58 @@ def load_archives( archive = get_archive(path=file) LOGGER.debug("Reading %s", file.stem) meta, details = read_meta(archive=archive) - if not meta or not details: - archives.append((file, archive, details)) - continue - difference = abs(date.today() - meta.date_) - if force or meta.tool != Tool() or difference.days >= 28: + if ( + not meta + or not details + or force + or meta.tool != Tool() + or abs(date.today() - meta.date_).days >= 28 + ): archives.append((file, archive, details)) - continue return archives def fetch_from_services( settings: Settings, details: Details ) -> tuple[Metadata | None, MetronInfo | None, ComicInfo | None]: - marvel = None - if settings.marvel and settings.marvel.public_key and settings.marvel.private_key: - marvel = Marvel(settings=settings.marvel) - metron = None - if settings.metron and settings.metron.username and settings.metron.password: - metron = Metron(settings=settings.metron) - comicvine = None - if settings.comicvine and settings.comicvine.api_key: - comicvine = Comicvine(settings.comicvine) - league = None - if ( - settings.league_of_comic_geeks + services = { + Service.COMICVINE: Comicvine(settings.comicvine) + if settings.comicvine and settings.comicvine.api_key + else None, + Service.LEAGUE_OF_COMIC_GEEKS: League(settings.league_of_comic_geeks) + if settings.league_of_comic_geeks and settings.league_of_comic_geeks.client_id and settings.league_of_comic_geeks.client_secret - ): - league = League(settings.league_of_comic_geeks) - if not marvel and not metron and not comicvine and not league: - LOGGER.warning("No external services configured") - return None, None, None - - services = { - Service.COMICVINE: comicvine, - Service.LEAGUE_OF_COMIC_GEEKS: league, - Service.MARVEL: marvel, - Service.METRON: metron, + else None, + Service.MARVEL: Marvel(settings.marvel) + if settings.marvel and settings.marvel.public_key and settings.marvel.private_key + else None, + Service.METRON: Metron(settings.metron) + if settings.metron and settings.metron.username and settings.metron.password + else None, } for service_name in settings.service_order: service = services[service_name] - if not service: - continue - LOGGER.info("Fetching details from %s", type(service).__name__) - metadata, metron_info, comic_info = service.fetch(details=details) - if metadata and metron_info and comic_info: - return metadata, metron_info, comic_info + if service: + LOGGER.info("Fetching details from %s", type(service).__name__) + metadata, metron_info, comic_info = service.fetch(details=details) + if metadata and metron_info and comic_info: + return metadata, metron_info, comic_info + LOGGER.warning("No external services configured or data incomplete") return None, None, None def generate_filename(root: Path, extension: str, metadata: Metadata) -> Path: - publisher_filename = metadata.issue.series.publisher.title - series_filename = ( + publisher_filename = sanitize(metadata.issue.series.publisher.title) + series_filename = sanitize( f"{metadata.issue.series.title} v{metadata.issue.series.volume}" if metadata.issue.series.volume > 1 else metadata.issue.series.title ) number_str = ( - f"_#{metadata.issue.number.zfill(3 if metadata.issue.format == Format.COMIC else 2)}" + f"_#{metadata.issue.number.zfill(3 if metadata.issue.format == Format.SINGLE_ISSUE else 2)}" if metadata.issue.number else "" ) @@ -244,19 +217,18 @@ def generate_filename(root: Path, extension: str, metadata: Metadata) -> Path: Format.HARDCOVER: "_HC", Format.TRADE_PAPERBACK: "_TP", }.get(metadata.issue.format, "") - if metadata.issue.format in {Format.ANNUAL, Format.DIGITAL_CHAPTER}: - issue_filename = sanitize(value=series_filename) + format_str + number_str - elif metadata.issue.format in {Format.GRAPHIC_NOVEL, Format.HARDCOVER, Format.TRADE_PAPERBACK}: - issue_filename = sanitize(value=series_filename) + number_str + format_str + + if metadata.issue.format in { + Format.GRAPHIC_NOVEL, + Format.HARDCOVER, + Format.TRADE_PAPERBACK, + Format.OMNIBUS, + }: + issue_filename = f"{series_filename}{number_str}{format_str}" else: - issue_filename = sanitize(value=series_filename) + number_str + issue_filename = f"{series_filename}{format_str}{number_str}" - return ( - root - / sanitize(value=publisher_filename) - / sanitize(value=series_filename) - / f"{issue_filename}.{extension}" - ) + return root / publisher_filename / series_filename / f"{issue_filename}.{extension}" def rename_images(folder: Path, filename: str) -> None: @@ -276,7 +248,7 @@ def process_pages( from perdoo.models.metadata import Page as MetadataPage from perdoo.models.metron_info import Page as MetronPage - rename_images(folder=folder, filename=filename) + rename_images(folder, filename) image_list = list_files(folder, *IMAGE_EXTENSIONS) metadata_pages = set() metron_info_pages = set() @@ -304,7 +276,6 @@ def process_pages( def start(settings: Settings, force: bool = False) -> None: LOGGER.info("Starting Perdoo") - convert_collection(path=settings.input_folder, output=settings.output.format) with CONSOLE.status( @@ -325,6 +296,7 @@ def start(settings: Settings, force: bool = False) -> None: if not metadata: LOGGER.warning("Not enough information to organize and rename this comic, skipping") continue + new_file = generate_filename( root=settings.output_folder, extension=settings.output.format.value, metadata=metadata ) @@ -344,6 +316,7 @@ def start(settings: Settings, force: bool = False) -> None: filename=new_file.stem, ) metadata.meta = Meta(date_=date.today()) + files = list_files(temp_folder, *IMAGE_EXTENSIONS) if settings.output.create_metadata: metadata_file = temp_folder / "Metadata.xml" @@ -357,6 +330,7 @@ def start(settings: Settings, force: bool = False) -> None: comic_info_file = temp_folder / "ComicInfo.xml" comic_info.to_file(file=comic_info_file) files.append(comic_info_file) + status.update(f"Archiving {new_file.stem}") archive_file = archive.archive_files( src=temp_folder, output_name=archive.path.stem, files=files @@ -366,6 +340,7 @@ def start(settings: Settings, force: bool = False) -> None: continue archive.path.unlink(missing_ok=True) shutil.move(archive_file, archive.path) + if file.relative_to(settings.input_folder) != new_file.relative_to(settings.output_folder): LOGGER.info( "Organizing comic, moving file to %s", new_file.relative_to(settings.output_folder) diff --git a/perdoo/archives/cb7.py b/perdoo/archives/cb7.py index 0b786ec..4324a94 100644 --- a/perdoo/archives/cb7.py +++ b/perdoo/archives/cb7.py @@ -84,6 +84,6 @@ def convert(old_archive: BaseArchive) -> CB7Archive | None: if not archive_file: return None new_filepath = old_archive.path.parent / f"{old_archive.path.stem}.cb7" - shutil.move(archive_file, new_filepath) old_archive.path.unlink(missing_ok=True) + shutil.move(archive_file, new_filepath) return CB7Archive(path=new_filepath) diff --git a/perdoo/archives/cbt.py b/perdoo/archives/cbt.py index 0dd9df4..cfb67f6 100644 --- a/perdoo/archives/cbt.py +++ b/perdoo/archives/cbt.py @@ -79,6 +79,6 @@ def convert(old_archive: BaseArchive) -> CBTArchive | None: if not archive_file: return None new_filepath = old_archive.path.parent / f"{old_archive.path.stem}.cbt" - shutil.move(archive_file, new_filepath) old_archive.path.unlink(missing_ok=True) + shutil.move(archive_file, new_filepath) return CBTArchive(path=new_filepath) diff --git a/perdoo/archives/cbz.py b/perdoo/archives/cbz.py index 63b1295..a8af9bd 100644 --- a/perdoo/archives/cbz.py +++ b/perdoo/archives/cbz.py @@ -67,6 +67,6 @@ def convert(old_archive: BaseArchive) -> CBZArchive | None: if not archive_file: return None new_filepath = old_archive.path.parent / f"{old_archive.path.stem}.cbz" - shutil.move(archive_file, new_filepath) old_archive.path.unlink(missing_ok=True) + shutil.move(archive_file, new_filepath) return CBZArchive(path=new_filepath) diff --git a/perdoo/models/comic_info.py b/perdoo/models/comic_info.py index 737077a..08c1fdd 100644 --- a/perdoo/models/comic_info.py +++ b/perdoo/models/comic_info.py @@ -75,17 +75,17 @@ class AgeRating(Enum): ADULTS_ONLY = "Adults Only 18+" EARLY_CHILDHOOD = "Early Childhood" EVERYONE = "Everyone" - EVERYONE_18 = "Everyone 10+" + EVERYONE_10 = "Everyone 10+" G = "G" KIDS_TO_ADULTS = "Kids to Adults" M = "M" - MA = "MA15+" - MATURE = "Mature 17+" + MA15 = "MA15+" + MATURE_17 = "Mature 17+" PG = "PG" - R = "R18+" + R18 = "R18+" RATING_PENDING = "Rating Pending" TEEN = "Teen" - X = "X18+" + X18 = "X18+" @staticmethod def load(value: str) -> AgeRating: diff --git a/perdoo/models/metadata.py b/perdoo/models/metadata.py index a47ed01..c648c36 100644 --- a/perdoo/models/metadata.py +++ b/perdoo/models/metadata.py @@ -119,12 +119,13 @@ def __hash__(self: Credit) -> int: class Format(Enum): - COMIC = "Comic" - DIGITAL_CHAPTER = "Digital Chapter" ANNUAL = "Annual" - TRADE_PAPERBACK = "Trade Paperback" - HARDCOVER = "Hardcover" + DIGITAL_CHAPTER = "Digital Chapter" GRAPHIC_NOVEL = "Graphic Novel" + HARDCOVER = "Hardcover" + OMNIBUS = "Omnibus" + SINGLE_ISSUE = "Single Issue" + TRADE_PAPERBACK = "Trade Paperback" @staticmethod def load(value: str) -> Format: @@ -184,7 +185,7 @@ class Issue(PascalModel): characters: list[TitledResource] = Field(default_factory=list) cover_date: date | None = None credits: list[Credit] = Field(default_factory=list) - format: Format = Format.COMIC + format: Format = Format.SINGLE_ISSUE language: str = Field(alias="@language", default="en") locations: list[TitledResource] = Field(default_factory=list) number: str | None = None diff --git a/perdoo/models/metron_info.py b/perdoo/models/metron_info.py index f2513b0..4682053 100644 --- a/perdoo/models/metron_info.py +++ b/perdoo/models/metron_info.py @@ -101,22 +101,20 @@ def __hash__(self: Resource) -> int: class Format(Enum): ANNUAL = "Annual" + DIGITAL_CHAPTER = "Digital Chapter" GRAPHIC_NOVEL = "Graphic Novel" + HARDCOVER = "Hardcover" LIMITED_SERIES = "Limited Series" + OMNIBUS = "Omnibus" ONE_SHOT = "One-Shot" - SERIES = "Series" + SINGLE_ISSUE = "Single Issue" TRADE_PAPERBACK = "Trade Paperback" - HARDCOVER = "Hardcover" @staticmethod def load(value: str) -> Format: for entry in Format: if entry.value.replace(" ", "").casefold() == value.replace(" ", "").casefold(): return entry - # region Manual matches - if value.casefold() in ["Cancelled Series".casefold(), "Ongoing Series".casefold()]: - return Format.SERIES - # endregion raise ValueError(f"`{value}` isn't a valid metron_info.Format") def __lt__(self: Format, other) -> int: # noqa: ANN001 diff --git a/perdoo/services/marvel.py b/perdoo/services/marvel.py index 3256132..9815725 100644 --- a/perdoo/services/marvel.py +++ b/perdoo/services/marvel.py @@ -117,7 +117,7 @@ def load_format(value: str) -> Format: try: return Format.load(value=value.strip()) except ValueError: - return Format.COMIC + return Format.SINGLE_ISSUE from perdoo.models.metadata import ( Credit, @@ -172,7 +172,7 @@ def load_format(value: str) -> Format: try: return Format.load(value=value.strip()) except ValueError: - return Format.SERIES + return Format.SINGLE_ISSUE def load_age_rating(value: str) -> AgeRating: try: diff --git a/perdoo/services/metron.py b/perdoo/services/metron.py index 6582f97..11db463 100644 --- a/perdoo/services/metron.py +++ b/perdoo/services/metron.py @@ -150,7 +150,7 @@ def load_format(value: str) -> Format: try: return Format.load(value=value.strip()) except ValueError: - return Format.COMIC + return Format.SINGLE_ISSUE from perdoo.models.metadata import ( Credit, diff --git a/pyproject.toml b/pyproject.toml index a62b57a..6f3e800 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "esak >= 1.3.2", "eval-type-backport >= 0.2.0 ; python_version < \"3.10\"", "himon >= 0.6.1", - "mokkari >= 3.1.0 ; python_version >= \"3.10\"", + "mokkari >= 3.2.0 ; python_version >= \"3.10\"", "mokkari@git+https://github.com/Buried-In-Code/mokkari ; python_version < \"3.10\"", "natsort >= 8.4.0", "pillow >= 10.4.0", diff --git a/requirements-dev.lock b/requirements-dev.lock index de1b34b..b7178d8 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -45,7 +45,7 @@ marshmallow==3.21.3 # via esak mdurl==0.1.2 # via markdown-it-py -mokkari @ git+https://github.com/Buried-In-Code/mokkari@e55697cb7eb0d52d206cd5df5af300752d86dd38 +mokkari @ git+https://github.com/Buried-In-Code/mokkari@7ce05cbde523969b4feb7b36698682711cef2d11 # via perdoo multivolumefile==0.2.3 # via py7zr @@ -81,7 +81,7 @@ pyppmd==1.1.0 # via py7zr pyyaml==6.0.1 # via pre-commit -pyzstd==0.16.0 +pyzstd==0.16.1 # via py7zr rarfile==4.2 # via perdoo diff --git a/requirements.lock b/requirements.lock index c3b2889..f3fab85 100644 --- a/requirements.lock +++ b/requirements.lock @@ -37,7 +37,7 @@ marshmallow==3.21.3 # via esak mdurl==0.1.2 # via markdown-it-py -mokkari @ git+https://github.com/Buried-In-Code/mokkari@e55697cb7eb0d52d206cd5df5af300752d86dd38 +mokkari @ git+https://github.com/Buried-In-Code/mokkari@7ce05cbde523969b4feb7b36698682711cef2d11 # via perdoo multivolumefile==0.2.3 # via py7zr @@ -66,7 +66,7 @@ pygments==2.18.0 # via rich pyppmd==1.1.0 # via py7zr -pyzstd==0.16.0 +pyzstd==0.16.1 # via py7zr rarfile==4.2 # via perdoo