From 0006602000da07fc2bd5e680715ddc9bb384c6ea Mon Sep 17 00:00:00 2001 From: BuriedInCode <6057651+Buried-In-Code@users.noreply.github.com> Date: Tue, 19 Mar 2024 20:24:22 +1300 Subject: [PATCH] Metron Service (#8) - Add Metron Implementation - Change urls to Pydantic's HttpUrl - Check for Info file that may have a leading "/" - Add more Manual matches for MetronInfo.Format --- .pre-commit-config.yaml | 2 +- perdoo/__main__.py | 45 +++- perdoo/models/comic_info.py | 12 +- perdoo/models/metadata.py | 8 +- perdoo/models/metron_info.py | 21 +- perdoo/services/_base.py | 54 ++++- perdoo/services/comicvine.py | 74 +++--- perdoo/services/league.py | 9 + perdoo/services/marvel.py | 9 + perdoo/services/metron.py | 441 ++++++++++++++++++++++++++++++++++- perdoo/utils.py | 19 +- pyproject.toml | 14 +- 12 files changed, 632 insertions(+), 76 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 574d3e4..5fff20f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.3.3 hooks: - id: ruff-format - id: ruff diff --git a/perdoo/__main__.py b/perdoo/__main__.py index 48054c1..cc38951 100644 --- a/perdoo/__main__.py +++ b/perdoo/__main__.py @@ -8,6 +8,8 @@ from platform import python_version from tempfile import TemporaryDirectory +from pydantic import ValidationError + from perdoo import ARCHIVE_EXTENSIONS, IMAGE_EXTENSIONS, __version__, setup_logging from perdoo.archives import BaseArchive, CB7Archive, CBTArchive, CBZArchive, get_archive from perdoo.console import CONSOLE @@ -53,14 +55,33 @@ def convert_collection(path: Path, output: OutputFormat) -> None: def read_archive(archive: BaseArchive) -> tuple[Metadata, MetronInfo, ComicInfo]: filenames = archive.list_filenames() metadata = None - if "Metadata.xml" in filenames: - metadata = Metadata.from_bytes(content=archive.read_file(filename="Metadata.xml")) + try: + if "/Metadata.xml" in filenames: + metadata = Metadata.from_bytes(content=archive.read_file(filename="/Metadata.xml")) + elif "Metadata.xml" in filenames: + metadata = Metadata.from_bytes(content=archive.read_file(filename="Metadata.xml")) + except ValidationError: + LOGGER.error("%s contains an invalid Metadata file", archive.path.name) # noqa: TRY400 metron_info = None - if "MetronInfo.xml" in filenames: - metron_info = MetronInfo.from_bytes(content=archive.read_file(filename="MetronInfo.xml")) + try: + if "/MetronInfo.xml" in filenames: + metron_info = MetronInfo.from_bytes( + content=archive.read_file(filename="/MetronInfo.xml") + ) + elif "MetronInfo.xml" in filenames: + metron_info = MetronInfo.from_bytes( + content=archive.read_file(filename="MetronInfo.xml") + ) + except ValidationError: + LOGGER.error("%s contains an invalid MetronInfo file", archive.path.name) # noqa: TRY400 comic_info = None - if "ComicInfo.xml" in filenames: - comic_info = ComicInfo.from_bytes(content=archive.read_file(filename="ComicInfo.xml")) + try: + if "/ComicInfo.xml" in filenames: + comic_info = ComicInfo.from_bytes(content=archive.read_file(filename="/ComicInfo.xml")) + elif "ComicInfo.xml" in filenames: + comic_info = ComicInfo.from_bytes(content=archive.read_file(filename="ComicInfo.xml")) + except ValidationError: + LOGGER.error("%s contains an invalid ComicInfo file", archive.path.name) # noqa: TRY400 if not metadata: if metron_info: @@ -80,7 +101,7 @@ def fetch_from_services( settings: Settings, metainfo: tuple[Metadata, MetronInfo, ComicInfo] ) -> None: marvel = None - if settings.marvel and settings.marvel.public_key and settings.marve.private_key: + if settings.marvel and settings.marvel.public_key and settings.marvel.private_key: marvel = Marvel(settings=settings.marvel) metron = None if settings.metron and settings.metron.username and settings.metron.password: @@ -115,7 +136,7 @@ def generate_filename(root: Path, extension: str, metadata: Metadata) -> Path: ) number_str = ( - f"_#{metadata.issue.number.zfill(3 if metadata.issue.format_ == Format.COMIC else 2)}" + f"_#{metadata.issue.number.zfill(3 if metadata.issue.format == Format.COMIC else 2)}" if metadata.issue.number else "" ) @@ -125,10 +146,10 @@ def generate_filename(root: Path, extension: str, metadata: Metadata) -> Path: Format.GRAPHIC_NOVEL: "_GN", Format.HARDCOVER: "_HC", Format.TRADE_PAPERBACK: "_TP", - }.get(metadata.issue.format_, "") - if metadata.issue.format_ in {Format.ANNUAL, Format.DIGITAL_CHAPTER}: + }.get(metadata.issue.format, "") + if metadata.issue.format in {Format.ANNUAL, Format.DIGITAL_CHAPTER}: issue_filename = sanitize(value=series_filename) + format_str + number_str - elif metadata.issue.format_ in {Format.GRAPHIC_NOVEL, Format.HARDCOVER, Format.TRADE_PAPERBACK}: + elif metadata.issue.format in {Format.GRAPHIC_NOVEL, Format.HARDCOVER, Format.TRADE_PAPERBACK}: issue_filename = sanitize(value=series_filename) + number_str + format_str else: issue_filename = sanitize(value=series_filename) + number_str @@ -145,7 +166,7 @@ def rename_images(folder: Path, filename: str) -> None: image_list = list_files(folder, *IMAGE_EXTENSIONS) pad_count = len(str(len(image_list))) for index, img_file in enumerate(image_list): - new_filename = f"{filename}-{str(index).zfill(pad_count)}{img_file.suffix}" + new_filename = f"{filename}_{str(index).zfill(pad_count)}{img_file.suffix}" if img_file.name != new_filename: LOGGER.info("Renamed %s to %s", img_file.name, new_filename) img_file.rename(folder / f"{filename}-{str(index).zfill(pad_count)}{img_file.suffix}") diff --git a/perdoo/models/comic_info.py b/perdoo/models/comic_info.py index 75877f9..4eae107 100644 --- a/perdoo/models/comic_info.py +++ b/perdoo/models/comic_info.py @@ -10,7 +10,7 @@ import xmltodict from natsort import humansorted, ns from PIL import Image -from pydantic import Field +from pydantic import Field, HttpUrl from perdoo.models._base import InfoModel, PascalModel @@ -134,7 +134,7 @@ def __str__(self: PageType) -> str: class Page(PascalModel): image: int = Field(alias="@Image") - type_: PageType = Field(alias="@Type", default=PageType.STORY) + type: PageType = Field(alias="@Type", default=PageType.STORY) double_page: bool = Field(alias="@DoublePage", default=False) image_size: int = Field(alias="@ImageSize", default=0) key: str | None = Field(alias="@Key", default=None) @@ -158,7 +158,7 @@ def __hash__(self: Page) -> int: @staticmethod def from_path(file: Path, index: int, is_final_page: bool, page: Page | None) -> Page: if page: - page_type = page.type_ + page_type = page.type elif index == 0: page_type = PageType.FRONT_COVER elif is_final_page: @@ -169,7 +169,7 @@ def from_path(file: Path, index: int, is_final_page: bool, page: Page | None) -> width, height = img.size return Page( image=index, - type_=page_type, + type=page_type, double_page=width >= height, image_size=file.stat().st_size, image_height=height, @@ -201,10 +201,10 @@ class ComicInfo(PascalModel, InfoModel): publisher: str | None = None imprint: str | None = None genre: str | None = None - web: str | None = None + web: HttpUrl | None = None page_count: int = 0 language_iso: str | None = Field(alias="LanguageISO", default=None) - format_: str | None = Field(alias="Format", default=None) + format: str | None = None black_and_white: YesNo = YesNo.UNKNOWN manga: Manga = Manga.UNKNOWN characters: str | None = None diff --git a/perdoo/models/metadata.py b/perdoo/models/metadata.py index f461b11..3d9402a 100644 --- a/perdoo/models/metadata.py +++ b/perdoo/models/metadata.py @@ -184,7 +184,7 @@ class Issue(PascalModel): characters: list[TitledResource] = Field(default_factory=list) cover_date: date | None = None credits: list[Credit] = Field(default_factory=list) - format_: Format = Field(alias="Format", default=Format.COMIC) + format: Format = Format.COMIC language: str = Field(alias="@language", default="en") locations: list[TitledResource] = Field(default_factory=list) number: str | None = None @@ -280,7 +280,7 @@ class Page(PascalModel): height: int = Field(alias="@height") index: int = Field(alias="@index") size: int = Field(alias="@size") - type_: PageType = Field(alias="@type", default=PageType.STORY) + type: PageType = Field(alias="@type", default=PageType.STORY) width: int = Field(alias="@width") def __lt__(self: Page, other) -> int: # noqa: ANN001 @@ -299,7 +299,7 @@ def __hash__(self: Page) -> int: @staticmethod def from_path(file: Path, index: int, is_final_page: bool, page: Page | None) -> Page: if page: - page_type = page.type_ + page_type = page.type elif index == 0: page_type = PageType.FRONT_COVER elif is_final_page: @@ -314,7 +314,7 @@ def from_path(file: Path, index: int, is_final_page: bool, page: Page | None) -> height=height, index=index, size=file.stat().st_size, - type_=page_type, + type=page_type, width=width, ) diff --git a/perdoo/models/metron_info.py b/perdoo/models/metron_info.py index 44ea5db..22e0704 100644 --- a/perdoo/models/metron_info.py +++ b/perdoo/models/metron_info.py @@ -27,7 +27,7 @@ import xmltodict from PIL import Image -from pydantic import Field +from pydantic import Field, HttpUrl from perdoo.models._base import InfoModel, PascalModel @@ -91,6 +91,14 @@ def load(value: str) -> Format: for entry in Format: if entry.value.replace(" ", "").casefold() == value.replace(" ", "").casefold(): return entry + # region Manual matches + if value.casefold() == "Limited Series".casefold(): + return Format.LIMITED + if value.casefold() == "Cancelled Series".casefold(): + return Format.SERIES + if value.casefold() == "Hard Cover".casefold(): + return Format.SERIES + # endregion raise ValueError(f"'{value}' isnt a valid metron_info.Format") def __lt__(self: Format, other) -> int: # noqa: ANN001 @@ -108,7 +116,7 @@ class Series(PascalModel): name: str sort_name: str | None = None volume: int | None = None - format_: Format | None = Field(alias="Format", default=None) + format: Format | None = None class Price(PascalModel): @@ -362,7 +370,7 @@ def __str__(self: PageType) -> str: class Page(PascalModel): image: int = Field(alias="@Image") - type_: PageType = Field(alias="@Type", default=PageType.STORY) + type: PageType = Field(alias="@Type", default=PageType.STORY) double_page: bool = Field(alias="@DoublePage", default=False) image_size: int = Field(alias="@ImageSize", default=0) key: str | None = Field(alias="@Key", default=None) @@ -386,7 +394,7 @@ def __hash__(self: Page) -> int: @staticmethod def from_path(file: Path, index: int, is_final_page: bool, page: Page | None) -> Page: if page: - page_type = page.type_ + page_type = page.type elif index == 0: page_type = PageType.FRONT_COVER elif is_final_page: @@ -397,7 +405,7 @@ def from_path(file: Path, index: int, is_final_page: bool, page: Page | None) -> width, height = img.size return Page( image=index, - type_=page_type, + type=page_type, double_page=width >= height, image_size=file.stat().st_size, image_height=height, @@ -428,7 +436,7 @@ class MetronInfo(PascalModel, InfoModel): gtin: GTIN | None = Field(alias="GTIN", default=None) age_rating: AgeRating = Field(default=AgeRating.UNKNOWN) reprints: list[Resource] = Field(default_factory=list) - url: str | None = Field(alias="URL", default=None) + url: HttpUrl | None = Field(alias="URL", default=None) credits: list[Credit] = Field(default_factory=list) pages: list[Page] = Field(default_factory=list) @@ -481,7 +489,6 @@ def to_file(self: MetronInfo, file: Path) -> None: with file.open("wb") as stream: xmltodict.unparse( {"MetronInfo": {k: content[k] for k in sorted(content)}}, - # {"MetronInfo": content}, output=stream, short_empty_elements=True, pretty=True, diff --git a/perdoo/services/_base.py b/perdoo/services/_base.py index 7155d27..1f572cb 100644 --- a/perdoo/services/_base.py +++ b/perdoo/services/_base.py @@ -3,11 +3,63 @@ __all__ = ["BaseService"] from abc import abstractmethod +from typing import Generic, TypeVar from perdoo.models import ComicInfo, Metadata, MetronInfo +P = TypeVar("P") +S = TypeVar("S") +C = TypeVar("C") + + +class BaseService(Generic[P, S, C]): + @abstractmethod + def _search_publishers(self: BaseService, title: str | None) -> int | None: ... + + @abstractmethod + def _get_publisher_id( + self: BaseService, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: ... + + @abstractmethod + def fetch_publisher( + self: BaseService, metadata: Metadata, metron_info: MetronInfo, comic_info: ComicInfo + ) -> P | None: ... + + @abstractmethod + def _search_series(self: BaseService, publisher_id: int, title: str | None) -> int | None: ... + + @abstractmethod + def _get_series_id( + self: BaseService, publisher_id: int, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: ... + + @abstractmethod + def fetch_series( + self: BaseService, + metadata: Metadata, + metron_info: MetronInfo, + comic_info: ComicInfo, + publisher_id: int, + ) -> S | None: ... + + @abstractmethod + def _search_issues(self: BaseService, series_id: int, number: str | None) -> int | None: ... + + @abstractmethod + def _get_issue_id( + self: BaseService, series_id: int, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: ... + + @abstractmethod + def fetch_issue( + self: BaseService, + metadata: Metadata, + metron_info: MetronInfo, + comic_info: ComicInfo, + series_id: int, + ) -> C | None: ... -class BaseService: @abstractmethod def fetch( self: BaseService, diff --git a/perdoo/services/comicvine.py b/perdoo/services/comicvine.py index c2b4a50..01863d7 100644 --- a/perdoo/services/comicvine.py +++ b/perdoo/services/comicvine.py @@ -108,7 +108,7 @@ def add_issue_to_metron_info(issue: Issue, metron_info: MetronInfo) -> None: metron_info.arcs = [Arc(id=x.id, name=x.name) for x in issue.story_arcs] metron_info.characters = [Resource(id=x.id, value=x.name) for x in issue.characters] metron_info.cover_date = ( - issue.cover_date or DatePrompt.ask("Cover Date", default=date.today()), + issue.cover_date or DatePrompt.ask("Cover Date", default=date.today(), console=CONSOLE), ) credits_ = [] for x in issue.creators: @@ -128,6 +128,7 @@ def add_issue_to_metron_info(issue: Issue, metron_info: MetronInfo) -> None: metron_info.summary = issue.summary metron_info.teams = [Resource(id=x.id, value=x.name) for x in issue.teams] metron_info.collection_title = issue.name + metron_info.url = issue.site_url def add_issue_to_comic_info(issue: Issue, comic_info: ComicInfo) -> None: @@ -142,9 +143,10 @@ def add_issue_to_comic_info(issue: Issue, comic_info: ComicInfo) -> None: comic_info.summary = issue.summary comic_info.team_list = [x.name for x in issue.teams] comic_info.title = issue.name + comic_info.web = issue.site_url -class Comicvine(BaseService): +class Comicvine(BaseService[Publisher, Volume, Issue]): def __init__(self: Comicvine, settings: ComicvineSettings): cache = SQLiteCache(path=get_cache_dir() / "simyan.sqlite", expiry=14) self.session = Simyan(api_key=settings.api_key, cache=cache) @@ -171,9 +173,9 @@ def _search_publishers(self: Comicvine, title: str | None) -> int | None: LOGGER.exception("") return None - def fetch_publisher( - self: Comicvine, metadata: Metadata, metron_info: MetronInfo, comic_info: ComicInfo - ) -> Publisher | None: + def _get_publisher_id( + self: Comicvine, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: publisher_id = next( ( x.value @@ -181,15 +183,17 @@ def fetch_publisher( if x.source == Source.COMICVINE ), None, - ) - publisher_id = publisher_id or ( + ) or ( metron_info.publisher.id if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE else None ) - publisher_id = publisher_id or self._search_publishers( - title=metadata.issue.series.publisher.title - ) + return publisher_id or self._search_publishers(title=metadata.issue.series.publisher.title) + + def fetch_publisher( + self: Comicvine, metadata: Metadata, metron_info: MetronInfo, comic_info: ComicInfo + ) -> Publisher | None: + publisher_id = self._get_publisher_id(metadata=metadata, metron_info=metron_info) if not publisher_id: return None try: @@ -233,24 +237,30 @@ def _search_series(self: Comicvine, publisher_id: int, title: str | None) -> int LOGGER.exception("") return None - def fetch_series( - self: Comicvine, - metadata: Metadata, - metron_info: MetronInfo, - comic_info: ComicInfo, - publisher_id: int, - ) -> Volume | None: + def _get_series_id( + self: Comicvine, publisher_id: int, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: series_id = next( (x.value for x in metadata.issue.series.resources if x.source == Source.COMICVINE), None - ) - series_id = series_id or ( + ) or ( metron_info.series.id if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE else None ) - series_id = series_id or self._search_series( + return series_id or self._search_series( publisher_id=publisher_id, title=metadata.issue.series.title ) + + def fetch_series( + self: Comicvine, + metadata: Metadata, + metron_info: MetronInfo, + comic_info: ComicInfo, + publisher_id: int, + ) -> Volume | None: + series_id = self._get_series_id( + publisher_id=publisher_id, metadata=metadata, metron_info=metron_info + ) if not series_id: return None try: @@ -294,6 +304,18 @@ def _search_issues(self: Comicvine, series_id: int, number: str | None) -> int | LOGGER.exception("") return None + def _get_issue_id( + self: Comicvine, series_id: int, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: + issue_id = next( + (x.value for x in metadata.issue.resources if x.source == Source.COMICVINE), None + ) or ( + metron_info.id.value + if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE + else None + ) + return issue_id or self._search_issues(series_id=series_id, number=metadata.issue.number) + def fetch_issue( self: Comicvine, metadata: Metadata, @@ -301,16 +323,8 @@ def fetch_issue( comic_info: ComicInfo, series_id: int, ) -> Issue | None: - issue_id = next( - (x.value for x in metadata.issue.resources if x.source == Source.COMICVINE), None - ) - issue_id = issue_id or ( - metron_info.id.value - if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE - else None - ) - issue_id = issue_id or self._search_issues( - series_id=series_id, number=metadata.number or metron_info.number or comic_info.number + issue_id = self._get_issue_id( + series_id=series_id, metadata=metadata, metron_info=metron_info ) if not issue_id: return None diff --git a/perdoo/services/league.py b/perdoo/services/league.py index 66e245d..3b34d56 100644 --- a/perdoo/services/league.py +++ b/perdoo/services/league.py @@ -8,6 +8,7 @@ from himon.sqlite_cache import SQLiteCache from perdoo import get_cache_dir +from perdoo.models import ComicInfo, Metadata, MetronInfo from perdoo.settings import LeagueofComicGeeks as LeagueSettings LOGGER = logging.getLogger(__name__) @@ -25,3 +26,11 @@ def __init__(self: League, settings: LeagueSettings): if not settings.access_token: LOGGER.info("Generating new access token") self.himon.access_token = settings.access_token = self.himon.generate_access_token() + + def fetch( + self: League, + metadata: Metadata, # noqa: ARG002 + metron_info: MetronInfo, # noqa: ARG002 + comic_info: ComicInfo, # noqa: ARG002 + ) -> bool: + return False diff --git a/perdoo/services/marvel.py b/perdoo/services/marvel.py index 6e45fe9..e89c481 100644 --- a/perdoo/services/marvel.py +++ b/perdoo/services/marvel.py @@ -8,6 +8,7 @@ from esak.sqlite_cache import SqliteCache from perdoo import get_cache_dir +from perdoo.models import ComicInfo, Metadata, MetronInfo from perdoo.settings import Marvel as MarvelSettings LOGGER = logging.getLogger(__name__) @@ -19,3 +20,11 @@ def __init__(self: Marvel, settings: MarvelSettings): self.esak = Esak( public_key=settings.public_key, private_key=settings.private_key, cache=cache ) + + def fetch( + self: Marvel, + metadata: Metadata, # noqa: ARG002 + metron_info: MetronInfo, # noqa: ARG002 + comic_info: ComicInfo, # noqa: ARG002 + ) -> bool: + return False diff --git a/perdoo/services/metron.py b/perdoo/services/metron.py index 11f476f..0ad256c 100644 --- a/perdoo/services/metron.py +++ b/perdoo/services/metron.py @@ -4,16 +4,455 @@ import logging +from mokkari.exceptions import ApiError +from mokkari.schemas.issue import Issue +from mokkari.schemas.publisher import Publisher +from mokkari.schemas.series import Series from mokkari.session import Session as Mokkari from mokkari.sqlite_cache import SqliteCache +from rich.prompt import Confirm, Prompt from perdoo import get_cache_dir +from perdoo.console import CONSOLE, create_menu +from perdoo.models import ComicInfo, Metadata, MetronInfo +from perdoo.models.metadata import Source +from perdoo.models.metron_info import InformationSource +from perdoo.services import BaseService from perdoo.settings import Metron as MetronSettings LOGGER = logging.getLogger(__name__) -class Metron: +def add_publisher_to_metadata(publisher: Publisher, metadata: Metadata) -> None: + from perdoo.models.metadata import Resource + + resources = set(metadata.issue.series.publisher.resources) + if publisher.cv_id: + resources.add(Resource(source=Source.COMICVINE, value=publisher.cv_id)) + resources.add(Resource(source=Source.METRON, value=publisher.id)) + metadata.issue.series.publisher.resources = list(resources) + metadata.issue.series.publisher.title = publisher.name + + +def add_publisher_to_metron_info(publisher: Publisher, metron_info: MetronInfo) -> None: + if not metron_info.id or metron_info.id.source == InformationSource.METRON: + metron_info.publisher.id = publisher.id + if not metron_info.id or metron_info.id.source == InformationSource.COMIC_VINE: + metron_info.publisher.id = publisher.cv_id + metron_info.publisher.value = publisher.name + + +def add_publisher_to_comic_info(publisher: Publisher, comic_info: ComicInfo) -> None: + comic_info.publisher = publisher.name + + +def add_series_to_metadata(series: Series, metadata: Metadata) -> None: + from perdoo.models.metadata import Resource, TitledResource + + resources = set(metadata.issue.series.resources) + if series.cv_id: + resources.add(Resource(source=Source.COMICVINE, value=series.cv_id)) + resources.add(Resource(source=Source.METRON, value=series.id)) + metadata.issue.series.resources = list(resources) + metadata.issue.series.genres = [ + TitledResource(resources=[Resource(source=Source.METRON, value=x.id)], title=x.name) + for x in series.genres + ] + metadata.issue.series.start_year = series.year_began + metadata.issue.series.title = series.name + metadata.issue.series.volume = series.volume + + +def add_series_to_metron_info(series: Series, metron_info: MetronInfo) -> None: + from perdoo.models.metron_info import Format + + if not metron_info.id or metron_info.id.source == InformationSource.METRON: + metron_info.series.id = series.id + if not metron_info.id or metron_info.id.source == InformationSource.COMIC_VINE: + metron_info.series.id = series.cv_id + metron_info.series.name = series.name + metron_info.series.sort_name = series.sort_name + metron_info.series.volume = series.volume + metron_info.series.format = Format.load(value=series.series_type.name) + + +def add_series_to_comic_info(series: Series, comic_info: ComicInfo) -> None: + comic_info.series = series.name + comic_info.volume = series.volume + comic_info.genre_list = [x.name for x in series.genres] + comic_info.format = series.series_type.name + + +def add_issue_to_metadata(issue: Issue, metadata: Metadata) -> None: + from perdoo.models.metadata import Credit, Format, Resource, StoryArc, TitledResource + + resources = set(metadata.issue.resources) + if issue.cv_id: + resources.add(Resource(source=Source.COMICVINE, value=issue.cv_id)) + resources.add(Resource(source=Source.METRON, value=issue.id)) + metadata.issue.resources = list(resources) + metadata.issue.story_arcs = [ + StoryArc(resources=[Resource(source=Source.METRON, value=x.id)], title=x.name) + for x in issue.arcs + ] + metadata.issue.characters = [ + TitledResource(resources=[Resource(source=Source.METRON, value=x.id)], title=x.name) + for x in issue.characters + ] + metadata.issue.title = issue.collection_title if issue.collection_title else None + metadata.issue.cover_date = issue.cover_date + metadata.issue.credits = [ + Credit( + creator=TitledResource( + resources=[Resource(source=Source.METRON, value=x.id)], title=x.creator + ), + roles=[ + TitledResource(resources=[Resource(source=Source.METRON, value=r.id)], title=r.name) + for r in x.role + ], + ) + for x in issue.credits + ] + metadata.issue.summary = issue.desc + metadata.issue.number = issue.number + metadata.issue.page_count = issue.page_count or metadata.issue.page_count + try: + metadata.issue.format = Format.load(value=issue.series.series_type.name) + except ValueError: + metadata.issue.format = Format.COMIC + metadata.issue.store_date = issue.store_date + metadata.issue.teams = [ + TitledResource(resources=[Resource(source=Source.METRON, value=x.id)], title=x.name) + for x in issue.teams + ] + + +def add_issue_to_metron_info(issue: Issue, metron_info: MetronInfo) -> None: + from perdoo.models.metron_info import ( + GTIN, + AgeRating, + Arc, + Credit, + Price, + Resource, + Role, + RoleResource, + Source, + ) + + if not metron_info.id or metron_info.id.source == InformationSource.METRON: + metron_info.id = Source(source=InformationSource.METRON, value=issue.id) + if not metron_info.id or metron_info.id.source == InformationSource.COMIC_VINE: + metron_info.id = Source(source=InformationSource.COMIC_VINE, value=issue.cv_id) + metron_info.arcs = [Arc(id=x.id, name=x.name) for x in issue.arcs] + metron_info.characters = [Resource(id=x.id, value=x.name) for x in issue.characters] + metron_info.collection_title = issue.collection_title if issue.collection_title else None + metron_info.cover_date = issue.cover_date + credits_ = [] + for x in issue.credits: + roles = [] + for r in x.role: + try: + roles.append(RoleResource(id=r.id, value=Role.load(value=r.name))) + except ValueError: # noqa: PERF203 + roles.append(RoleResource(id=r.id, value=Role.OTHER)) + credits_.append(Credit(creator=Resource(id=x.id, value=x.creator), roles=roles)) + metron_info.credits = credits_ + metron_info.summary = issue.desc + metron_info.number = issue.number + metron_info.page_count = issue.page_count or metron_info.page_count + metron_info.prices = [Price(country="US", value=issue.price)] if issue.price else [] + metron_info.age_rating = AgeRating.load(value=issue.rating.name) + metron_info.reprints = [Resource(id=x.id, value=x.issue) for x in issue.reprints] + metron_info.url = str(issue.resource_url) + metron_info.store_date = issue.store_date + metron_info.stories = [Resource(value=x) for x in issue.story_titles] + metron_info.teams = [Resource(id=x.id, value=x.name) for x in issue.teams] + metron_info.gtin = GTIN(upc=issue.upc) if issue.upc else None + + +def add_issue_to_comic_info(issue: Issue, comic_info: ComicInfo) -> None: + comic_info.story_arc_list = [x.name for x in issue.arcs] + comic_info.character_list = [x.name for x in issue.characters] + comic_info.title = issue.collection_title + comic_info.cover_date = issue.cover_date + comic_info.credits = {x.creator: [r.name for r in x.role] for x in issue.credits} + comic_info.summary = issue.desc + comic_info.number = issue.number + comic_info.page_count = issue.page_count or comic_info.page_count + comic_info.web = issue.resource_url + comic_info.team_list = [x.name for x in issue.teams] + + +class Metron(BaseService[Publisher, Series, Issue]): def __init__(self: Metron, settings: MetronSettings): cache = SqliteCache(db_name=str(get_cache_dir() / "mokkari.sqlite"), expire=14) self.session = Mokkari(username=settings.username, passwd=settings.password, cache=cache) + + def _get_publisher(self: Metron, comicvine_id: int) -> int | None: + try: + publisher = self.session.publishers_list({"cv_id": comicvine_id}) + if publisher and len(publisher) >= 1: + return publisher[0].id + return None + except ApiError: + LOGGER.exception("") + return None + + def _search_publishers(self: Metron, title: str | None) -> int | None: + title = title or Prompt.ask("Publisher title", console=CONSOLE) + try: + options = sorted( + self.session.publishers_list(params={"name": title}), key=lambda x: x.name + ) + if not options: + LOGGER.warning("Unable to find any publishers with the title: '%s'", title) + index = create_menu( + options=[f"{x.id} | {x.name}" for x in options], + title="Metron Publisher", + default="None of the Above", + ) + if index != 0: + return options[index - 1].id + if not Confirm.ask("Try Again", console=CONSOLE): + return None + return self._search_publishers(title=None) + except ApiError: + LOGGER.exception("") + return None + + def _get_publisher_id(self: Metron, metadata: Metadata, metron_info: MetronInfo) -> int | None: + publisher_id = next( + ( + x.value + for x in metadata.issue.series.publisher.resources + if x.source == Source.METRON + ), + None, + ) or ( + metron_info.publisher.id + if metron_info.id and metron_info.id.source == InformationSource.METRON + else None + ) + if not publisher_id: + comicvine_id = next( + ( + x.value + for x in metadata.issue.series.publisher.resources + if x.source == Source.COMICVINE + ), + None, + ) or ( + metron_info.publisher.id + if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE + else None + ) + if comicvine_id: + publisher_id = self._get_publisher(comicvine_id=comicvine_id) + return publisher_id or self._search_publishers(title=metadata.issue.series.publisher.title) + + def fetch_publisher( + self: Metron, metadata: Metadata, metron_info: MetronInfo, comic_info: ComicInfo + ) -> Publisher | None: + publisher_id = self._get_publisher_id(metadata=metadata, metron_info=metron_info) + if not publisher_id: + return None + try: + publisher = self.session.publisher(_id=publisher_id) + add_publisher_to_metadata(publisher=publisher, metadata=metadata) + add_publisher_to_metron_info(publisher=publisher, metron_info=metron_info) + add_publisher_to_comic_info(publisher=publisher, comic_info=comic_info) + return publisher + except ApiError: + LOGGER.exception("") + return None + + def _get_series(self: Metron, comicvine_id: int) -> int | None: + try: + series = self.session.series_list({"cv_id": comicvine_id}) + if series and len(series) >= 1: + return series[0].id + return None + except ApiError: + LOGGER.exception("") + return None + + def _search_series(self: Metron, publisher_id: int, title: str | None) -> int | None: + title = title or Prompt.ask("Series title", console=CONSOLE) + try: + options = sorted( + self.session.series_list(params={"publisher_id": publisher_id, "name": title}), + key=lambda x: x.display_name, + ) + if not options: + LOGGER.warning( + "Unable to find any Series with a PublisherId: %s and the title: '%s'", + publisher_id, + title, + ) + index = create_menu( + options=[f"{x.id} | {x.display_name}" for x in options], + title="Metron Series", + default="None of the Above", + ) + if index != 0: + return options[index - 1].id + if not Confirm.ask("Try Again", console=CONSOLE): + return None + return self._search_series(publisher_id=publisher_id, title=None) + except ApiError: + LOGGER.exception("") + return None + + def _get_series_id( + self: Metron, publisher_id: int, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: + series_id = next( + (x.value for x in metadata.issue.series.resources if x.source == Source.METRON), None + ) or ( + metron_info.series.id + if metron_info.id and metron_info.id.source == InformationSource.METRON + else None + ) + if not series_id: + comicvine_id = next( + (x.value for x in metadata.issue.series.resources if x.source == Source.COMICVINE), + None, + ) or ( + metron_info.series.id + if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE + else None + ) + if comicvine_id: + series_id = self._get_series(comicvine_id=comicvine_id) + return series_id or self._search_series( + publisher_id=publisher_id, title=metadata.issue.series.title + ) + + def fetch_series( + self: Metron, + metadata: Metadata, + metron_info: MetronInfo, + comic_info: ComicInfo, + publisher_id: int, + ) -> Series | None: + series_id = self._get_series_id( + publisher_id=publisher_id, metadata=metadata, metron_info=metron_info + ) + if not series_id: + return None + try: + series = self.session.series(_id=series_id) + add_series_to_metadata(series=series, metadata=metadata) + add_series_to_metron_info(series=series, metron_info=metron_info) + add_series_to_comic_info(series=series, comic_info=comic_info) + return series + except ApiError: + LOGGER.exception("") + return None + + def _get_issue(self: Metron, comicvine_id: int) -> int | None: + try: + issue = self.session.issues_list({"cv_id": comicvine_id}) + if issue and len(issue) >= 1: + return issue[0].id + return None + except ApiError: + LOGGER.exception("") + return None + + def _search_issues(self: Metron, series_id: int, number: str | None) -> int | None: + try: + options = sorted( + self.session.issues_list(params={"series_id": series_id, "number": number}) + if number + else self.session.issues_list(params={"series_id": series_id}), + key=lambda x: (x.number, x.issue_name), + ) + if not options: + LOGGER.warning( + "Unable to find any Issues with a SeriesId: %s and number: '%s'", + series_id, + number, + ) + index = create_menu( + options=[f"{x.id} | {x.issue_name}" for x in options], + title="Metron Issue", + default="None of the Above", + ) + if index != 0: + return options[index - 1].id + if number: + LOGGER.info("Searching again without the issue number") + return self._search_issues(series_id=series_id, number=None) + return None + except ApiError: + LOGGER.exception("") + return None + + def _get_issue_id( + self: Metron, series_id: int, metadata: Metadata, metron_info: MetronInfo + ) -> int | None: + issue_id = next( + (x.value for x in metadata.issue.resources if x.source == Source.METRON), None + ) or ( + metron_info.id.value + if metron_info.id and metron_info.id.source == InformationSource.METRON + else None + ) + if not issue_id: + comicvine_id = next( + (x.value for x in metadata.issue.resources if x.source == Source.COMICVINE), None + ) or ( + metron_info.id.value + if metron_info.id and metron_info.id.source == InformationSource.COMIC_VINE + else None + ) + if comicvine_id: + issue_id = self._get_issue(comicvine_id=comicvine_id) + return issue_id or self._search_issues(series_id=series_id, number=metadata.issue.number) + + def fetch_issue( + self: Metron, + metadata: Metadata, + metron_info: MetronInfo, + comic_info: ComicInfo, + series_id: int, + ) -> Issue | None: + issue_id = self._get_issue_id( + series_id=series_id, metadata=metadata, metron_info=metron_info + ) + if not issue_id: + return None + try: + issue = self.session.issue(_id=issue_id) + add_issue_to_metadata(issue=issue, metadata=metadata) + add_issue_to_metron_info(issue=issue, metron_info=metron_info) + add_issue_to_comic_info(issue=issue, comic_info=comic_info) + return issue + except ApiError: + LOGGER.exception("") + return None + + def fetch( + self: Metron, metadata: Metadata, metron_info: MetronInfo, comic_info: ComicInfo + ) -> bool: + publisher = self.fetch_publisher( + metadata=metadata, metron_info=metron_info, comic_info=comic_info + ) + if not publisher: + return False + series = self.fetch_series( + metadata=metadata, + metron_info=metron_info, + comic_info=comic_info, + publisher_id=publisher.id, + ) + if not series: + return False + issue = self.fetch_issue( + metadata=metadata, metron_info=metron_info, comic_info=comic_info, series_id=series.id + ) + if not issue: + return False + return True diff --git a/perdoo/utils.py b/perdoo/utils.py index b24cdfe..f258458 100644 --- a/perdoo/utils.py +++ b/perdoo/utils.py @@ -70,7 +70,7 @@ def metron_to_metadata(metron_info: MetronInfo) -> Metadata: LOGGER.warning(err) source = None try: - format_ = Format.load(value=metron_info.series.format_.value) + format_ = Format.load(value=metron_info.series.format.value) except ValueError as err: LOGGER.warning(err) format_ = Format.COMIC @@ -78,7 +78,7 @@ def metron_to_metadata(metron_info: MetronInfo) -> Metadata: pages = [] for x in metron_info.pages: try: - type_ = PageType.load(value=x.type_.value) + type_ = PageType.load(value=x.type.value) except ValueError as err: LOGGER.warning(err) type_ = PageType.OTHER @@ -197,7 +197,7 @@ def comic_to_metadata(comic_info: ComicInfo) -> Metadata: ) try: - format_ = Format.load(value=comic_info.format_) if comic_info.format_ else Format.COMIC + format_ = Format.load(value=comic_info.format) if comic_info.format else Format.COMIC except ValueError as err: LOGGER.warning(err) format_ = Format.COMIC @@ -205,7 +205,7 @@ def comic_to_metadata(comic_info: ComicInfo) -> Metadata: pages = [] for x in comic_info.pages: try: - type_ = PageType.load(value=x.type_.value) + type_ = PageType.load(value=x.type.value) except ValueError as err: LOGGER.warning(err) type_ = PageType.OTHER @@ -319,7 +319,7 @@ def get_primary_source( ) try: - format_ = Format.load(value=metadata.issue.format_.value) + format_ = Format.load(value=metadata.issue.format.value) except ValueError as err: LOGGER.warning(err) format_ = None @@ -367,7 +367,7 @@ def get_primary_source( pages = [] for page in metadata.pages: try: - type_ = PageType.load(value=page.type_.value) + type_ = PageType.load(value=page.type.value) except ValueError as err: LOGGER.warning(err) type_ = PageType.OTHER @@ -417,7 +417,8 @@ def get_primary_source( number=metadata.issue.number, summary=metadata.issue.summary, notes=metadata.notes, - cover_date=metadata.issue.cover_date or DatePrompt.ask("Cover date", default=date.today()), + cover_date=metadata.issue.cover_date + or DatePrompt.ask("Cover date", default=date.today(), console=CONSOLE), store_date=metadata.issue.store_date, page_count=metadata.issue.page_count, genres=genres, @@ -462,7 +463,7 @@ def metadata_to_comic(metadata: Metadata) -> ComicInfo: pages = [] for page in metadata.pages: try: - type_ = PageType.load(value=page.type_.value) + type_ = PageType.load(value=page.type.value) except ValueError as err: LOGGER.warning(err) type_ = PageType.OTHER @@ -487,7 +488,7 @@ def metadata_to_comic(metadata: Metadata) -> ComicInfo: publisher=metadata.issue.series.publisher.title, page_count=metadata.issue.page_count, language_iso=metadata.issue.language, - format_=metadata.issue.format_.value, + format_=metadata.issue.format.value, pages=pages, ) output.cover_date = metadata.issue.cover_date or metadata.issue.store_date diff --git a/pyproject.toml b/pyproject.toml index 768a3e0..7b0d605 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,16 +25,17 @@ classifiers = [ ] dependencies = [ "esak >= 1.3.2", - "eval-type-backport >= 0.1.3; python_version < \"3.10\"", + "eval-type-backport >= 0.1.3 ; python_version < \"3.10\"", "himon >= 0.5.0", - "mokkari >= 2.6.0, < 3.0.0", + "mokkari >= 3.0.0 ; python_version >= \"3.10\"", + "mokkari@git+https://github.com/Buried-In-Code/mokkari ; python_version < \"3.10\"", "natsort >= 8.4.0", "pillow >= 10.2.0", "pydantic >= 2.6.4", "rarfile >= 4.1", "rich >= 13.7.1", - "simyan >= 1.2.0", - "tomli >= 2.0.1; python_version < \"3.11\"", + "simyan >= 1.2.1", + "tomli >= 2.0.1 ; python_version < \"3.11\"", "tomli-w >= 1.0.0", "xmltodict >= 0.13.0" ] @@ -51,7 +52,7 @@ cb7 = [ "py7zr >= 0.21.0" ] dev = [ - "pre-commit >= 3.5.0, < 3.6.0" + "pre-commit >= 3.5.0" ] [project.scripts] @@ -63,6 +64,9 @@ Homepage = "https://pypi.org/project/Perdoo" Issues = "https://github.com/ComicCorps/Perdoo/issues" Source = "https://github.com/ComicCorps/Perdoo" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] path = "perdoo/__init__.py"