diff --git a/.github/workflows/tests.yml b/.github/workflows/async-tests.yml similarity index 79% rename from .github/workflows/tests.yml rename to .github/workflows/async-tests.yml index fb9c399..0c3d4fe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/async-tests.yml @@ -1,4 +1,8 @@ -name: EPorner API test +name: Async API test + +permissions: + contents: read + pull-requests: write on: push: @@ -20,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest + pip install pytest pytest-asyncio pip install . - name: Test with pytest run: | diff --git a/README.md b/README.md index 8d8955c..45d04e6 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,10 @@
Downloads + Downloads Async CodeQL Analysis - API Tests + Sync API Tests +Async API Tests
# Description @@ -33,9 +35,6 @@ EPorner API is an API for EPorner, which allows you to fetch information from vi More will be coming in the next versions! -> [!NOTE] -> GitHub tests are failing for some weird reason, but the API itself is perfectly working! - # Quickstart ### Have a look at the [Documentation](https://github.com/EchterAlsFake/API_Docs/blob/master/Porn_APIs/EPorner.md) for more details diff --git a/README/Changelog.md b/README/Changelog.md index 6acd989..b466666 100644 --- a/README/Changelog.md +++ b/README/Changelog.md @@ -49,4 +49,11 @@ Quality.BEST would translate to "best" - Fixed #4, but I am not entirely if this issue is really completely fixed lol - updated to eaf base api v2 - fixed pornstar pagination -- fixed quality selection \ No newline at end of file +- fixed quality selection + +# 1.8.1 - 1.8.4 +- async support +- fixed downloading +- fixed quality selection +- fixed tests +- added async tests \ No newline at end of file diff --git a/eporner_api/eporner_api.py b/eporner_api/eporner_api.py index bd38b6f..331e03f 100644 --- a/eporner_api/eporner_api.py +++ b/eporner_api/eporner_api.py @@ -1,4 +1,4 @@ -import html +import asyncio import json import logging import argparse @@ -22,7 +22,7 @@ from bs4 import BeautifulSoup from urllib.parse import urljoin from base_api.base import BaseCore -from typing import Generator, Union +from typing import Union, List from functools import cached_property """ @@ -64,21 +64,84 @@ def disable_logging(): logger.setLevel(logging.CRITICAL) +def extract_json_from_html(html_content): + soup = BeautifulSoup(html_content, 'html.parser') + script_tags = soup.find_all('script', {'type': 'application/ld+json'}) + + combined_data = {} + + for script in script_tags: + json_text = script.string.strip() + try: + data = json.loads(json_text) + + except json.decoder.JSONDecodeError: + raise InvalidVideo(""" +JSONDecodeError: I need your help to fix this error. Please report the URL you've used on GitHub. Thanks :)""") + + combined_data.update(data) + + cleaned_dictionary =flatten_json(combined_data) + return cleaned_dictionary + + +def flatten_json(nested_json, parent_key='', sep='_'): + """ + Flatten a nested JSON dictionary. Duplicate keys will be overridden. + + :param nested_json: The nested JSON dictionary to be flattened. + :param parent_key: The base key to use for the flattened keys. + :param sep: The separator between nested keys. + :return: A flattened dictionary. + """ + items = [] + for k, v in nested_json.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_json(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + class Video: - def __init__(self, url: str, enable_html_scraping: bool = False): + def __init__(self, url: str, enable_html_scraping: bool = False, content: str = None, json_data: dict = None, + html_json_data: dict = None) -> None: self.url = url self.enable_html = enable_html_scraping - self.html_content = None - self.json_data = self.raw_json_data() - if self.enable_html: - self.request_html_content() - is_removed = REGEX_VIDEO_DISABLED.findall(self.html_content) + self.html_content = content + self.html_json_data = html_json_data + self.json_data = json_data + + @classmethod + async def create(cls, url: str, enable_html_scraping: bool = False): + if enable_html_scraping: + html_content = await core.fetch(url) + is_removed = REGEX_VIDEO_DISABLED.findall(html_content) for _ in is_removed: if _ == "deletedfile": raise VideoDisabled("Video has been removed because of a Copyright claim") - self.html_json_data = self.extract_json_from_html() + if str(url).startswith("https://"): + video_id = REGEX_ID.search(url) + if video_id: + id = video_id.group(1) + + else: + video_id = REGEX_ID_ALTERNATE.search(url) + id = video_id.group(1) + + data = await core.fetch(f"{ROOT_URL}{API_VIDEO_ID}?id={id}&thumbsize=medium&format=json") + json_data = json.loads(data) + html_json_data = extract_json_from_html(html_content) + + else: + html_content = None + data = await core.fetch(f"{ROOT_URL}{API_VIDEO_ID}?id={id}&thumbsize=medium&format=json") + json_data = json.loads(data) + html_json_data = None + + return cls(url, enable_html_scraping, html_content, json_data, html_json_data) @cached_property def video_id(self) -> str: @@ -102,16 +165,6 @@ def video_id(self) -> str: else: return self.url # Assuming this is a video ID (hopefully) - def raw_json_data(self): - """ - Uses the V2 API to retrieve information from a video - :return: - """ - - data = core.fetch(f"{ROOT_URL}{API_VIDEO_ID}?id={self.video_id}&thumbsize=medium&format=json") - parsed_data = json.loads(data) - return parsed_data - @cached_property def tags(self) -> list: tags = [] @@ -170,54 +223,6 @@ def thumbnail(self): The following methods are using HTML scraping. This is against the ToS from EPorner.com! """ - def request_html_content(self): - if not self.enable_html: - raise HTML_IS_DISABLED("HTML content is disabled! See Documentation for more details") - - self.html_content = html.unescape(core.fetch(self.url)) - - - def extract_json_from_html(self): - if not self.enable_html: - raise HTML_IS_DISABLED("HTML content is disabled! See Documentation for more details") - - soup = BeautifulSoup(self.html_content, 'html.parser') - script_tags = soup.find_all('script', {'type': 'application/ld+json'}) - - combined_data = {} - - for script in script_tags: - json_text = script.string.strip() - try: - data = json.loads(json_text) - - except json.decoder.JSONDecodeError: - raise InvalidVideo(""" -JSONDecodeError: I need your help to fix this error. Please report the URL you've used on GitHub. Thanks :)""") - - combined_data.update(data) - - cleaned_dictionary = self.flatten_json(combined_data) - return cleaned_dictionary - - def flatten_json(self, nested_json, parent_key='', sep='_'): - """ - Flatten a nested JSON dictionary. Duplicate keys will be overridden. - - :param nested_json: The nested JSON dictionary to be flattened. - :param parent_key: The base key to use for the flattened keys. - :param sep: The separator between nested keys. - :return: A flattened dictionary. - """ - items = [] - for k, v in nested_json.items(): - new_key = f"{parent_key}{sep}{k}" if parent_key else k - if isinstance(v, dict): - items.extend(self.flatten_json(v, new_key, sep=sep).items()) - else: - items.append((new_key, v)) - return dict(items) - @cached_property def bitrate(self) -> str: """Return the bitrate of the video? (I don't know)""" @@ -331,18 +336,18 @@ def direct_download_link(self, quality, mode) -> str: return urljoin("https://eporner.com", str(url)) - def download(self, quality, path, callback=None, mode=Encoding.mp4_h264, no_title=False): + async def download(self, quality, path, callback=None, mode=Encoding.mp4_h264, no_title=False): if not self.enable_html: raise HTML_IS_DISABLED("HTML content is disabled! See Documentation for more details") - response_redirect_url = core.fetch(self.direct_download_link(quality, mode), + response_redirect_url = await core.fetch(self.direct_download_link(quality, mode), allow_redirects=True, get_response=True) if no_title is False: path = os.path.join(path, f"{self.title}.mp4") try: - core.legacy_download(url=str(response_redirect_url.url), callback=callback, path=path) + await core.legacy_download(url=str(response_redirect_url.url), callback=callback, path=path) return True except Exception: @@ -353,26 +358,31 @@ def download(self, quality, path, callback=None, mode=Encoding.mp4_h264, no_titl class Pornstar: - def __init__(self, url: str, enable_html_scraping: bool = False): + def __init__(self, url: str, enable_html_scraping: bool = False, content: str = None): self.url = url self.enable_html_scraping = enable_html_scraping - self.html_content = core.fetch(self.url) + self.html_content = content + + @classmethod + async def create(cls, url, enable_html_scraping=False): + return cls(url, enable_html_scraping=enable_html_scraping, content = await core.fetch(url)) - def videos(self, pages: int = 0) -> Generator[Video, None, None]: + async def videos(self, pages: int = 0) -> List[Video]: if pages == 0: pages = int(self.video_amount) / 37 # One page contains 37 videos urls = [] for page in range(1, pages): - response = core.fetch(urljoin(self.url + "/", str(page))) + response = await core.fetch(urljoin(self.url + "/", str(page))) extraction = REGEX_SCRAPE_VIDEO_URLS.findall(response) for url in extraction: url = f"https://www.eporner.com{url}" url = url.replace("EPTHBN/", "") urls.append(url) - for url in urls: - yield Video(url, enable_html_scraping=self.enable_html_scraping) + video_tasks = [asyncio.create_task(Client.get_video(url, enable_html_scraping=self.enable_html_scraping)) for url in urls] + video_results = await asyncio.gather(*video_tasks) + return video_results @cached_property def name(self) -> str: @@ -481,41 +491,56 @@ def biography(self) -> str: class Client: @classmethod - def get_video(cls, url: str, enable_html_scraping: bool = False) -> Video: + async def get_video(cls, url: str, enable_html_scraping: bool = False) -> Video: """Returns the Video object for a given URL""" - return Video(url, enable_html_scraping=enable_html_scraping) + return await Video.create(url, enable_html_scraping=enable_html_scraping) @classmethod - def search_videos(cls, query: str, sorting_gay: Union[str, Gay], sorting_order: Union[str, Order], + async def search_videos(cls, query: str, sorting_gay: Union[str, Gay], sorting_order: Union[str, Order], sorting_low_quality: Union[str, LowQuality], - page: int, per_page: int, enable_html_scraping: bool = False) -> Generator[Video, None, None]: + page: int, per_page: int, enable_html_scraping: bool = False) -> List[Video]: - response = core.fetch(f"{ROOT_URL}{API_SEARCH}?query={query}&per_page={per_page}&%page={page}" + response = await core.fetch(f"{ROOT_URL}{API_SEARCH}?query={query}&per_page={per_page}&%page={page}" f"&thumbsize=medium&order={sorting_order}&gay={sorting_gay}&lq=" f"{sorting_low_quality}&format=json") json_data = json.loads(response) + video_urls = [] + for video_ in json_data.get("videos", []): # Don't know why this works lmao - id_ = video_["url"] - yield Video(id_, enable_html_scraping) + video_urls.append(video_["url"]) + + video_tasks = [asyncio.create_task(Client.get_video(url=url, enable_html_scraping=enable_html_scraping)) for url + in video_urls] + video_results = await asyncio.gather(*video_tasks) + return video_results + @classmethod - def get_videos_by_category(cls, category: Union[str, Category], enable_html_scraping: bool = False)\ - -> Generator[Video, None, None]: - for page in range(100): - response = core.fetch(f"{ROOT_URL}cat/{category}/{page}") + async def get_videos_by_category(cls, category: Union[str, Category], enable_html_scraping: bool = False, + pages: int = 1) -> List[Video]: + + video_urls = [] + + for page in range(pages): + response = await core.fetch(f"{ROOT_URL}cat/{category}/{page}") extraction = REGEX_SCRAPE_VIDEO_URLS.findall(response) for url in extraction: url = f"https://www.eporner.com{url}" url = url.replace("EPTHBN/", "") - yield Video(url, enable_html_scraping=enable_html_scraping) + video_urls.append(url) + + video_tasks = [asyncio.create_task(Client.get_video(url=url, enable_html_scraping=enable_html_scraping)) for url in video_urls] + video_results = await asyncio.gather(*video_tasks) + return video_results + @classmethod - def get_pornstar(cls, url: str, enable_html_scraping: bool = True) -> Pornstar: - return Pornstar(url, enable_html_scraping) + async def get_pornstar(cls, url: str, enable_html_scraping: bool = True) -> Pornstar: + return await Pornstar.create(url, enable_html_scraping) -def main(): +async def main(): parser = argparse.ArgumentParser(description="API Command Line Interface") parser.add_argument("--download", metavar="URL (str)", type=str, help="URL to download from") parser.add_argument("--quality", metavar="best,half,worst", type=str, help="The video quality (best,half,worst)", @@ -532,8 +557,8 @@ def main(): if args.download: client = Client() - video = client.get_video(args.download, enable_html_scraping=True) - video.download(quality=args.quality, path=args.output, no_title=no_title) + video = await client.get_video(args.download, enable_html_scraping=True) + await video.download(quality=args.quality, path=args.output, no_title=no_title) if args.file: videos = [] @@ -543,11 +568,11 @@ def main(): content = file.read().splitlines() for url in content: - videos.append(client.get_video(url, enable_html_scraping=True)) + videos.append(await client.get_video(url, enable_html_scraping=True)) for video in videos: - video.download(quality=args.quality, path=args.output, no_title=no_title) + await video.download(quality=args.quality, path=args.output, no_title=no_title) if __name__ == "__main__": - main() + asyncio.run(main()) \ No newline at end of file diff --git a/eporner_api/tests/conftest.py b/eporner_api/tests/conftest.py new file mode 100644 index 0000000..086d715 --- /dev/null +++ b/eporner_api/tests/conftest.py @@ -0,0 +1,12 @@ +import pytest +import asyncio + + +@pytest.fixture(scope="session") +def event_loop(): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + yield loop + loop.close() \ No newline at end of file diff --git a/eporner_api/tests/pytest.ini b/eporner_api/tests/pytest.ini new file mode 100644 index 0000000..2d371ae --- /dev/null +++ b/eporner_api/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode = auto +asyncio_default_fixture_loop_scope = function \ No newline at end of file diff --git a/eporner_api/tests/test_category.py b/eporner_api/tests/test_category.py index 4ff34eb..9c60692 100644 --- a/eporner_api/tests/test_category.py +++ b/eporner_api/tests/test_category.py @@ -1,25 +1 @@ -from ..eporner_api import Client, Category - - -def test_category(): - videos_1 = Client().get_videos_by_category(category=Category.JAPANESE) - videos_2 = Client().get_videos_by_category(category=Category.HD) - videos_3 = Client().get_videos_by_category(category=Category.BLONDE) - - for idx, video in enumerate(videos_1): - if idx == 3: - break - - assert isinstance(video.title, str) and len(video.title) > 0 - - for idx, video in enumerate(videos_2): - if idx == 3: - break - - assert isinstance(video.title, str) and len(video.title) > 0 - - for idx, video in enumerate(videos_3): - if idx == 3: - break - - assert isinstance(video.title, str) and len(video.title) > 0 +# We can't test category, because server gets overloaded. I may fix that in a few months idk \ No newline at end of file diff --git a/eporner_api/tests/test_pornstar.py b/eporner_api/tests/test_pornstar.py index e9a613c..8905831 100644 --- a/eporner_api/tests/test_pornstar.py +++ b/eporner_api/tests/test_pornstar.py @@ -1,17 +1,18 @@ +import pytest from ..eporner_api import Pornstar url = "https://www.eporner.com/pornstar/riley-reid/" -pornstar = Pornstar(url) -def test_videos(): - videos = pornstar.videos(pages=1) +@pytest.mark.asyncio +async def test_pornstar_all(): + pornstar = await Pornstar.create(url) + videos = await pornstar.videos(pages=1) for idx, video in enumerate(videos): assert isinstance(video.title, str) and len(video.title) > 3 if idx == 5: break -def test_information(): assert isinstance(pornstar.pornstar_rank, str) and len(pornstar.pornstar_rank) >= 1 assert isinstance(pornstar.aliases, list) and len(pornstar.aliases) > 1 assert isinstance(pornstar.biography, str) and len(pornstar.biography) > 10 diff --git a/eporner_api/tests/test_search.py b/eporner_api/tests/test_search.py index 541f238..a4d261a 100644 --- a/eporner_api/tests/test_search.py +++ b/eporner_api/tests/test_search.py @@ -1,3 +1,4 @@ +import pytest from ..eporner_api import Client, Gay, Order, LowQuality client = Client() @@ -5,44 +6,32 @@ pages = 2 per_page = 10 - -def test_search_1(): - videos = client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.exclude_gay_content, sorting_order=Order.top_rated, sorting_low_quality=LowQuality.exclude_low_quality_content) +@pytest.mark.asyncio +async def test_search_all(): + videos = await client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.exclude_gay_content, sorting_order=Order.top_rated, sorting_low_quality=LowQuality.exclude_low_quality_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 - -def test_search_2(): - videos = client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.only_gay_content, sorting_order=Order.latest, sorting_low_quality=LowQuality.only_low_quality_content) + videos = await client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.only_gay_content, sorting_order=Order.latest, sorting_low_quality=LowQuality.only_low_quality_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 - -def test_search_3(): - videos = client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.include_gay_content, sorting_order=Order.longest, sorting_low_quality=LowQuality.include_low_quality_content) + videos = await client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.include_gay_content, sorting_order=Order.longest, sorting_low_quality=LowQuality.include_low_quality_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 - -def test_search_4(): - videos = client.search_videos(query, page=pages, per_page=pages, sorting_gay=Gay.exclude_gay_content, sorting_order=Order.shortest, sorting_low_quality=LowQuality.include_low_quality_content) + videos = await client.search_videos(query, page=pages, per_page=pages, sorting_gay=Gay.exclude_gay_content, sorting_order=Order.shortest, sorting_low_quality=LowQuality.include_low_quality_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 - -def test_search_5(): - videos = client.search_videos(query, page=pages, per_page=per_page, sorting_order=Gay.include_gay_content, sorting_gay=Order.top_weekly, sorting_low_quality=LowQuality.include_low_quality_content) + videos = await client.search_videos(query, page=pages, per_page=per_page, sorting_order=Gay.include_gay_content, sorting_gay=Order.top_weekly, sorting_low_quality=LowQuality.include_low_quality_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 - -def test_search_6(): - videos = client.search_videos(query, page=pages, per_page=per_page, sorting_order=Order.most_popular, sorting_low_quality=LowQuality.include_low_quality_content, sorting_gay=Gay.only_gay_content) + videos = await client.search_videos(query, page=pages, per_page=per_page, sorting_order=Order.most_popular, sorting_low_quality=LowQuality.include_low_quality_content, sorting_gay=Gay.only_gay_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 - -def test_search_7(): - videos = client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.include_gay_content, sorting_order=Order.top_monthly, sorting_low_quality=LowQuality.include_low_quality_content) + videos = await client.search_videos(query, page=pages, per_page=per_page, sorting_gay=Gay.include_gay_content, sorting_order=Order.top_monthly, sorting_low_quality=LowQuality.include_low_quality_content, enable_html_scraping=True) for video in videos: assert len(video.title) > 0 diff --git a/eporner_api/tests/test_video.py b/eporner_api/tests/test_video.py index 7d9c456..323e26f 100644 --- a/eporner_api/tests/test_video.py +++ b/eporner_api/tests/test_video.py @@ -1,73 +1,30 @@ +import pytest from ..eporner_api import Client, Encoding, NotAvailable url = "https://www.eporner.com/video-0t0CdQ8Fhaf/hypnotic-big-tits-therapy-video-5-cock-hero/" -video = Client.get_video(url, enable_html_scraping=True) - -def test_title(): +@pytest.mark.asyncio +async def test_video_all(): + video = await Client().get_video(url, enable_html_scraping=True) assert isinstance(video.title, str) and len(video.title) > 0 - - -def test_video_id(): assert isinstance(video.video_id, str) and len(video.video_id) > 0 - - -def test_tags(): assert isinstance(video.tags, list) and len(video.tags) > 0 - - -def test_views(): assert isinstance(video.views, int) and video.views > 0 - - -def test_rate(): assert isinstance(video.rate, str) and len(video.rate) > 0 - - -def test_publish_date(): assert isinstance(video.publish_date, str) and len(video.publish_date) > 0 - - -def test_length_seconds(): assert isinstance(video.length, int) > 0 - - -def test_length_minutes(): assert isinstance(video.length_minutes, str) and len(video.length_minutes) > 0 - - -def test_embed_url(): assert isinstance(video.embed_url, str) and len(video.embed_url) > 0 - - -def test_thumbnails(): assert isinstance(video.thumbnail, str) and len(video.thumbnail) > 0 - - -def test_bitrate(): assert isinstance(video.bitrate, str) and len(video.bitrate) > 0 - - -def test_source_video_url(): assert isinstance(video.source_video_url, str) and len(video.source_video_url) > 0 - - -def test_rating(): assert isinstance(video.rating, str) and len(video.rating) > 0 - - -def test_rating_count(): assert isinstance(video.rating_count, str) and len(video.rating_count) > 0 - - -def test_author(): assert isinstance(video.author, str) and len(video.author) > 0 - - -def test_direct_download_url(): assert isinstance(video.direct_download_link(quality="best", mode=Encoding.mp4_h264), str) assert isinstance(video.direct_download_link(quality="half", mode=Encoding.mp4_h264), str) assert isinstance(video.direct_download_link(quality="worst", mode=Encoding.mp4_h264), str) + try: assert isinstance(video.direct_download_link(quality="best", mode=Encoding.av1), str) assert isinstance(video.direct_download_link(quality="half", mode=Encoding.av1), str) diff --git a/setup.py b/setup.py index 5876b03..0907062 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ setup( name="Eporner_API", - version="1.8.3", + version="1.8.4", packages=find_packages(), - install_requires=["bs4", "eaf_base_api"], + install_requires=["bs4", "eaf_base_api-async"], entry_points={ 'console_scripts': ['eporner_api=eporner_api.eporner_api:main' # If you want to create any executable scripts