V5.1.0 - Memory Improvement, Jupyter Support (on Linux), Hierarchical Exceptions, (#860)

* Popesites/master (#853)

* Add Video().create_time and Video().stats

* Exception reorganization & renaming

* Update bug_report.md

* Memory Improvement & Playwright is not threadsafe(#846)

* Update for #846

* Async browser handling

* Black formatting & ipynb note

* Resize Trendpop

* Ensure cleanup on tests

Co-authored-by: Popesites <conner@popesites.tech>
Co-authored-by: Montel Edwards <m@monteledwards.com>
Co-authored-by: davidteather <davidteather@users.noreply.github.com>

Author: 3 people

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,7 +18,15 @@ Please add any relevant code that is giving you unexpected results.
 
 Preferably the smallest amount of code to reproduce the issue.
 
+
+**SET LOGGING LEVEL TO INFO BEFORE POSTING CODE OUTPUT**
+```py
+import logging
+TikTokApi(logging_level=logging.INFO) # SETS LOGGING_LEVEL TO INFO
+# Hopefully the info level will help you debug or at least someone else on the issue
 ```
+
+```py
 # Code Goes Here
 ```
 

.gitignore

@@ -26,4 +26,5 @@ test2.py
 build
 MANIFEST
 src
-.vscode
+.vscode
+.env

CITATION.cff

@@ -5,5 +5,5 @@ authors:
     orcid: "https://orcid.org/0000-0002-9467-4676"
 title: "TikTokAPI"
 url: "https://github.com/davidteather/tiktok-api"
-version: 5.0.0
-date-released: 2022-2-11
+version: 5.1.0
+date-released: 2022-3-19

README.md

@@ -46,7 +46,9 @@ If you run into an issue please check the closed issues on the github, although
 pip install TikTokApi
 python -m playwright install
 ```
-If you would prefer a video walk through of setting up this package I created a currently semi-outdated [YouTube video](https://www.youtube.com/watch?v=-uCt1x8kINQ) just for that.
+If you would prefer a video walk through of setting up this package [YouTube video](https://www.youtube.com/watch?v=-uCt1x8kINQ) just for that.
+
+If you want a quick video to listen for [TikTok Live](https://www.youtube.com/watch?v=307ijmA3_lc) events in python.
 
 #### Docker Installation
 
@@ -73,17 +75,15 @@ Here's a quick bit of code to get the most recent trending videos on TikTok. The
 ```py
 from TikTokApi import TikTokApi
 
-# In your web browser you will need to go to TikTok, check the cookies 
-# and under www.tiktok.com s_v_web_id should exist, and use that value
-# as input to custom_verify_fp
-# Or watch https://www.youtube.com/watch?v=-uCt1x8kINQ for a visual
-api = TikTokApi(custom_verify_fp="")
-
-for trending_video in api.trending.videos(count=50):
-    # Prints the author's username of the trending video.
-    print(trending_video.author.username)
+# Watch https://www.youtube.com/watch?v=-uCt1x8kINQ for a brief setup tutorial
+with TikTokApi() as api:
+    for trending_video in api.trending.videos(count=50):
+        # Prints the author's username of the trending video.
+        print(trending_video.author.username)
 ```
 
+**Note**: Jupyter (ipynb) only works on linux, see [microsoft/playwright-python #178](https://github.com/microsoft/playwright-python/issues/178)
+
 To run the example scripts from the repository root, make sure you use the `-m` option on python.
 ```sh
 python -m examples.get_trending
@@ -128,10 +128,10 @@ Here's a few more examples that help illustrate the differences in the flow of t
 api = TikTokApi.get_instance()
 trending_videos = api.by_trending()
 
-#V5
-api = TikTokApi() # .get_instance no longer exists
-for trending_video in api.trending.videos():
-    # do something
+#V5.1
+with TikTokApi() as api: # .get_instance no longer exists
+    for trending_video in api.trending.videos():
+        # do something
 ```
 
 Where in V4 you had to extract information yourself, the package now handles that for you. So it's much easier to do chained related function calls.

TikTokApi/api/hashtag.py

@@ -79,7 +79,7 @@ def info_full(self, **kwargs) -> dict:
         data = self.parent.get_data(path, **kwargs)
 
         if data["challengeInfo"].get("challenge") is None:
-            raise TikTokNotFoundError("Challenge {} does not exist".format(self.name))
+            raise NotFoundException("Challenge {} does not exist".format(self.name))
 
         return data
 

TikTokApi/api/sound.py

@@ -63,6 +63,7 @@ def info(self, use_html=False, **kwargs) -> dict:
         sound_data = api.sound(id='7016547803243022337').info()
         ```
         """
+        self.__ensure_valid()
         if use_html:
             return self.info_full(**kwargs)["musicInfo"]
 
@@ -73,7 +74,7 @@ def info(self, use_html=False, **kwargs) -> dict:
         res = self.parent.get_data(path, **kwargs)
 
         if res.get("statusCode", 200) == 10203:
-            raise TikTokNotFoundError()
+            raise NotFoundException()
 
         return res["musicInfo"]["music"]
 
@@ -89,6 +90,7 @@ def info_full(self, **kwargs) -> dict:
         sound_data = api.sound(id='7016547803243022337').info_full()
         ```
         """
+        self.__ensure_valid()
         r = requests.get(
             "https://www.tiktok.com/music/-{}".format(self.id),
             headers={
@@ -119,6 +121,7 @@ def videos(self, count=30, offset=0, **kwargs) -> Iterator[Video]:
             # do something
         ```
         """
+        self.__ensure_valid()
         processed = self.parent._process_kwargs(kwargs)
         kwargs["custom_device_id"] = processed.device_id
 
@@ -154,6 +157,9 @@ def __extract_from_data(self):
         data = self.as_dict
         keys = data.keys()
 
+        if data.get("id") == "":
+            self.id = ""
+
         if "authorName" in keys:
             self.id = data["id"]
             self.title = data["title"]
@@ -166,6 +172,10 @@ def __extract_from_data(self):
                 f"Failed to create Sound with data: {data}\nwhich has keys {data.keys()}"
             )
 
+    def __ensure_valid(self):
+        if self.id == "":
+            raise SoundRemovedException("This sound has been removed!")
+
     def __repr__(self):
         return self.__str__()
 

TikTokApi/api/user.py

@@ -105,7 +105,7 @@ def info_full(self, **kwargs) -> dict:
 
         user_props = user["props"]["pageProps"]
         if user_props["statusCode"] == 404:
-            raise TikTokNotFoundError(
+            raise NotFoundException(
                 "TikTok user with username {} does not exist".format(self.username)
             )
 

TikTokApi/api/video.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 from urllib.parse import urlencode
-
 from ..helpers import extract_video_id_from_url
-
-import logging
 from typing import TYPE_CHECKING, ClassVar, Optional
+from datetime import datetime
 
 if TYPE_CHECKING:
     from ..tiktok import TikTokApi
@@ -28,6 +26,10 @@ class Video:
 
     id: Optional[str]
     """TikTok's ID of the Video"""
+    create_time: Optional[datetime]
+    """The creation time of the Video"""
+    stats: Optional[dict]
+    """TikTok's stats of the Video"""
     author: Optional[User]
     """The User who created the Video"""
     sound: Optional[Sound]
@@ -116,6 +118,8 @@ def __extract_from_data(self) -> None:
 
         if "author" in keys:
             self.id = data["id"]
+            self.create_time = datetime.fromtimestamp(data["createTime"])
+            self.stats = data["stats"]
             self.author = self.parent.user(data=data["author"])
             self.sound = self.parent.sound(data=data["music"])
 
@@ -137,7 +141,7 @@ def __str__(self):
 
     def __getattr__(self, name):
         # Handle author, sound, hashtags, as_dict
-        if name in ["author", "sound", "hashtags", "as_dict"]:
+        if name in ["author", "sound", "hashtags", "stats", "create_time", "as_dict"]:
             self.as_dict = self.info()
             self.__extract_from_data()
             return self.__getattribute__(name)

TikTokApi/browser_utilities/browser.py

@@ -9,32 +9,24 @@
 import re
 from .browser_interface import BrowserInterface
 from urllib.parse import parse_qsl, urlparse
-
+import threading
 from ..utilities import LOGGER_NAME
 from .get_acrawler import _get_acrawler, _get_tt_params_script
-from playwright.sync_api import sync_playwright
-
-playwright = None
+from playwright.async_api import async_playwright
+import asyncio
 
 logger = logging.getLogger(LOGGER_NAME)
 
 
-def get_playwright():
-    global playwright
-    if playwright is None:
-        try:
-            playwright = sync_playwright().start()
-        except Exception as e:
-            raise e
-
-    return playwright
-
-
 class browser(BrowserInterface):
-    def __init__(
-        self,
+    def __init__(self, **kwargs):
+        pass
+
+    @staticmethod
+    async def create(
         **kwargs,
     ):
+        self = browser()
         self.kwargs = kwargs
         self.debug = kwargs.get("debug", False)
         self.proxy = kwargs.get("proxy", None)
@@ -76,25 +68,25 @@ def __init__(
         if self.executable_path is not None:
             self.options["executable_path"] = self.executable_path
 
-        try:
-            self.browser = get_playwright().webkit.launch(
-                args=self.args, **self.options
-            )
-        except Exception as e:
-            logger.critical("Webkit launch failed", exc_info=True)
-            raise e
+        self._thread_locals = threading.local()
+        self._thread_locals.playwright = await async_playwright().start()
+        self.playwright = self._thread_locals.playwright
+        self.browser = await self.playwright.webkit.launch(
+            args=self.args, **self.options
+        )
+        context = await self._create_context(set_useragent=True)
+        page = await context.new_page()
+        await self.get_params(page)
+        await context.close()
 
-        context = self._create_context(set_useragent=True)
-        page = context.new_page()
-        self.get_params(page)
-        context.close()
+        return self
 
-    def get_params(self, page) -> None:
+    async def get_params(self, page) -> None:
         self.browser_language = self.kwargs.get(
             "browser_language",
-            page.evaluate("""() => { return navigator.language; }"""),
+            await page.evaluate("""() => { return navigator.language; }"""),
         )
-        self.browser_version = page.evaluate(
+        self.browser_version = await page.evaluate(
             """() => { return window.navigator.appVersion; }"""
         )
 
@@ -112,15 +104,15 @@ def get_params(self, page) -> None:
 
         self.timezone_name = self.kwargs.get(
             "timezone_name",
-            page.evaluate(
+            await page.evaluate(
                 """() => { return Intl.DateTimeFormat().resolvedOptions().timeZone; }"""
             ),
         )
-        self.width = page.evaluate("""() => { return screen.width; }""")
-        self.height = page.evaluate("""() => { return screen.height; }""")
+        self.width = await page.evaluate("""() => { return screen.width; }""")
+        self.height = await page.evaluate("""() => { return screen.height; }""")
 
-    def _create_context(self, set_useragent=False):
-        iphone = playwright.devices["iPhone 11 Pro"]
+    async def _create_context(self, set_useragent=False):
+        iphone = self.playwright.devices["iPhone 11 Pro"]
         iphone["viewport"] = {
             "width": random.randint(320, 1920),
             "height": random.randint(320, 1920),
@@ -131,7 +123,7 @@ def _create_context(self, set_useragent=False):
 
         iphone["bypass_csp"] = True
 
-        context = self.browser.new_context(**iphone)
+        context = await self.browser.new_context(**iphone)
         if set_useragent:
             self.user_agent = iphone["user_agent"]
 
@@ -174,17 +166,19 @@ def gen_verifyFp(self):
 
         return f'verify_{scenario_title.lower()}_{"".join(uuid)}'
 
-    def sign_url(self, url, calc_tt_params=False, **kwargs):
-        def process(route):
-            route.abort()
+    async def sign_url(self, url, calc_tt_params=False, **kwargs):
+        async def process(route):
+            await route.abort()
 
         tt_params = None
-        context = self._create_context()
-        page = context.new_page()
+        context = await self._create_context()
+        page = await context.new_page()
 
         if calc_tt_params:
-            page.route(re.compile(r"(\.png)|(\.jpeg)|(\.mp4)|(x-expire)"), process)
-            page.goto(
+            await page.route(
+                re.compile(r"(\.png)|(\.jpeg)|(\.mp4)|(x-expire)"), process
+            )
+            await page.goto(
                 kwargs.get("default_url", "https://www.tiktok.com/@redbull"),
                 wait_until="load",
             )
@@ -212,8 +206,8 @@ def process(route):
 
         url = "{}&verifyFp={}&device_id={}".format(url, verifyFp, device_id)
 
-        page.add_script_tag(content=_get_acrawler())
-        evaluatedPage = page.evaluate(
+        await page.add_script_tag(content=_get_acrawler())
+        evaluatedPage = await page.evaluate(
             '''() => {
             var url = "'''
             + url
@@ -227,9 +221,9 @@ def process(route):
         url = "{}&_signature={}".format(url, evaluatedPage)
 
         if calc_tt_params:
-            page.add_script_tag(content=_get_tt_params_script())
+            await page.add_script_tag(content=_get_tt_params_script())
 
-            tt_params = page.evaluate(
+            tt_params = await page.evaluate(
                 """() => {
                     return window.genXTTParams("""
                 + json.dumps(dict(parse_qsl(urlparse(url).query)))
@@ -238,15 +232,12 @@ def process(route):
                 }"""
             )
 
-        context.close()
+        await context.close()
         return (verifyFp, device_id, evaluatedPage, tt_params)
 
-    def _clean_up(self):
-        try:
-            self.browser.close()
-        except Exception:
-            logger.exception("cleanup failed")
-        # playwright.stop()
+    async def _clean_up(self):
+        await self.browser.close()
+        await self.playwright.stop()
 
     def find_redirect(self, url):
         self.page.goto(url, {"waitUntil": "load"})