🎉 V6.2.0 - Various Fixes (#1089)

* GitHub actions failing bc TikTok seems to be detecting the IPs of the GitHub actions 💀 (or ms_tokens are now tied to a specific ip) tests pass locally 
* changed docs to address bug reports (#1052)

* Update video_example.py class name (#1056)

I was looking at the examples and noticed that the name of the function in video_example.py is the same as in user_example.py, which led me to believe it was a mistake. I just changed it for better understanding.

* Update tiktok.py (#1070)

proxy support fix

* feat: add comments replies retrieval (#1072)

* Correctly filling in emptyresponseexception constructor parameters (#1073)

* Update tiktok.py (#1080)

Resolves #1074

* Correct file name (#1082)

* Update id and keys for video info scraping (Issue fix #1064) (#1086)

* Update id and keys for video info scraping

* Added error handling for the case such as no video

* V6.2.0

* bump version

---------

Co-authored-by: Pranav Dronavalli <62522813+dronavallipranav@users.noreply.github.com>
Co-authored-by: Edmundo Meyer <80543721+e-meyer@users.noreply.github.com>
Co-authored-by: Vijay Raghuwanshi <37011729+ErVijayRaghuwanshi@users.noreply.github.com>
Co-authored-by: Giancarlo Rocha <giancarloiff@gmail.com>
Co-authored-by: Ben Steel <bendavidsteel@gmail.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: Marco <80593920+Marco2929@users.noreply.github.com>
Co-authored-by: kenki931128 <kenki.nkmr@gmail.com>

Author: 9 people

.sphinx/conf.py

@@ -16,7 +16,7 @@
 project = "TikTokAPI"
 copyright = "2023, David Teather"
 author = "David Teather"
-release = "v6.1.1"
+release = "v6.2.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/main/usage/configuration.html#general-configuration

CITATION.cff

@@ -5,5 +5,5 @@ authors:
     orcid: "https://orcid.org/0000-0002-9467-4676"
 title: "TikTokAPI"
 url: "https://github.com/davidteather/tiktok-api"
-version: 6.1.1
-date-released: 2023-8-20
+version: 6.2.0
+date-released: 2023-11-27

README.md

@@ -85,6 +85,8 @@ Please don't open an issue if you're experiencing one of these just comment if t
 
 * **Browser Has no Attribute** - make sure you ran `python3 -m playwright install`, if your error persists try the [playwright-python](https://github.com/microsoft/playwright-python) quickstart guide and diagnose issues from there.
 
+* **API methods returning Coroutine** - many of the API's methods are async so make sure your program awaits them for proper functionality
+
 ## Quick Start Guide
 
 Here's a quick bit of code to get the most recent trending videos on TikTok. There's more examples in the [examples](https://github.com/davidteather/TikTok-Api/tree/main/examples) directory.
@@ -111,7 +113,7 @@ if __name__ == "__main__":
 
 To directly run the example scripts from the repository root, use the `-m` option on python.
 ```sh
-python -m examples.get_trending
+python -m examples.trending_example
 ```
 
 You can access the full data dictionary the object was created from with `.as_dict`. On a video this may look like

TikTokApi/api/comment.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
-from typing import ClassVar, Optional
+from typing import ClassVar, Iterator, Optional
 from typing import TYPE_CHECKING, ClassVar, Optional
 
+from TikTokApi.exceptions import InvalidResponseException
+
 if TYPE_CHECKING:
     from ..tiktok import TikTokApi
     from .user import User
@@ -49,6 +51,38 @@ def __extract_from_data(self):
         )
         self.likes_count = self.as_dict["digg_count"]
 
+    async def replies(self, count=20, cursor=0, **kwargs) -> Iterator[Comment]:
+        found = 0
+
+        while found < count:
+            params = {
+                "count": 20,
+                "cursor": cursor,
+                "item_id": self.author.user_id,
+                "comment_id": self.id,
+            }
+
+            resp = await self.parent.make_request(
+                url="https://www.tiktok.com/api/comment/list/reply/",
+                params=params,
+                headers=kwargs.get("headers"),
+                session_index=kwargs.get("session_index"),
+            )
+
+            if resp is None:
+                raise InvalidResponseException(
+                    resp, "TikTok returned an invalid response."
+                )
+
+            for comment in resp.get("comments", []):
+                yield self.parent.comment(data=comment)
+                found += 1
+
+            if not resp.get("has_more", False):
+                return
+
+            cursor = resp.get("cursor")
+
     def __repr__(self):
         return self.__str__()
 

TikTokApi/api/video.py

@@ -102,25 +102,55 @@ async def info(self, **kwargs) -> dict:
                 r.text, "TikTok returned an invalid response.", error_code=r.status_code
             )
 
+        # Try SIGI_STATE first
         # extract tag <script id="SIGI_STATE" type="application/json">{..}</script>
         # extract json in the middle
 
         start = r.text.find('<script id="SIGI_STATE" type="application/json">')
-        if start == -1:
-            raise InvalidResponseException(
-                r.text, "TikTok returned an invalid response.", error_code=r.status_code
-            )
+        if start != -1:
+            start += len('<script id="SIGI_STATE" type="application/json">')
+            end = r.text.find("</script>", start)
 
-        start += len('<script id="SIGI_STATE" type="application/json">')
-        end = r.text.find("</script>", start)
+            if end == -1:
+                raise InvalidResponseException(
+                    r.text, "TikTok returned an invalid response.", error_code=r.status_code
+                )
 
-        if end == -1:
-            raise InvalidResponseException(
-                r.text, "TikTok returned an invalid response.", error_code=r.status_code
-            )
+            data = json.loads(r.text[start:end])
+            video_info = data["ItemModule"][self.id]
+        else:
+            # Try __UNIVERSAL_DATA_FOR_REHYDRATION__ next
 
-        data = json.loads(r.text[start:end])
-        video_info = data["ItemModule"][self.id]
+            # extract tag <script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">{..}</script>
+            # extract json in the middle
+
+            start = r.text.find('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')
+            if start == -1:
+                raise InvalidResponseException(
+                    r.text, "TikTok returned an invalid response.", error_code=r.status_code
+                )
+
+            start += len('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')
+            end = r.text.find("</script>", start)
+
+            if end == -1:
+                raise InvalidResponseException(
+                    r.text, "TikTok returned an invalid response.", error_code=r.status_code
+                )
+
+            data = json.loads(r.text[start:end])
+            default_scope = data.get("__DEFAULT_SCOPE__", {})
+            video_detail = default_scope.get("webapp.video-detail", {})
+            if video_detail.get("statusCode", 0) != 0: # assume 0 if not present
+                raise InvalidResponseException(
+                    r.text, "TikTok returned an invalid response structure.", error_code=r.status_code
+                )
+            video_info = video_detail.get("itemInfo", {}).get("itemStruct")
+            if video_info is None:
+                raise InvalidResponseException(
+                    r.text, "TikTok returned an invalid response structure.", error_code=r.status_code
+                )
+            
         self.as_dict = video_info
         self.__extract_from_data()
         return video_info

TikTokApi/tiktok.py

@@ -243,7 +243,7 @@ async def create_sessions(
             override_browser_args = ["--headless=new"]
             headless = False  # managed by the arg
         self.browser = await self.playwright.chromium.launch(
-            headless=headless, args=override_browser_args
+            headless=headless, args=override_browser_args, proxy=random_choice(proxies)
         )
 
         await asyncio.gather(
@@ -338,6 +338,7 @@ async def run_fetch_script(self, url: str, headers: dict, **kwargs):
     async def generate_x_bogus(self, url: str, **kwargs):
         """Generate the X-Bogus header for a url"""
         _, session = self._get_session(**kwargs)
+        await session.page.wait_for_function("window.byted_acrawler !== undefined")
         result = await session.page.evaluate(
             f'() => {{ return window.byted_acrawler.frontierSign("{url}") }}'
         )
@@ -426,7 +427,7 @@ async def make_request(
                 raise Exception("TikTokApi.run_fetch_script returned None")
 
             if result == "":
-                raise EmptyResponseException()
+                raise EmptyResponseException(result, "TikTok returned an empty response")
 
             try:
                 data = json.loads(result)

examples/video_example.py

@@ -7,7 +7,7 @@
 )  # set your own ms_token, think it might need to have visited a profile
 
 
-async def user_example():
+async def get_video_example():
     async with TikTokApi() as api:
         await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
         video = api.video(
@@ -23,4 +23,4 @@ async def user_example():
 
 
 if __name__ == "__main__":
-    asyncio.run(user_example())
+    asyncio.run(get_video_example())

setup.py

@@ -8,7 +8,7 @@
 setuptools.setup(
     name="TikTokApi",
     packages=setuptools.find_packages(),
-    version="6.1.1",
+    version="6.2.0",
     license="MIT",
     description="The Unofficial TikTok API Wrapper in Python 3.",
     author="David Teather",

tests/test_search.py

@@ -4,9 +4,20 @@
 
 ms_token = os.environ.get("ms_token", None)
 
-
 @pytest.mark.asyncio
-async def test_users():
+async def test_users_single_page():
+    api = TikTokApi()
+    async with api:
+        await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+        count = 0
+        async for user in api.search.users("therock", count=10):
+            count += 1
+
+        assert count >= 10
+
+#@pytest.mark.asyncio
+@pytest.mark.skip(reason="Known issue, see #1088 (https://github.com/davidteather/TikTok-Api/issues/1088)")
+async def test_users_multi_page():
     api = TikTokApi()
     async with api:
         await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)