Merge branch 'main' into update_wait_for_entire_page_load

Author: frankfeng98

.github/workflows/precommit.yml

@@ -0,0 +1,51 @@
+name: Pre-commit checks
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  pre-commit:
+    name: Pre-commit checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.3
+          virtualenvs-create: false
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      #----------------------------------------------
+      #  ----- install dependencies -----
+      #----------------------------------------------
+      - name: Install dependencies
+        run: |
+          poetry install --no-interaction --no-root --with dev
+
+      - name: Lint check
+        run: pylint --disable=R,C application_examples examples
+
+      - name: Code style check
+        run: black . --check
+
+      - name: Imports sort check
+        uses: isort/isort-action@master
+
+      - name: Static check
+        uses: jakebailey/pyright-action@v2
+        continue-on-error: true
+        with:
+          pylance-version: latest-release

application_examples/flight_price_tracker/flight_price_tracker.py

@@ -1,4 +1,5 @@
 """"This is an example of how to use AgentQL to track the cheapest flight price from Skyscanner website."""
+
 import agentql
 from agentql.sync_api import ScrollDirection
 
@@ -21,8 +22,8 @@
         webgl_vendor=VENDOR_INFO,
         webgl_renderer=RENDERER_INFO,
         nav_user_agent=USER_AGENT_INFO,
-        )
-    
+    )
+
     # Define the queries to interact with the page (You could tweak the queries as per item you want to track on the website)
     QUERY_1 = """
     {
@@ -72,7 +73,7 @@
         depart_date_picker
     }
     """
-    
+
     response_5 = session.query(QUERY_5)
 
     response_5.depart_date_picker.click(force=True)
@@ -127,6 +128,3 @@
 
     # Stop the session
     session.stop()
-
-
-    

application_examples/sentiment_analysis_youtube_comments.py

@@ -1,5 +1,5 @@
-from openai import OpenAI
 import agentql
+from openai import OpenAI
 
 URL = "https://www.youtube.com/watch?v=JfM1mr2bCuk"
 
@@ -17,7 +17,7 @@
 
         # Wait for the page to load (helps to load the comments on the video)
         driver.wait_for_page_ready_state()
-    
+
     # Define the queries to interact with the page
     QUERY = """
     {
@@ -47,23 +47,20 @@
     for comment in response_data["comments"]:
         USER_MESSAGE += comment["comment_text"]
 
-
     SYSTEM_MESSAGE = """You are an expert in understanding the social media analytics and analysis and specialize in analyzing sentiment of the comments. 
     Please find the comments on the video as follows:
     
     """
-    
+
     USER_MESSAGE += "Could you please provide a summary of the comments on the video. Additionaly, just give only 3 takeaways which would be important for me as the creator of the video."
 
     completion = client.chat.completions.create(
         model="gpt-3.5-turbo",
         messages=[
-            { "role": "system", "content": SYSTEM_MESSAGE },
-            { "role": "user", "content": USER_MESSAGE },
-        ]
+            {"role": "system", "content": SYSTEM_MESSAGE},
+            {"role": "user", "content": USER_MESSAGE},
+        ],
     )
 
     # Print the response from the GPT-3.5 model (Sentiment Analysis of the comments on the video)
     print(completion.choices[0].message.content)
-
-    

application_examples/xpath/xpath.py

@@ -1,6 +1,6 @@
 import agentql
 
-#import https://pypi.org/project/playwright-dompath/ 
+# import https://pypi.org/project/playwright-dompath/
 # Playwright Dompath is a Python library that helps you to generate XPath from Playwright selectors.
 from playwright_dompath.dompath_sync import xpath_path
 
@@ -18,11 +18,10 @@
 
 print(response.about_link.text_content())
 
-# Get the XPath 
+# Get the XPath
 print("XPath:", xpath_path(response.search_btn))
 
 response.search_box.fill("tinyfish")
 response.search_btn.click(force=True)
 
 session.stop()
-

examples/async_example/async_example.py

@@ -1,21 +1,28 @@
 """This example demonstrates how to levergae to_data() method provided by AgentQL."""
-import agentql
+
 import asyncio
 
+import agentql
+
 # Set the URL to the desired website
-WALMART_URL = "https://www.walmart.com/ip/Nintendo-Switch-OLED-Model-w-White-Joy-Con/910582148?athbdg=L1600"
+WALMART_URL = (
+    "https://www.walmart.com/ip/Nintendo-Switch-OLED-Model-w-White-Joy-Con/910582148?athbdg=L1600"
+)
 TARGET_URL = "https://www.target.com/p/nintendo-switch-oled-model-with-white-joy-con/-/A-83887639#lnk=sametab"
 NINETENDO_URL = "https://www.nintendo.com/us/store/products/nintendo-switch-oled-model-white-set/"
 
+
 def print_header():
     """Prints the header for the data table"""
     print(f"{'Website':<25} | {'Product ':<20} | {'Price ':<20} ")
     print("-" * 75)
 
+
 def print_row(website, product, price):
     """Prints the data row"""
     print(f"{website:<25} | {product:<20} | {price:<20} ")
 
+
 async def fetch_price(session_url, query):
     session = await agentql.start_async_session(session_url)
     await session.driver.wait_for_page_ready_state()
@@ -24,8 +31,8 @@ async def fetch_price(session_url, query):
     await session.stop()
     return data["price"]
 
+
 async def get_price_across_websites():
-    
 
     # Define the queries to interact with the page
     PRODUCT_INFO_QUERY = """
@@ -42,12 +49,13 @@ async def get_price_across_websites():
     walmart_price, nintendo_price, target_price = await asyncio.gather(
         fetch_price(WALMART_URL, PRODUCT_INFO_QUERY),
         fetch_price(NINETENDO_URL, PRODUCT_INFO_QUERY),
-        fetch_price(TARGET_URL, PRODUCT_INFO_QUERY)
+        fetch_price(TARGET_URL, PRODUCT_INFO_QUERY),
     )
 
     print_row("Walmart", "Nintendo Switch", walmart_price)
     print_row("Nintendo site", "Nintendo Switch", nintendo_price)
     print_row("Target", "Nintendo Switch", target_price)
 
+
 if __name__ == "__main__":
-    asyncio.run(get_price_across_websites())
+    asyncio.run(get_price_across_websites())

examples/customizing_web_driver/README.md

@@ -0,0 +1,16 @@
+# Example script: customize a web drive with AgentQL
+
+This example demonstrates how to customize the web driver used by AgentQL.
+
+## Run the script
+
+* [Install AgentQL SDK](https://docs.agentql.com/docs/installation/sdk-installation)
+* Save this python file locally as **customizing_web_driver.py**
+* Run the following command from the project's folder:
+```bash
+python3 customizing_web_driver.py
+```
+
+## Play with it
+
+Install the [AgentQL Chrome DevTools extension](https://docs.agentql.com/docs/installation/chrome-extension-installation/) to play with the AgentQL query. [Learn more about the AgentQL query language](https://docs.agentql.com/docs/agentql-query/query-intro)

examples/customizing_web_driver/customizing_web_driver.py

@@ -1,4 +1,5 @@
 """This example demonstrates how to customize the web driver used by AgentQL."""
+
 import agentql
 
 # Importing the default PlaywrightWebDriver from AgentQL library

examples/debug_script/debug_agentql_script.py

@@ -8,6 +8,7 @@
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger(__name__)
 
+
 def debug_manager_showcase():
     # The following context manager will enable debug mode for the script.
     with DebugManager.debug_mode():

examples/debug_support_methods/last_query.py

@@ -1,8 +1,9 @@
 """ This example demonstrates how to use the last_query and last_response attributes of the session object to debug the queries and responses. """
+
 import agentql
 
 # Set the URL to the desired website
-URL ="https://www.badmintonwarehouse.com/"
+URL = "https://www.badmintonwarehouse.com/"
 
 if __name__ == "__main__":
     # Start a session with the specified URL
@@ -18,10 +19,10 @@
 
     # Fetch the response of the query
     response = session.query(HOMEPAGE_QUERY)
-    
+
     response.search_box.fill("Mavis 350 Nylon Shuttlecock (Yellow/Fast)")
     response.search_btn.click(force=True)
-    
+
     CHOOSE_PRODUCT_QUERY = """
     {
         products[] 
@@ -71,7 +72,7 @@
             response = session.query(ADD_TO_CART_QUERY)
 
             response.add_to_cart_btn.click(force=True)
-        
+
         else:
             print("Let's wait for sale, current price is too high: ", response.product.to_data())
 
@@ -80,6 +81,3 @@
     print("This was the last query executed ", session.last_query)
 
     print("This was the last response received ", session.last_response)
-
-
-

examples/default_driver/default_driver.py

@@ -1,4 +1,5 @@
 """This example demonstrates how to work with web driver configuration (headless) used by AgentQL."""
+
 import agentql
 
 # Set the URL to the desired website
@@ -11,7 +12,7 @@
         search_input
         search_btn
     }"""
-    
+
     # Start a session with the specified URL and the custom driver
     session = agentql.start_session(URL)
 
@@ -35,5 +36,3 @@
 
     # Stop the session
     session.stop()
-
-

examples/default_popup_handler/default_popup_handler.py

@@ -1,14 +1,15 @@
 """This example demonstrates how to use the default popup handler to close all popups."""
+
 import time
-import agentql
 
+import agentql
 from agentql.sync_api import close_all_popups_handler
 
 # Set the URL to the desired website
 URL = "https://kinfield.com/"
 
 if __name__ == "__main__":
-    
+
     session = agentql.start_session(URL)
 
     # Define the queries to interact with the page
@@ -27,4 +28,4 @@
     time.sleep(5)
 
     # Stop the session
-    session.stop()
+    session.stop()

examples/ecommerce_pricing_data/pricing_data.py

@@ -4,8 +4,9 @@
 """
 
 import asyncio
-from playwright.async_api import async_playwright
+
 from agentql.ext.playwright.async_api import Page
+from playwright.async_api import async_playwright
 
 # URL of the ecommerce website
 # You can replace it with any other ecommerce website but the queries should be updated accordingly
@@ -106,9 +107,7 @@ async def extract_pricing_data(
 ) -> list:
     """Extract pricing data for a product within a price range."""
     # Search for the product with the specified price range
-    print(
-        f"Searching for product: {product} with price range: ${min_price} - ${max_price}"
-    )
+    print(f"Searching for product: {product} with price range: ${min_price} - ${max_price}")
     await _search_product(page, product, min_price, max_price)
 
     current_page = 1

examples/leveraging_to_data_method/levaraging_to_data_method.py

@@ -1,4 +1,5 @@
 """This example demonstrates how to leverage to_data() method provided by AgentQL."""
+
 import agentql
 
 # Set the URL to the desired website
@@ -31,4 +32,4 @@
     print(data)
 
     # Stop the session
-    session.stop()
+    session.stop()