Merge pull request #63 from ScrapeGraphAI/add-docstring

feat: add docstring

Author: VinciGit00

scrapegraph-py/scrapegraph_py/__init__.py

@@ -1,3 +1,51 @@
+"""
+ScrapeGraphAI Python SDK
+
+A comprehensive Python SDK for the ScrapeGraphAI API, providing both synchronous
+and asynchronous clients for all API endpoints.
+
+Main Features:
+    - SmartScraper: AI-powered web scraping with structured data extraction
+    - SearchScraper: Web research across multiple sources
+    - Agentic Scraper: Automated browser interactions and form filling
+    - Crawl: Website crawling with AI extraction or markdown conversion
+    - Markdownify: Convert web pages to clean markdown
+    - Schema Generation: AI-assisted schema creation for data extraction
+    - Scheduled Jobs: Automate recurring scraping tasks
+
+Quick Start:
+    >>> from scrapegraph_py import Client
+    >>>
+    >>> # Initialize client from environment variables
+    >>> client = Client.from_env()
+    >>>
+    >>> # Basic scraping
+    >>> result = client.smartscraper(
+    ...     website_url="https://example.com",
+    ...     user_prompt="Extract all product information"
+    ... )
+    >>>
+    >>> # With context manager
+    >>> with Client.from_env() as client:
+    ...     result = client.scrape(website_url="https://example.com")
+
+Async Usage:
+    >>> import asyncio
+    >>> from scrapegraph_py import AsyncClient
+    >>>
+    >>> async def main():
+    ...     async with AsyncClient.from_env() as client:
+    ...         result = await client.smartscraper(
+    ...             website_url="https://example.com",
+    ...             user_prompt="Extract products"
+    ...         )
+    >>>
+    >>> asyncio.run(main())
+
+For more information visit: https://scrapegraphai.com
+Documentation: https://docs.scrapegraphai.com
+"""
+
 from .async_client import AsyncClient
 from .client import Client
 

scrapegraph-py/scrapegraph_py/async_client.py

@@ -1,3 +1,38 @@
+"""
+Asynchronous HTTP client for the ScrapeGraphAI API.
+
+This module provides an asynchronous client for interacting with all ScrapeGraphAI
+API endpoints including smartscraper, searchscraper, crawl, agentic scraper,
+markdownify, schema generation, scheduled jobs, and utility functions.
+
+The AsyncClient class supports:
+- API key authentication
+- SSL verification configuration
+- Request timeout configuration
+- Automatic retry logic with exponential backoff
+- Mock mode for testing
+- Async context manager support for proper resource cleanup
+- Concurrent requests using asyncio
+
+Example:
+    Basic usage with environment variables:
+        >>> import asyncio
+        >>> from scrapegraph_py import AsyncClient
+        >>> async def main():
+        ...     client = AsyncClient.from_env()
+        ...     result = await client.smartscraper(
+        ...         website_url="https://example.com",
+        ...         user_prompt="Extract product information"
+        ...     )
+        ...     await client.close()
+        >>> asyncio.run(main())
+
+    Using async context manager:
+        >>> async def main():
+        ...     async with AsyncClient(api_key="sgai-...") as client:
+        ...         result = await client.scrape(website_url="https://example.com")
+        >>> asyncio.run(main())
+"""
 import asyncio
 from typing import Any, Dict, Optional, Callable
 
@@ -45,6 +80,30 @@
 
 
 class AsyncClient:
+    """
+    Asynchronous client for the ScrapeGraphAI API.
+
+    This class provides asynchronous methods for all ScrapeGraphAI API endpoints.
+    It handles authentication, request management, error handling, and supports
+    mock mode for testing. Uses aiohttp for efficient async HTTP requests.
+
+    Attributes:
+        api_key (str): The API key for authentication
+        headers (dict): Default headers including API key
+        timeout (ClientTimeout): Request timeout configuration
+        max_retries (int): Maximum number of retry attempts
+        retry_delay (float): Base delay between retries in seconds
+        mock (bool): Whether mock mode is enabled
+        session (ClientSession): Aiohttp session for connection pooling
+
+    Example:
+        >>> async def example():
+        ...     async with AsyncClient.from_env() as client:
+        ...         result = await client.smartscraper(
+        ...             website_url="https://example.com",
+        ...             user_prompt="Extract all products"
+        ...         )
+    """
     @classmethod
     def from_env(
         cls,
@@ -145,7 +204,25 @@ def __init__(
         logger.info("✅ AsyncClient initialized successfully")
 
     async def _make_request(self, method: str, url: str, **kwargs) -> Any:
-        """Make HTTP request with retry logic."""
+        """
+        Make asynchronous HTTP request with retry logic and error handling.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            url: Full URL for the request
+            **kwargs: Additional arguments to pass to aiohttp
+
+        Returns:
+            Parsed JSON response data
+
+        Raises:
+            APIError: If the API returns an error response
+            ConnectionError: If unable to connect after all retries
+
+        Note:
+            In mock mode, this method returns deterministic responses without
+            making actual HTTP requests.
+        """
         # Short-circuit when mock mode is enabled
         if getattr(self, "mock", False):
             return self._mock_response(method, url, **kwargs)

scrapegraph-py/scrapegraph_py/client.py

@@ -1,4 +1,31 @@
-# Client implementation goes here
+"""
+Synchronous HTTP client for the ScrapeGraphAI API.
+
+This module provides a synchronous client for interacting with all ScrapeGraphAI
+API endpoints including smartscraper, searchscraper, crawl, agentic scraper,
+markdownify, schema generation, scheduled jobs, and utility functions.
+
+The Client class supports:
+- API key authentication
+- SSL verification configuration
+- Request timeout configuration
+- Automatic retry logic with exponential backoff
+- Mock mode for testing
+- Context manager support for proper resource cleanup
+
+Example:
+    Basic usage with environment variables:
+        >>> from scrapegraph_py import Client
+        >>> client = Client.from_env()
+        >>> result = client.smartscraper(
+        ...     website_url="https://example.com",
+        ...     user_prompt="Extract product information"
+        ... )
+
+    Using context manager:
+        >>> with Client(api_key="sgai-...") as client:
+        ...     result = client.scrape(website_url="https://example.com")
+"""
 import uuid as _uuid
 from typing import Any, Callable, Dict, Optional
 from urllib.parse import urlparse
@@ -51,6 +78,29 @@
 
 
 class Client:
+    """
+    Synchronous client for the ScrapeGraphAI API.
+
+    This class provides synchronous methods for all ScrapeGraphAI API endpoints.
+    It handles authentication, request management, error handling, and supports
+    mock mode for testing.
+
+    Attributes:
+        api_key (str): The API key for authentication
+        headers (dict): Default headers including API key
+        timeout (Optional[float]): Request timeout in seconds
+        max_retries (int): Maximum number of retry attempts
+        retry_delay (float): Delay between retries in seconds
+        mock (bool): Whether mock mode is enabled
+        session (requests.Session): HTTP session for connection pooling
+
+    Example:
+        >>> client = Client.from_env()
+        >>> result = client.smartscraper(
+        ...     website_url="https://example.com",
+        ...     user_prompt="Extract all products"
+        ... )
+    """
     @classmethod
     def from_env(
         cls,
@@ -174,7 +224,25 @@ def __init__(
         logger.info("✅ Client initialized successfully")
 
     def _make_request(self, method: str, url: str, **kwargs) -> Any:
-        """Make HTTP request with error handling."""
+        """
+        Make HTTP request with error handling and retry logic.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            url: Full URL for the request
+            **kwargs: Additional arguments to pass to requests
+
+        Returns:
+            Parsed JSON response data
+
+        Raises:
+            APIError: If the API returns an error response
+            ConnectionError: If unable to connect to the API
+
+        Note:
+            In mock mode, this method returns deterministic responses without
+            making actual HTTP requests.
+        """
         # Short-circuit when mock mode is enabled
         if getattr(self, "mock", False):
             return self._mock_response(method, url, **kwargs)

scrapegraph-py/scrapegraph_py/config.py

@@ -1,4 +1,13 @@
-# Configuration and constants
+"""
+Configuration and constants for the ScrapeGraphAI SDK.
+
+This module contains API configuration settings including the base URL
+and default headers used for all API requests.
+
+Attributes:
+    API_BASE_URL (str): Base URL for the ScrapeGraphAI API endpoints
+    DEFAULT_HEADERS (dict): Default HTTP headers for API requests
+"""
 API_BASE_URL = "https://api.scrapegraphai.com/v1"
 DEFAULT_HEADERS = {
     "accept": "application/json",

scrapegraph-py/scrapegraph_py/exceptions.py

@@ -1,5 +1,28 @@
+"""
+Custom exceptions for the ScrapeGraphAI SDK.
+
+This module defines custom exception classes used throughout the SDK
+for handling API errors and other exceptional conditions.
+"""
+
+
 class APIError(Exception):
-    """Base exception for API errors."""
+    """
+    Exception raised for API errors.
+
+    This exception is raised when the API returns an error response,
+    providing both the error message and HTTP status code for debugging.
+
+    Attributes:
+        message (str): The error message from the API
+        status_code (int): HTTP status code of the error response
+
+    Example:
+        >>> try:
+        ...     client.smartscraper(website_url="invalid")
+        ... except APIError as e:
+        ...     print(f"API error {e.status_code}: {e.message}")
+    """
 
     def __init__(self, message: str, status_code: int = None):
         self.status_code = status_code