fix: Fixed ZIA Pagination (#232)

* fix: Fixed ZIA Pagination

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Zscaler Python SDK Changelog
 
+## 0.10.2 (January,6 2024)
+
+### Notes
+
+- Python Versions: **v3.8, v3.9, v3.10, v3.11**
+
+### Bug Fix:
+
+* ([#231](https://github.com/zscaler/zscaler-sdk-python/pull/231)) - Improved ZIA pagination logic to enhance flexibility and address user-reported issues. The changes include:
+  - Fixed behavior where `pagesize` was being ignored, defaulting to 100. The SDK now respects the user-specified `pagesize` value within API limits (100-10,000).
+  - Added explicit handling for the `page` parameter. When provided, the SDK fetches data from only the specified page without iterating through all pages.
+  - Updated docstrings and documentation to clarify the correct usage of `page` and `pagesize` parameters.
+
+
 ## 0.10.1 (December,18 2024)
 
 ### Notes

docsrc/conf.py

@@ -28,9 +28,9 @@
 html_title = ""
 
 # The short X.Y version
-version = "0.10.1"
+version = "0.10.2"
 # The full version, including alpha/beta/rc tags
-release = "0.10.1"
+release = "0.10.2"
 
 # -- General configuration ---------------------------------------------------
 

docsrc/zs/guides/release_notes.rst

@@ -6,6 +6,20 @@ Release Notes
 Zscaler Python SDK Changelog
 ----------------------------
 
+## 0.10.2 (January,6 2024)
+
+### Notes
+
+- Python Versions: **v3.8, v3.9, v3.10, v3.11**
+
+Bug Fixes
+------------
+
+* Fixed behavior where `pagesize` was being ignored, defaulting to 100. The SDK now respects the user-specified `pagesize` value within API limits (100-10,000). (`231 <https://github.com/zscaler/zscaler-sdk-python/pull/231>`_). 
+* Added explicit handling for the `page` parameter. When provided, the SDK fetches data from only the specified page without iterating through all pages. (`231 <https://github.com/zscaler/zscaler-sdk-python/pull/231>`_). 
+* Updated docstrings and documentation to clarify the correct usage of `page` and `pagesize` parameters.. (`231 <https://github.com/zscaler/zscaler-sdk-python/pull/231>`_). 
+
+
 ## 0.10.1 (December,18 2024)
 
 ### Notes

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zscaler-sdk-python"
-version = "0.10.1"
+version = "0.10.2"
 description = "Official Python SDK for the Zscaler Products (Beta)"
 authors = ["Zscaler, Inc. <devrel@zscaler.com>"]
 license = "MIT"

zscaler/__init__.py

@@ -29,7 +29,7 @@
 __contributors__ = [
     "William Guilherme",
 ]
-__version__ = "0.10.1"
+__version__ = "0.10.2"
 
 from zscaler.zdx import ZDXClientHelper  # noqa
 from zscaler.zia import ZIAClientHelper  # noqa

zscaler/zia/__init__.py

@@ -422,15 +422,15 @@ def get_paginated_data(
             path (str): The API endpoint path to send requests to.
             expected_status_code (int): The expected HTTP status code for a successful request. Defaults to 200.
             page (int): Specific page number to fetch. Defaults to 1 if not provided.
-            pagesize (int): Number of items per page, default is 100, with a maximum of 1000.
+            pagesize (int): Number of items per page, default is 100, with a maximum of 10000.
             search (str): Search query to filter the results.
             max_items (int): Maximum number of items to retrieve.
             max_pages (int): Maximum number of pages to fetch.
             type (str, optional): Type of VPN credentials (e.g., CN, IP, UFQDN, XAUTH).
             include_only_without_location (bool, optional): Filter to include only VPN credentials not associated with a location.
             location_id (int, optional): Retrieve VPN credentials for the specified location ID.
             managed_by (int, optional): Retrieve VPN credentials managed by the specified partner.
-            prefix (int, optional): Retrieve VPN credentials managed by the specified partner.
+            prefix (int, optional): Retrieve VPN credentials managed by a given partner.
 
         Returns:
             tuple: A tuple containing:
@@ -444,17 +444,12 @@ def get_paginated_data(
             "EMPTY_RESULTS": "No results found for page {page}.",
         }
 
-        # Initialize pagination parameters
-        # params = {
-        #     "page": page if page is not None else 1,  # Start at page 1 if not specified
-        #     "pagesize": min(pagesize if pagesize is not None else 100, max_page_size),  # Apply max_page_size limit
-        # }
-
+        # Initialize parameters
         params = {
             "page": page if page is not None else 1,  # Start at page 1 if not specified
-            "pagesize": max(100, min(pagesize or 100, 10000)),  # Ensure pagesize is within API limits
+            "pageSize": pagesize if pagesize is not None else 100,  # Allow any user-defined pagesize
         }
-        
+
         # Add optional filters to the params if provided
         if search:
             params["search"] = search
@@ -469,12 +464,30 @@ def get_paginated_data(
         if prefix:
             params["prefix"] = prefix
 
+        # If the user specifies a page, fetch only that page
+        if page is not None:
+            response = self.send("GET", path=path, params=params)
+            if response.status_code != expected_status_code:
+                error_msg = ERROR_MESSAGES["UNEXPECTED_STATUS"].format(
+                    status_code=response.status_code, page=params["page"]
+                )
+                logger.error(error_msg)
+                return BoxList([]), error_msg
+
+            response_data = response.json()
+            if not isinstance(response_data, list):
+                error_msg = ERROR_MESSAGES["EMPTY_RESULTS"].format(page=params["page"])
+                logger.warn(error_msg)
+                return BoxList([]), error_msg
+
+            data = convert_keys_to_snake(response_data)
+            return BoxList(data), None
+
+        # If no page is specified, iterate through pages to fetch all items
         ret_data = []
         total_collected = 0
-
         try:
             while True:
-                # Apply rate-limiting if necessary
                 should_wait, delay = self.rate_limiter.wait("GET")
                 if should_wait:
                     time.sleep(delay)
@@ -499,13 +512,6 @@ def get_paginated_data(
 
                 data = convert_keys_to_snake(response_data)
 
-                # If searching for a specific item, stop if we find a match
-                if search:
-                    for item in data:
-                        if item.get("name") == search:
-                            ret_data.append(item)
-                            return BoxList(ret_data), None
-
                 # Limit data collection based on max_items
                 if max_items is not None:
                     data = data[: max_items - total_collected]  # Limit items on the current page
@@ -518,8 +524,8 @@ def get_paginated_data(
                 ):
                     break
 
-                # Stop if we've processed all available pages (i.e., less than requested page size)
-                if len(data) < params["pagesize"]:
+                # Stop if fewer items than pageSize are returned
+                if len(data) < params["pageSize"]:
                     break
 
                 # Move to the next page
@@ -535,6 +541,142 @@ def get_paginated_data(
 
         return BoxList(ret_data), None
 
+
+    # def get_paginated_data(
+    #     self,
+    #     path=None,
+    #     expected_status_code=200,
+    #     page=None,
+    #     pagesize=None,
+    #     search=None,
+    #     max_items=None,  # Maximum number of items to retrieve across pages
+    #     max_pages=None,  # Maximum number of pages to retrieve
+    #     type=None,  # Specify type of VPN credentials (CN, IP, UFQDN, XAUTH)
+    #     include_only_without_location=None,  # Include only VPN credentials not associated with any location
+    #     location_id=None,  # VPN credentials for a specific location ID
+    #     managed_by=None,  # VPN credentials managed by a given partner
+    #     prefix=None,  # VPN credentials managed by a given partner
+    # ):
+    #     """
+    #     Fetches paginated data from the API based on specified parameters and handles pagination.
+
+    #     Args:
+    #         path (str): The API endpoint path to send requests to.
+    #         expected_status_code (int): The expected HTTP status code for a successful request. Defaults to 200.
+    #         page (int): Specific page number to fetch. Defaults to 1 if not provided.
+    #         pagesize (int): Number of items per page, default is 100, with a maximum of 1000.
+    #         search (str): Search query to filter the results.
+    #         max_items (int): Maximum number of items to retrieve.
+    #         max_pages (int): Maximum number of pages to fetch.
+    #         type (str, optional): Type of VPN credentials (e.g., CN, IP, UFQDN, XAUTH).
+    #         include_only_without_location (bool, optional): Filter to include only VPN credentials not associated with a location.
+    #         location_id (int, optional): Retrieve VPN credentials for the specified location ID.
+    #         managed_by (int, optional): Retrieve VPN credentials managed by the specified partner.
+    #         prefix (int, optional): Retrieve VPN credentials managed by the specified partner.
+
+    #     Returns:
+    #         tuple: A tuple containing:
+    #             - BoxList: A list of fetched items wrapped in a BoxList for easy access.
+    #             - str: An error message if any occurred during the data fetching process.
+    #     """
+    #     logger = logging.getLogger(__name__)
+
+    #     ERROR_MESSAGES = {
+    #         "UNEXPECTED_STATUS": "Unexpected status code {status_code} received for page {page}.",
+    #         "EMPTY_RESULTS": "No results found for page {page}.",
+    #     }
+
+    #     # Initialize pagination parameters
+    #     # params = {
+    #     #     "page": page if page is not None else 1,  # Start at page 1 if not specified
+    #     #     "pagesize": min(pagesize if pagesize is not None else 100, max_page_size),  # Apply max_page_size limit
+    #     # }
+
+    #     params = {
+    #         "page": page if page is not None else 1,  # Start at page 1 if not specified
+    #         "pagesize": max(100, min(pagesize or 100, 10000)),  # Ensure pagesize is within API limits
+    #     }
+
+    #     # Add optional filters to the params if provided
+    #     if search:
+    #         params["search"] = search
+    #     if type:
+    #         params["type"] = type
+    #     if include_only_without_location is not None:
+    #         params["includeOnlyWithoutLocation"] = include_only_without_location
+    #     if location_id:
+    #         params["locationId"] = location_id
+    #     if managed_by:
+    #         params["managedBy"] = managed_by
+    #     if prefix:
+    #         params["prefix"] = prefix
+
+    #     ret_data = []
+    #     total_collected = 0
+
+    #     try:
+    #         while True:
+    #             # Apply rate-limiting if necessary
+    #             should_wait, delay = self.rate_limiter.wait("GET")
+    #             if should_wait:
+    #                 time.sleep(delay)
+
+    #             # Send the request to the API
+    #             response = self.send("GET", path=path, params=params)
+
+    #             # Check for unexpected status code
+    #             if response.status_code != expected_status_code:
+    #                 error_msg = ERROR_MESSAGES["UNEXPECTED_STATUS"].format(
+    #                     status_code=response.status_code, page=params["page"]
+    #                 )
+    #                 logger.error(error_msg)
+    #                 return BoxList([]), error_msg
+
+    #             # Parse the response as a flat list of items
+    #             response_data = response.json()
+    #             if not isinstance(response_data, list):
+    #                 error_msg = ERROR_MESSAGES["EMPTY_RESULTS"].format(page=params["page"])
+    #                 logger.warn(error_msg)
+    #                 return BoxList([]), error_msg
+
+    #             data = convert_keys_to_snake(response_data)
+
+    #             # If searching for a specific item, stop if we find a match
+    #             if search:
+    #                 for item in data:
+    #                     if item.get("name") == search:
+    #                         ret_data.append(item)
+    #                         return BoxList(ret_data), None
+
+    #             # Limit data collection based on max_items
+    #             if max_items is not None:
+    #                 data = data[: max_items - total_collected]  # Limit items on the current page
+    #             ret_data.extend(data)
+    #             total_collected += len(data)
+
+    #             # Check if we've reached max_items or max_pages limits
+    #             if (max_items is not None and total_collected >= max_items) or (
+    #                 max_pages is not None and params["page"] >= max_pages
+    #             ):
+    #                 break
+
+    #             # Stop if we've processed all available pages (i.e., less than requested page size)
+    #             if len(data) < params["pagesize"]:
+    #                 break
+
+    #             # Move to the next page
+    #             params["page"] += 1
+
+    #     finally:
+    #         time.sleep(2)  # Ensure a delay between requests regardless of outcome
+
+    #     if not ret_data:
+    #         error_msg = ERROR_MESSAGES["EMPTY_RESULTS"].format(page=params["page"])
+    #         logger.warn(error_msg)
+    #         return BoxList([]), error_msg
+
+    #     return BoxList(ret_data), None
+
     @property
     def admin_and_role_management(self):
         """