fix: device pagination | sim list pagination | sms retrieval (#223)

* fix and add paging behaviour to sim listing

* fix sms retrieval

* apply tests cassetes sanitization

* update local debugging docs

---------

Co-authored-by: mikhail.gubenko <mikhail.gubenko@emnify.com>

DEVELOPMENT.md

@@ -24,7 +24,7 @@ To set up your sandbox, modify the code in this file as needed.
 
 Once your sandbox is set up, you can launch the file and view the results.
 ```shell
-docker run -t -e EMNIFY_APPLICATION_TOKEN=<your_token_here> -v $(pwd):/sdk emnify/python-sdk python docs/examples/local_debug.py
+docker run -t -e EMNIFY_SDK_APPLICATION_TOKEN=<your_token_here> -e EMINFY_SDK_API_ENDPOINT_URL=<your_debug_API_endpoint> -v $(pwd):/sdk emnify/python-sdk python docs/examples/local_debug.py
 ```
 
 ## Version Bump

docs/examples/local_debug.py

@@ -10,13 +10,38 @@
 from emnify import EMnify
 
 # Get the token from environment variable
-TOKEN = os.environ.get('EMNIFY_APPLICATION_TOKEN')
+TOKEN = os.environ.get('EMNIFY_SDK_APPLICATION_TOKEN')
 
 # Initiate SDK instance using application token
 emnify = EMnify(TOKEN)
 
+def get_iterator_length(iterator):
+    return len(list(iterator))
+
 # Execute a command against desired API
 devices = emnify.devices.get_devices_list()
+#
+#
+## print first 100 devices
+for i, device in enumerate(devices):
+    if i >= 100:
+        break
+    print(device)
+
+# Count remaining devices
+print(get_iterator_length(devices))
+
+
+# Get list of sims
+sims = emnify.sim.get_sim_list()
+
+# Count sims
+print(get_iterator_length(sims))
+
+sms_list = emnify.devices.get_device_sms_list(device=11379224)
+print(list(sms_list))
+print(sms_list)
 
-# Showing all the devices
-print([device for device in devices])
+# Get list of events
+events = emnify.devices.get_device_events_list(device=11379224)
+print(list(events))

emnify/api_manager.py

@@ -1,12 +1,14 @@
+import os
 import typing
 import requests
 
 from emnify import errors as emnify_errors
 from emnify.modules.api.models import AuthenticationResponse
 from emnify import constants as emnify_constants
 
-MAIN_URL = 'https://cdn.emnify.net/api'
+MAIN_URL = os.environ.get('EMINFY_SDK_API_ENDPOINT_URL', 'https://cdn.emnify.net/api')
 
+MAX_PAGES_IN_PAGINATOR = 1000 # with regular page size 1000...2000 gives max 2_000_000 records
 
 class BaseApiManager:
     """
@@ -46,8 +48,9 @@ def process_exception(self, response: requests.Response, client, data: dict = No
     def return_paginator(
             self, response: requests.Response, client, data, files, query_params, path_params
     ) -> typing.Generator:
+        query_params = query_params or {}
         page = query_params.get('page', 1) if query_params else 1
-
+        total_pages = int(response.headers.get(emnify_constants.ResponseHeaders.TOTAL_PAGES.value))
         try:
             response_data = response.json()
         except requests.exceptions.JSONDecodeError:
@@ -56,9 +59,13 @@ def return_paginator(
         for item in response_data:
             yield item
 
-        if int(response.headers.get(emnify_constants.ResponseHeaders.TOTAL_PAGES.value)) > page:
+        if page < total_pages and page < MAX_PAGES_IN_PAGINATOR:
             query_params['page'] = page + 1
-            self.call_api(client, data=data, files=files, query_params=query_params, path_params=path_params)
+
+            next_page_response = self.call_api(client, data, files, query_params=query_params, path_params=path_params)
+
+            for item in next_page_response:
+                yield item
 
     def build_method_url(self, url_params):
         return self.request_url_prefix.format(**url_params)
@@ -74,7 +81,7 @@ def unauthorised(self, response: requests.Response, client, data: dict = None, p
 
         return self.call_api(client, data, path_params=path_params, *args, **kwargs)
 
-    def call_api(self, client, data: dict = None, files=None, path_params: dict = None, query_params: dict = None):
+    def call_api(self, client, data: dict = None, files=None, path_params: dict = None, query_params: dict = None, handler=None):
         url = self.request_url_prefix
         if path_params:
             url = self.build_method_url(path_params)
@@ -83,6 +90,7 @@ def call_api(self, client, data: dict = None, files=None, path_params: dict = No
             raise emnify_errors.UnknownStatusCodeException(
                 "Unknown status code {status_code}".format(status_code=response.status_code)
             )
+
         return getattr(self, self.response_handlers[response.status_code])\
             (response, client, data=data, files=files, query_params=query_params, path_params=path_params)
 

emnify/modules/device/api_call_manager.py

@@ -16,6 +16,11 @@ class GetAllDevicesApiCall(BaseApiManager):
 class GetEventsByDevice(BaseApiManager):
     request_url_prefix = '/v1/endpoint/{endpoint_id}/event'
     request_method_name = RequestsType.GET.value
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.response_handlers = self.response_handlers.copy() | {
+            200: 'return_paginator'
+        }
 
 
 class CreateDevice(BaseApiManager):

emnify/modules/device/manager.py

@@ -64,7 +64,7 @@ def get_device_filter_model(self) -> typing.Type[device_models.FilterDeviceModel
 
     def get_device_sms_list(self, *, device: typing.Union[device_models.Device, int]) -> device_models.ListSms:
         device_id = self.validate_device(device)
-        sms_response = device_call_managers.GetEventsByDevice().call_api(
+        sms_response = device_call_managers.GetAllSmsFromDevice().call_api(
             client=self.client, path_params={'endpoint_id': device_id}
         )
         for sms in sms_response:
@@ -147,7 +147,7 @@ def get_devices_list(
             query_params = self.__transform_all_devices_filter_params(filter_model, sort_enum)
         devices_response = device_call_managers.GetAllDevicesApiCall()\
             .call_api(client=self.client, query_params=query_params, *args, **kwargs)
-        return [device_models.Device(**i) for i in devices_response]
+        return (device_models.Device(**i) for i in devices_response)
 
     def delete_device(self, device_id: int) -> True:
         """
@@ -199,8 +199,8 @@ def get_device_events_list(self, device: typing.Union[device_models.Device, int]
         events_response = device_call_managers.GetEventsByDevice().call_api(
             client=self.client, path_params={'endpoint_id': device_id}
         )
-        for event in events_response:
-            yield device_models.DeviceEvent(**event)
+
+        return (device_models.DeviceEvent(**i) for i in events_response)
 
     def change_status(
             self, device: typing.Union[

emnify/modules/device/models.py

@@ -48,7 +48,7 @@ class SmsCreateModel(generated_models.SubmitMTSMSrequest):
     dcs: Optional[int] = None
 
 
-class ListSms(generated_models.RetrieveEventsresponse5):
+class ListSms(generated_models.ListofSMSresponse):
     """
     Renamed generated model
     """

emnify/modules/sim/api_call_manager.py

@@ -8,6 +8,12 @@ class SimListApi(BaseApiManager):
     request_url_prefix = '/v1/sim'
     request_method_name = RequestsType.GET.value
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.response_handlers = self.response_handlers.copy() | {
+            200: 'return_paginator'
+        }
+
 
 class SimActivateApi(BaseApiManager):
     request_url_prefix = '/v1/sim_batch/bic/{bic}'

emnify/modules/sim/manager.py

@@ -48,8 +48,9 @@ def get_sim_list(
             without_device: bool = None,
             filter_model: sim_models.SimFilter = None,
             sort_enum: emnify_const.SimSort = None
-    ):
+    ) -> typing.Generator[sim_models.SimList, None, None]:
         """
+        Retrieve iterable list of SIM`s.
         :param without_device: Allows to add a filter for request to find all SIM`s without device
         :param filter_model: Model for request`s filtering
         :param sort_enum: Model for request`s sorting
@@ -59,8 +60,8 @@ def get_sim_list(
         )
 
         sim_response = SimListApi().call_api(client=self.client, query_params=query_params)
-        for sim in sim_response:
-            yield self.get_sim_list_model(**sim)
+
+        return (self.get_sim_list_model(**i) for i in sim_response)
 
     def retrieve_sim(self, sim_id: int):
         """

settings.py

@@ -1,4 +1,4 @@
 import os
 
-MAIN_URL = 'https://cdn.emnify.net/api'
-TOKEN = os.environ.get('APP_TOKEN')
+MAIN_URL = os.environ.get('EMINFY_SDK_API_ENDPOINT_URL', 'https://cdn.emnify.net/api')
+TOKEN = os.environ.get('EMNIFY_SDK_APPLICATION_TOKEN', os.environ.get('APP_TOKEN'))