Use ModePaginatedRestApiQuery for all extractor that calls list repor…

…ts API (#254)
amundsen-io · Apr 30, 2020 · 4708dd9 · 4708dd9
1 parent 449c960
commit 4708dd9
Show file tree

Hide file tree

Showing 9 changed files with 34 additions and 23 deletions.
diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py
@@ -6,6 +6,7 @@
 from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
@@ -79,8 +80,8 @@ def _build_restapi_query(self):
         report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '(_embedded.reports[*].token)'
         field_names = ['dashboard_id']
-        reports_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
-                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
+                                                  json_path=json_path, field_names=field_names, skip_no_result=True)
 
         queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
         json_path = '_embedded.queries[*].[token,name]'

diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py
@@ -6,6 +6,7 @@
 from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
@@ -77,9 +78,10 @@ def _build_restapi_query(self):
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)'
         field_names = ['dashboard_id', 'last_run_resource_path']
-        last_run_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                                    json_path=json_path, field_names=field_names, skip_no_result=True,
-                                                    json_path_contains_or=True)
+        last_run_resource_path_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                                 json_path=json_path, field_names=field_names,
+                                                                 skip_no_result=True,
+                                                                 json_path_contains_or=True)
 
         # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun
         url = 'https://app.mode.com{last_run_resource_path}'

diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py
@@ -108,6 +108,5 @@ def _build_restapi_query(self):
         json_path = '_embedded.reports[*].[token,name,description,created_at]'
         field_names = ['dashboard_id', 'dashboard_name', 'description', 'created_timestamp']
         reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
-                                                  json_path=json_path, field_names=field_names, skip_no_result=True,
-                                                  pagination_json_path='_embedded.reports[*]')
+                                                  json_path=json_path, field_names=field_names, skip_no_result=True)
         return reports_query
diff --git a/...er/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py b/...er/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py
@@ -6,7 +6,8 @@
     ModeDashboardExecutionsExtractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
 from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT
-from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
 from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
 
@@ -56,7 +57,8 @@ def _build_restapi_query(self):
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '_embedded.reports[*].[token,edited_at]'
         field_names = ['dashboard_id', 'last_modified_timestamp']
-        last_modified_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                           json_path=json_path, field_names=field_names, skip_no_result=True)
+        last_modified_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                        json_path=json_path, field_names=field_names,
+                                                        skip_no_result=True)
 
         return last_modified_query
diff --git a/...extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py b/...extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py
@@ -7,7 +7,8 @@
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
 from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT
 from databuilder.models.dashboard.dashboard_execution import DashboardExecution
-from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
 
 LOGGER = logging.getLogger(__name__)
 
@@ -53,7 +54,8 @@ def _build_restapi_query(self):
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '_embedded.reports[*].[token,last_successfully_run_at]'
         field_names = ['dashboard_id', 'execution_timestamp']
-        last_successful_run_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                                 json_path=json_path, field_names=field_names, skip_no_result=True)
+        last_successful_run_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                              json_path=json_path, field_names=field_names,
+                                                              skip_no_result=True)
 
         return last_successful_run_query
diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py
@@ -6,6 +6,7 @@
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
 from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_failure_handlers import HttpFailureSkipOnStatus
 from databuilder.rest_api.rest_api_query import RestApiQuery
 
@@ -61,9 +62,11 @@ def _build_restapi_query(self):
         # Reports
         json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)'
         field_names = ['dashboard_id', 'creator_resource_path']
-        creator_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
-                                                   json_path=json_path, field_names=field_names, skip_no_result=True,
-                                                   json_path_contains_or=True)
+        creator_resource_path_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template,
+                                                                params=params,
+                                                                json_path=json_path, field_names=field_names,
+                                                                skip_no_result=True,
+                                                                json_path_contains_or=True)
 
         json_path = 'email'
         field_names = ['email']

diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py
@@ -6,6 +6,7 @@
 from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
@@ -81,8 +82,8 @@ def _build_restapi_query(self):
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '(_embedded.reports[*].token)'
         field_names = ['dashboard_id']
-        reports_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                  json_path=json_path, field_names=field_names, skip_no_result=True)
 
         queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
         json_path = '_embedded.queries[*].[token,name]'

diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py
@@ -5,8 +5,8 @@
 
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
-from databuilder.rest_api.rest_api_query import RestApiQuery
-
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
 
 LOGGER = logging.getLogger(__name__)
 
@@ -54,6 +54,6 @@ def _build_restapi_query(self):
         # and view_count
         json_path = '_embedded.reports[*].[token,view_count]'
         field_names = ['dashboard_id', 'accumulated_view_count']
-        reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
-                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
+                                                  json_path=json_path, field_names=field_names, skip_no_result=True)
         return reports_query
diff --git a/databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py b/databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py
@@ -9,6 +9,7 @@
 #  How many records considers as full and indicating there might be next page? In list reports on space API, it's 30.
 DEFAULT_MAX_RECORD_SIZE = 30
 PAGE_SUFFIX_TEMPLATE = '?page={}'
+LIST_REPORTS_PAGINATION_JSON_PATH = '_embedded.reports[*]'  # So far this is the only paginated API that we need.
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,7 +23,7 @@ class ModePaginatedRestApiQuery(RestApiQuery):
     """
 
     def __init__(self,
-                 pagination_json_path,  # type: str
+                 pagination_json_path=LIST_REPORTS_PAGINATION_JSON_PATH,  # type: str
                  max_record_size=DEFAULT_MAX_RECORD_SIZE,  # type: int
                  **kwargs  # type: Any
                  ):