From 4708dd9c6e0aeea8bdb535d58cc925b29e87190a Mon Sep 17 00:00:00 2001 From: Jin Hyuk Chang Date: Thu, 30 Apr 2020 14:09:25 -0700 Subject: [PATCH] Use ModePaginatedRestApiQuery for all extractor that calls list reports API (#254) --- .../mode_analytics/mode_dashboard_charts_extractor.py | 5 +++-- .../mode_dashboard_executions_extractor.py | 8 +++++--- .../dashboard/mode_analytics/mode_dashboard_extractor.py | 3 +-- .../mode_dashboard_last_modified_timestamp_extractor.py | 8 +++++--- ...ode_dashboard_last_successful_executions_extractor.py | 8 +++++--- .../mode_analytics/mode_dashboard_owner_extractor.py | 9 ++++++--- .../mode_analytics/mode_dashboard_queries_extractor.py | 5 +++-- .../mode_analytics/mode_dashboard_usage_extractor.py | 8 ++++---- .../mode_analytics/mode_paginated_rest_api_query.py | 3 ++- 9 files changed, 34 insertions(+), 23 deletions(-) diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py index 2355cd51b..42a763e8f 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py @@ -6,6 +6,7 @@ from databuilder import Scoped from databuilder.extractor.base_extractor import Extractor from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery from databuilder.rest_api.rest_api_query import RestApiQuery from databuilder.transformer.base_transformer import ChainedTransformer from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS @@ -79,8 +80,8 @@ def _build_restapi_query(self): report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token)' field_names = ['dashboard_id'] - reports_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True) + reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params, + json_path=json_path, field_names=field_names, skip_no_result=True) queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries' json_path = '_embedded.queries[*].[token,name]' diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py index 4ede26ffc..279774cda 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py @@ -6,6 +6,7 @@ from databuilder import Scoped from databuilder.extractor.base_extractor import Extractor from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery from databuilder.rest_api.rest_api_query import RestApiQuery from databuilder.transformer.base_transformer import ChainedTransformer from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS @@ -77,9 +78,10 @@ def _build_restapi_query(self): url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)' field_names = ['dashboard_id', 'last_run_resource_path'] - last_run_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True, - json_path_contains_or=True) + last_run_resource_path_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params, + json_path=json_path, field_names=field_names, + skip_no_result=True, + json_path_contains_or=True) # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun url = 'https://app.mode.com{last_run_resource_path}' diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py index 74e6853e2..6ea2c8f5f 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py @@ -108,6 +108,5 @@ def _build_restapi_query(self): json_path = '_embedded.reports[*].[token,name,description,created_at]' field_names = ['dashboard_id', 'dashboard_name', 'description', 'created_timestamp'] reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True, - pagination_json_path='_embedded.reports[*]') + json_path=json_path, field_names=field_names, skip_no_result=True) return reports_query diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py index f43986265..0a4b1272e 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py @@ -6,7 +6,8 @@ ModeDashboardExecutionsExtractor from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT -from databuilder.rest_api.rest_api_query import RestApiQuery +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery +from databuilder.rest_api.rest_api_query import RestApiQuery # noqa: F401 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME @@ -56,7 +57,8 @@ def _build_restapi_query(self): url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '_embedded.reports[*].[token,edited_at]' field_names = ['dashboard_id', 'last_modified_timestamp'] - last_modified_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True) + last_modified_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params, + json_path=json_path, field_names=field_names, + skip_no_result=True) return last_modified_query diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py index d9ed7e237..3c37804db 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py @@ -7,7 +7,8 @@ from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT from databuilder.models.dashboard.dashboard_execution import DashboardExecution -from databuilder.rest_api.rest_api_query import RestApiQuery +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery +from databuilder.rest_api.rest_api_query import RestApiQuery # noqa: F401 LOGGER = logging.getLogger(__name__) @@ -53,7 +54,8 @@ def _build_restapi_query(self): url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '_embedded.reports[*].[token,last_successfully_run_at]' field_names = ['dashboard_id', 'execution_timestamp'] - last_successful_run_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True) + last_successful_run_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params, + json_path=json_path, field_names=field_names, + skip_no_result=True) return last_successful_run_query diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py index ba151eef1..558dba791 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py @@ -6,6 +6,7 @@ from databuilder.extractor.base_extractor import Extractor from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery from databuilder.rest_api.rest_api_failure_handlers import HttpFailureSkipOnStatus from databuilder.rest_api.rest_api_query import RestApiQuery @@ -61,9 +62,11 @@ def _build_restapi_query(self): # Reports json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)' field_names = ['dashboard_id', 'creator_resource_path'] - creator_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True, - json_path_contains_or=True) + creator_resource_path_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template, + params=params, + json_path=json_path, field_names=field_names, + skip_no_result=True, + json_path_contains_or=True) json_path = 'email' field_names = ['email'] diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py index 577168b38..a005d3512 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py @@ -6,6 +6,7 @@ from databuilder import Scoped from databuilder.extractor.base_extractor import Extractor from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery from databuilder.rest_api.rest_api_query import RestApiQuery from databuilder.transformer.base_transformer import ChainedTransformer from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS @@ -81,8 +82,8 @@ def _build_restapi_query(self): url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' json_path = '(_embedded.reports[*].token)' field_names = ['dashboard_id'] - reports_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True) + reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params, + json_path=json_path, field_names=field_names, skip_no_result=True) queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries' json_path = '_embedded.queries[*].[token,name]' diff --git a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py index 305addcf8..f708357a9 100644 --- a/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py +++ b/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py @@ -5,8 +5,8 @@ from databuilder.extractor.base_extractor import Extractor from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils -from databuilder.rest_api.rest_api_query import RestApiQuery - +from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery +from databuilder.rest_api.rest_api_query import RestApiQuery # noqa: F401 LOGGER = logging.getLogger(__name__) @@ -54,6 +54,6 @@ def _build_restapi_query(self): # and view_count json_path = '_embedded.reports[*].[token,view_count]' field_names = ['dashboard_id', 'accumulated_view_count'] - reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params, - json_path=json_path, field_names=field_names, skip_no_result=True) + reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params, + json_path=json_path, field_names=field_names, skip_no_result=True) return reports_query diff --git a/databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py b/databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py index 8c9ad0dc5..b9ba10fbd 100644 --- a/databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py +++ b/databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py @@ -9,6 +9,7 @@ # How many records considers as full and indicating there might be next page? In list reports on space API, it's 30. DEFAULT_MAX_RECORD_SIZE = 30 PAGE_SUFFIX_TEMPLATE = '?page={}' +LIST_REPORTS_PAGINATION_JSON_PATH = '_embedded.reports[*]' # So far this is the only paginated API that we need. LOGGER = logging.getLogger(__name__) @@ -22,7 +23,7 @@ class ModePaginatedRestApiQuery(RestApiQuery): """ def __init__(self, - pagination_json_path, # type: str + pagination_json_path=LIST_REPORTS_PAGINATION_JSON_PATH, # type: str max_record_size=DEFAULT_MAX_RECORD_SIZE, # type: int **kwargs # type: Any ):