add more logging to quicksight (#1062)

metaphor/quick_sight/client.py

@@ -129,6 +129,7 @@ def _get_dataset_detail(self) -> None:
                 logger.warning(f"Error getting dataset {name} id {dataset_id}: {e}")
                 continue
 
+        logger.info(f"Fetched {len(results)} datasets")
         json_dump_to_debug_file(results, "datasets.json")
 
     def _get_dashboard_detail(self):
@@ -156,6 +157,7 @@ def _get_dashboard_detail(self):
                 logger.error(f"Error getting dashboard {name} id {dashboard_id}: {e}")
                 continue
 
+        logger.info(f"Fetched {len(results)} dashboards")
         json_dump_to_debug_file(results, "dashboards.json")
 
     def _get_data_source_detail(self):
@@ -185,4 +187,5 @@ def _get_data_source_detail(self):
                 )
                 continue
 
+        logger.info(f"Fetched {len(results)} data sources")
         json_dump_to_debug_file(results, "data_sources.json")

metaphor/quick_sight/extractor.py

@@ -87,6 +87,8 @@ def _extract_virtual_views(self):
                 view.entity_upstream = output.entity_upstream
                 self._virtual_views.pop(output_logical_table_id)
 
+        logger.info(f"Parsed {len(self._virtual_views)} virtual views")
+
     def _extract_dashboards(self) -> None:
         for dashboard in self._resources.values():
             if (
@@ -101,6 +103,8 @@ def _extract_dashboards(self) -> None:
                 dataset_arns=dashboard.Version.DataSetArns or []
             )
 
+        logger.info(f"Parsed {len(self._dashboards)} dashboards")
+
     def _make_entities_list(self) -> Collection[ENTITY_TYPES]:
         entities: List[ENTITY_TYPES] = []
         entities.extend(self._virtual_views.values())

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.172"
+version = "0.14.173"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <dev@metaphor.io>"]