feat: updated heatmap analytics to be generator!

tc_analyzer_lib/metrics/heatmaps/heatmaps.py

@@ -39,7 +39,11 @@ def __init__(
         self.analyzer_config = analyzer_config
         self.utils = HeatmapsUtils(platform_id)
 
-    async def start(self, from_start: bool = False) -> list[dict]:
+    async def start(
+        self,
+        from_start: bool = False,
+        batch_return: int = 5,
+    ):
         """
         Based on the rawdata creates and stores the heatmap data
 
@@ -75,6 +79,7 @@ async def start(self, from_start: bool = False) -> list[dict]:
         async for bot in cursor:
             bot_ids.append(bot["id"])
 
+        index = 0
         while analytics_date.date() < datetime.now().date():
             start_day = analytics_date.replace(
                 hour=0, minute=0, second=0, microsecond=0
@@ -150,10 +155,18 @@ async def start(self, from_start: bool = False) -> list[dict]:
 
                 heatmaps_results.extend(day_results)
 
+                if index == batch_return:
+                    yield heatmaps_results
+                    # emptying it
+                    heatmaps_results = []
+
+                index += 1
+
             # analyze next day
             analytics_date += timedelta(days=1)
 
-        return heatmaps_results
+        if heatmaps_results != []:
+            yield heatmaps_results
 
     async def _prepare_heatmaps_document(
         self, date: datetime, resource_id: str, author_id: str

tc_analyzer_lib/tc_analyzer.py

@@ -85,20 +85,19 @@ async def run_once(self):
             resources=self.resources,
             analyzer_config=self.analyzer_config,
         )
-        heatmaps_data = await heatmaps_analysis.start(from_start=False)
-
-        # storing heatmaps since memberactivities use them
-        analytics_data = {}
-        analytics_data["heatmaps"] = heatmaps_data
-        analytics_data["memberactivities"] = (None, None)
-
-        self.DB_connections.store_analytics_data(
-            analytics_data=analytics_data,
-            platform_id=self.platform_id,
-            graph_schema=self.graph_schema,
-            remove_memberactivities=False,
-            remove_heatmaps=False,
-        )
+        async for heatmaps_data in heatmaps_analysis.start(from_start=False):
+            # storing heatmaps since memberactivities use them
+            analytics_data = {}
+            analytics_data["heatmaps"] = heatmaps_data
+            analytics_data["memberactivities"] = (None, None)
+
+            self.DB_connections.store_analytics_data(
+                analytics_data=analytics_data,
+                platform_id=self.platform_id,
+                graph_schema=self.graph_schema,
+                remove_memberactivities=False,
+                remove_heatmaps=False,
+            )
 
         memberactivity_analysis = MemberActivities(
             platform_id=self.platform_id,
@@ -149,20 +148,19 @@ async def recompute(self):
             resources=self.resources,
             analyzer_config=self.analyzer_config,
         )
-        heatmaps_data = await heatmaps_analysis.start(from_start=True)
-
-        # storing heatmaps since memberactivities use them
-        analytics_data = {}
-        analytics_data["heatmaps"] = heatmaps_data
-        analytics_data["memberactivities"] = (None, None)
-
-        self.DB_connections.store_analytics_data(
-            analytics_data=analytics_data,
-            platform_id=self.platform_id,
-            graph_schema=self.graph_schema,
-            remove_memberactivities=False,
-            remove_heatmaps=True,
-        )
+        async for heatmaps_data in heatmaps_analysis.start(from_start=True):
+            # storing heatmaps since memberactivities use them
+            analytics_data = {}
+            analytics_data["heatmaps"] = heatmaps_data
+            analytics_data["memberactivities"] = (None, None)
+
+            self.DB_connections.store_analytics_data(
+                analytics_data=analytics_data,
+                platform_id=self.platform_id,
+                graph_schema=self.graph_schema,
+                remove_memberactivities=False,
+                remove_heatmaps=True,
+            )
 
         # run the member_activity analyze
         logging.info(