ratelimiting and cron tweaks, version bump

pyproject.toml

@@ -8,7 +8,7 @@ line-length = 120
 
 [tool.poetry]
 name = "siem-query-utils"
-version = "2.1.0"
+version = "2.1.1"
 description = ""
 authors = ["Adon Metcalfe <adonm@fastmail.fm>"]
 readme = "README.md"

siem_query_utils/__init__.py

@@ -71,9 +71,9 @@ def schedule_jobs():
     logger.debug("Job scheduling enabled!!!")
     scheduler = BackgroundScheduler({"apscheduler.timezone": "Australia/Perth"})
     # Add schedules, configure tasks here
-    scheduler.add_job(api.update_jira_issues, "cron", second="*/20", max_instances=1)
-    scheduler.add_job(api.ingest_datalake_hot, "cron", second="*/30", max_instances=1)
-    scheduler.add_job(api.export_jira_issues, "cron", minute="*/15", max_instances=1)
+    scheduler.add_job(api.ingest_datalake_hot, "cron", second="*/15", max_instances=1)
+    scheduler.add_job(api.update_jira_issues, "cron", minute="*/3", max_instances=1)
+    scheduler.add_job(api.export_jira_issues, "cron", minute="*/20", max_instances=1)
     scheduler.add_job(api.list_workspaces, "cron", minute="10")
     scheduler.add_job(generate_reports, "cron", hour="18", max_instances=1)
     scheduler.add_job(api.configure_datalake_hot, "cron", hour="22")

siem_query_utils/api.py

@@ -1043,11 +1043,11 @@ def incidents(after="ago(1h)", rows=1000):
             sleep(10)
             return incidents(rows=rows)
         df["siemref"] = df["TenantId"] + "_" + df["IncidentNumber"].astype(str)
-        dfs = [df[i : i + 64] for i in range(0, df.shape[0], 64)]
-        for df in dfs:
-            with ThreadPoolExecutor(max_workers=8) as executor:
+        dfs = [df[i : i + 32] for i in range(0, df.shape[0], 32)]
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            for df in dfs:
                 df["jira"] = list(executor.map(jiradata, df["siemref"]))
-            sleep(0.5)
+                sleep(0.5)
         df = pandas.concat(dfs)
         df["sync_action"] = df.apply(checkrow, axis=1)
         return df