From f73a92ea15daca1a22875daf4d90479f982704d3 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Wed, 2 Mar 2022 18:08:13 +0100 Subject: [PATCH 1/6] add script to automatically update project status --- .../mapswipe_workers/definitions.py | 2 + .../mapswipe_workers/utils/slack_helper.py | 18 +++++ .../python_scripts/update_project_status.py | 69 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 mapswipe_workers/python_scripts/update_project_status.py diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index 07cadfb42..a88d7830c 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -113,6 +113,8 @@ class MessageType(Enum): FAIL = 2 NOTIFICATION_90 = 3 NOTIFICATION_100 = 4 + PROJECT_STATUS_FINISHED = 5 + PROJECT_STATUS_ACTIVE = 6 class ProjectType(Enum): diff --git a/mapswipe_workers/mapswipe_workers/utils/slack_helper.py b/mapswipe_workers/mapswipe_workers/utils/slack_helper.py index 887e09d97..4a41bf475 100644 --- a/mapswipe_workers/mapswipe_workers/utils/slack_helper.py +++ b/mapswipe_workers/mapswipe_workers/utils/slack_helper.py @@ -58,6 +58,24 @@ def send_slack_message( + "and activate another one." ) slack_client.chat_postMessage(channel="mapswipe_managers", text=message) + elif message_type == MessageType.PROJECT_STATUS_FINISHED: + message = ( + "### SET PROJECT STATUS TO FINISHED ###\n" + + f"Project Name: {project_name}\n" + + f"Project Id: {project_id}\n\n" + + "The status of the project has been set to 'finished' " + + "by MapSwipe's backend workers." + ) + slack_client.chat_postMessage(channel=SLACK_CHANNEL, text=message) + elif message_type == MessageType.PROJECT_STATUS_ACTIVE: + message = ( + "### SET PROJECT STATUS TO ACTIVE ###\n" + + f"Project Name: {project_name}\n" + + f"Project Id: {project_id}\n\n" + + "The status of the project has been set to 'active' " + + "by MapSwipe's backend workers." + ) + slack_client.chat_postMessage(channel=SLACK_CHANNEL, text=message) else: # TODO: Raise an Exception pass diff --git a/mapswipe_workers/python_scripts/update_project_status.py b/mapswipe_workers/python_scripts/update_project_status.py new file mode 100644 index 000000000..435206639 --- /dev/null +++ b/mapswipe_workers/python_scripts/update_project_status.py @@ -0,0 +1,69 @@ +from mapswipe_workers.auth import firebaseDB +from mapswipe_workers.definitions import MessageType, logger +from mapswipe_workers.utils.slack_helper import send_slack_message + + +def get_projects(status): + """Load 'active' projects from Firebase.""" + fb_db = firebaseDB() + projects = ( + fb_db.reference("v2/projects/").order_by_child("status").equal_to(status).get() + ) + logger.info(f"got {len(projects)} {status} projects from firebase") + return projects + + +def filter_projects_by_name_and_progress(projects, filter_string, progress_threshold): + """Load 'active' projects from Firebase.""" + selected_project_ids = [] + for project_id in projects.keys(): + name = projects[project_id]["name"] + progress = projects[project_id]["progress"] + if filter_string in name and progress >= progress_threshold: + selected_project_ids.append(project_id) + + logger.info( + f"selected {len(selected_project_ids)} project(s) which contain(s) " + f"'{filter_string}' in the project name and progress >= {progress_threshold}%." + ) + return selected_project_ids + + +def set_status_in_firebase(project_id, project_name, new_status): + """Change status of a project in Firebase.""" + # change status in firebase + fb_db = firebaseDB() + fb_db.reference(f"v2/projects/{project_id}/status").set(new_status) + logger.info(f"set project status to '{new_status}' for project {project_id}") + + # send slack message + if new_status == "finished": + send_slack_message( + MessageType.PROJECT_STATUS_FINISHED, project_name, project_id + ) + elif new_status == "active": + send_slack_message(MessageType.PROJECT_STATUS_ACTIVE, project_name, project_id) + + +if __name__ == "__main__": + """Check if project status should be updated and change value in Firebase.""" + filter_string = "Test" + + active_projects = get_projects(status="active") + finished_projects = filter_projects_by_name_and_progress( + active_projects, filter_string, progress_threshold=1, + ) + + inactive_projects = get_projects(status="inactive") + new_active_projects = filter_projects_by_name_and_progress( + inactive_projects, filter_string, progress_threshold=0, + )[0 : len(finished_projects)] + + if len(new_active_projects) > 0: + for project_id in finished_projects: + project_name = active_projects[project_id]["name"] + set_status_in_firebase(project_id, project_name, new_status="finished") + + for project_id in new_active_projects: + project_name = inactive_projects[project_id]["name"] + set_status_in_firebase(project_id, project_name, new_status="active") From 23d54740cafd69a9b43881e9bc6c0b35d119f830 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 3 Mar 2022 10:56:25 +0100 Subject: [PATCH 2/6] add schedule to script --- .../python_scripts/update_project_status.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/mapswipe_workers/python_scripts/update_project_status.py b/mapswipe_workers/python_scripts/update_project_status.py index 435206639..2486c17f9 100644 --- a/mapswipe_workers/python_scripts/update_project_status.py +++ b/mapswipe_workers/python_scripts/update_project_status.py @@ -1,3 +1,8 @@ +import sys +import time + +import schedule as sched + from mapswipe_workers.auth import firebaseDB from mapswipe_workers.definitions import MessageType, logger from mapswipe_workers.utils.slack_helper import send_slack_message @@ -45,13 +50,12 @@ def set_status_in_firebase(project_id, project_name, new_status): send_slack_message(MessageType.PROJECT_STATUS_ACTIVE, project_name, project_id) -if __name__ == "__main__": - """Check if project status should be updated and change value in Firebase.""" - filter_string = "Test" - +def run_update_project_status(filter_string): + """Run the workflow to update project status for all filtered projects.""" + logger.info("### Start update project status workflow ###") active_projects = get_projects(status="active") finished_projects = filter_projects_by_name_and_progress( - active_projects, filter_string, progress_threshold=1, + active_projects, filter_string, progress_threshold=100, ) inactive_projects = get_projects(status="inactive") @@ -59,6 +63,9 @@ def set_status_in_firebase(project_id, project_name, new_status): inactive_projects, filter_string, progress_threshold=0, )[0 : len(finished_projects)] + # Here we check that there is at least one inactive project + # which can be activated in the app. + # We do this to avoid that there is no project left in the app. if len(new_active_projects) > 0: for project_id in finished_projects: project_name = active_projects[project_id]["name"] @@ -67,3 +74,22 @@ def set_status_in_firebase(project_id, project_name, new_status): for project_id in new_active_projects: project_name = inactive_projects[project_id]["name"] set_status_in_firebase(project_id, project_name, new_status="active") + logger.info("### Finished update project status workflow ###") + + +if __name__ == "__main__": + """Check if project status should be updated and change value in Firebase.""" + try: + filter_string = sys.argv[1] + time_interval = int(sys.argv[2]) + except IndexError: + logger.info("Please provide the filter_string and time_interval arguments.") + sys.exit(1) + + sched.every(time_interval).minutes.do( + run_update_project_status, filter_string=filter_string + ).run() + + while True: + sched.run_pending() + time.sleep(1) From e5aa57513482f4ca07210a25f44a666483181b9c Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 3 Mar 2022 11:08:48 +0100 Subject: [PATCH 3/6] lowercase --- mapswipe_workers/python_scripts/update_project_status.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/python_scripts/update_project_status.py b/mapswipe_workers/python_scripts/update_project_status.py index 2486c17f9..f348e9bb9 100644 --- a/mapswipe_workers/python_scripts/update_project_status.py +++ b/mapswipe_workers/python_scripts/update_project_status.py @@ -19,12 +19,12 @@ def get_projects(status): def filter_projects_by_name_and_progress(projects, filter_string, progress_threshold): - """Load 'active' projects from Firebase.""" + """Filter projects by name (lowercase) and progress.""" selected_project_ids = [] for project_id in projects.keys(): name = projects[project_id]["name"] progress = projects[project_id]["progress"] - if filter_string in name and progress >= progress_threshold: + if filter_string.lower() in name.lower() and progress >= progress_threshold: selected_project_ids.append(project_id) logger.info( @@ -78,7 +78,9 @@ def run_update_project_status(filter_string): if __name__ == "__main__": - """Check if project status should be updated and change value in Firebase.""" + """Use this command to run in docker container. + docker-compose run -d mapswipe_workers_creation python3 python_scripts/update_project_status.py "test" 30 # noqa + """ try: filter_string = sys.argv[1] time_interval = int(sys.argv[2]) From 31713783dc62eec97424fde227c4017a767d8d9b Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 3 Mar 2022 12:24:00 +0100 Subject: [PATCH 4/6] add type hints and better docstrings --- .../mapswipe_workers/definitions.py | 2 ++ .../python_scripts/update_project_status.py | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index a88d7830c..26d6aac13 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -109,6 +109,8 @@ class CustomError(Exception): class MessageType(Enum): + """The different types of messages sent via Slack.""" + SUCCESS = 1 FAIL = 2 NOTIFICATION_90 = 3 diff --git a/mapswipe_workers/python_scripts/update_project_status.py b/mapswipe_workers/python_scripts/update_project_status.py index f348e9bb9..ccc7c1265 100644 --- a/mapswipe_workers/python_scripts/update_project_status.py +++ b/mapswipe_workers/python_scripts/update_project_status.py @@ -1,5 +1,6 @@ import sys import time +from typing import List import schedule as sched @@ -8,7 +9,7 @@ from mapswipe_workers.utils.slack_helper import send_slack_message -def get_projects(status): +def get_projects(status: str) -> dict: """Load 'active' projects from Firebase.""" fb_db = firebaseDB() projects = ( @@ -18,7 +19,9 @@ def get_projects(status): return projects -def filter_projects_by_name_and_progress(projects, filter_string, progress_threshold): +def filter_projects_by_name_and_progress( + projects: dict, filter_string: str, progress_threshold: int +) -> List[str]: """Filter projects by name (lowercase) and progress.""" selected_project_ids = [] for project_id in projects.keys(): @@ -34,7 +37,7 @@ def filter_projects_by_name_and_progress(projects, filter_string, progress_thres return selected_project_ids -def set_status_in_firebase(project_id, project_name, new_status): +def set_status_in_firebase(project_id: str, project_name: str, new_status: str) -> None: """Change status of a project in Firebase.""" # change status in firebase fb_db = firebaseDB() @@ -50,7 +53,7 @@ def set_status_in_firebase(project_id, project_name, new_status): send_slack_message(MessageType.PROJECT_STATUS_ACTIVE, project_name, project_id) -def run_update_project_status(filter_string): +def run_update_project_status(filter_string: str) -> None: """Run the workflow to update project status for all filtered projects.""" logger.info("### Start update project status workflow ###") active_projects = get_projects(status="active") @@ -80,6 +83,14 @@ def run_update_project_status(filter_string): if __name__ == "__main__": """Use this command to run in docker container. docker-compose run -d mapswipe_workers_creation python3 python_scripts/update_project_status.py "test" 30 # noqa + + You can two arguments to the script + - filter string, e.g. "test" + - time interval in minutes, e.g. 30 + + Make sure that you don't run this script too frequently as it pulls data from firebase and + this will have implications on costs. Running this script once every 15-30 minutes should be totally fine. + This means that there can be a "delay" in setting a project to finished about roughly the same time. """ try: filter_string = sys.argv[1] From df8e720366b9d974585f00dead39e26efc5c9ce7 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 3 Mar 2022 12:36:59 +0100 Subject: [PATCH 5/6] sort projects by projectNumber --- .../python_scripts/update_project_status.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mapswipe_workers/python_scripts/update_project_status.py b/mapswipe_workers/python_scripts/update_project_status.py index ccc7c1265..d66db12fc 100644 --- a/mapswipe_workers/python_scripts/update_project_status.py +++ b/mapswipe_workers/python_scripts/update_project_status.py @@ -1,6 +1,6 @@ import sys import time -from typing import List +from typing import List, OrderedDict import schedule as sched @@ -9,7 +9,7 @@ from mapswipe_workers.utils.slack_helper import send_slack_message -def get_projects(status: str) -> dict: +def get_projects(status: str) -> OrderedDict: """Load 'active' projects from Firebase.""" fb_db = firebaseDB() projects = ( @@ -20,7 +20,7 @@ def get_projects(status: str) -> dict: def filter_projects_by_name_and_progress( - projects: dict, filter_string: str, progress_threshold: int + projects: OrderedDict, filter_string: str, progress_threshold: int ) -> List[str]: """Filter projects by name (lowercase) and progress.""" selected_project_ids = [] @@ -62,6 +62,12 @@ def run_update_project_status(filter_string: str) -> None: ) inactive_projects = get_projects(status="inactive") + # We sort projects by their attribute "projectNumber" to ensure that + # always the lowest one will be set to "status=active" next. + inactive_projects = OrderedDict( + sorted(inactive_projects.items(), key=lambda x: x[1]["projectNumber"]) + ) + new_active_projects = filter_projects_by_name_and_progress( inactive_projects, filter_string, progress_threshold=0, )[0 : len(finished_projects)] From 997ec11caefbd5baab187ee642a0e8ee4f2bd078 Mon Sep 17 00:00:00 2001 From: Hagellach37 Date: Thu, 3 Mar 2022 12:42:04 +0100 Subject: [PATCH 6/6] fix typo --- mapswipe_workers/python_scripts/update_project_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapswipe_workers/python_scripts/update_project_status.py b/mapswipe_workers/python_scripts/update_project_status.py index d66db12fc..4708cea62 100644 --- a/mapswipe_workers/python_scripts/update_project_status.py +++ b/mapswipe_workers/python_scripts/update_project_status.py @@ -90,7 +90,7 @@ def run_update_project_status(filter_string: str) -> None: """Use this command to run in docker container. docker-compose run -d mapswipe_workers_creation python3 python_scripts/update_project_status.py "test" 30 # noqa - You can two arguments to the script + You need to use two arguments for the script - filter string, e.g. "test" - time interval in minutes, e.g. 30