From 43fe756b2454f4ff4f2dd75a10c159f6df1aca47 Mon Sep 17 00:00:00 2001
From: Sujan Adhikari <109404840+Sujanadh@users.noreply.github.com>
Date: Wed, 3 Jan 2024 18:15:28 +0545
Subject: [PATCH] fix: optimize the performance of project dashboard (#1079)

* fix: optimze the performance of project dashboard

* updated project schema

---------

Co-authored-by: sujanadh <sujanadh07@gmail.com>
---
 src/backend/app/projects/project_crud.py    | 29 ++++++++++++++-------
 src/backend/app/projects/project_routes.py  | 14 ++++++++--
 src/backend/app/projects/project_schemas.py |  6 ++---
 3 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py
index 1a95050961..c079bee075 100644
--- a/src/backend/app/projects/project_crud.py
+++ b/src/backend/app/projects/project_crud.py
@@ -2397,21 +2397,32 @@ async def get_dashboard_detail(project_id: int, db: Session):
     db_organization = await organization_crud.get_organisation_by_id(db, project.organisation_id)
 
     s3_project_path = f"/{project.organisation_id}/{project_id}"
-    s3_submission_path = f"/{s3_project_path}/submissions.meta.json"
+    s3_submission_path = f"/{s3_project_path}/submission.zip"
+    s3_submission_meta_path = f"/{s3_project_path}/submissions.meta.json"
+
     try:
-        file = get_obj_from_bucket(settings.S3_BUCKET_NAME, s3_submission_path)
-        project.last_active = (json.loads(file.getvalue()))["last_submission"]
+        submission = get_obj_from_bucket(settings.S3_BUCKET_NAME, s3_submission_path)
+        with zipfile.ZipFile(submission, "r") as zip_ref:
+            with zip_ref.open("submissions.json") as file_in_zip:
+                content = file_in_zip.read()
+        content = json.loads(content)
+        project.total_submission = len(content)
+        submission_meta = get_obj_from_bucket(settings.S3_BUCKET_NAME, s3_submission_meta_path)
+        project.last_active = (json.loads(submission_meta.getvalue()))["last_submission"]
     except ValueError:
+        project.total_submission = 0
         pass
 
-    contributors = db.query(db_models.DbTaskHistory).filter(db_models.DbTaskHistory.project_id==project_id).all()
-    unique_user_ids = {user.user_id for user in contributors if user.user_id is not None}
+    contributors = db.query(
+        db_models.DbTaskHistory.user_id
+        ).filter(
+        db_models.DbTaskHistory.project_id == project_id,
+        db_models.DbTaskHistory.user_id.isnot(None)
+        ).distinct().count()
 
-    project.organization = db_organization.name
-    project.organization_logo = db_organization.logo
-    project.total_contributors = len(unique_user_ids)
-    project.total_submission = await submission_crud.get_submission_count_of_a_project(db, project_id)
     project.total_tasks = await tasks_crud.get_task_count_in_project(db, project_id)
+    project.organization, project.organization_logo = db_organization.name, db_organization.logo
+    project.total_contributors = contributors
 
     return project
 
diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py
index 8e4748ebfb..95230210e9 100644
--- a/src/backend/app/projects/project_routes.py
+++ b/src/backend/app/projects/project_routes.py
@@ -46,6 +46,7 @@
 from sqlalchemy.orm import Session
 
 from app.auth.osm import AuthUser, login_required
+from app.submission import submission_crud
 
 from ..central import central_crud
 from ..db import database, db_models
@@ -1220,7 +1221,9 @@ async def get_template_file(
 
 @router.get("/project_dashboard/{project_id}", response_model=project_schemas.ProjectDashboard)
 async def project_dashboard(
-    project_id: int, db: Session = Depends(database.get_db)
+    project_id: int, 
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(database.get_db)
 ):
     """
     Get the project dashboard details.
@@ -1232,8 +1235,15 @@ async def project_dashboard(
     Returns:
         ProjectDashboard: The project dashboard details.
     """
+    data = await project_crud.get_dashboard_detail(project_id, db)
+    background_task_id = await project_crud.insert_background_task_into_database(
+        db, "sync_submission", project_id
+    )
 
-    return await project_crud.get_dashboard_detail(project_id, db)
+    background_tasks.add_task(
+        submission_crud.update_submission_in_s3, db, project_id, background_task_id
+    )
+    return data
 
 @router.get("/contributors/{project_id}")
 async def get_contributors(project_id: int, db: Session = Depends(database.get_db)):
diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py
index 01f5a5d1a7..149997aeb1 100644
--- a/src/backend/app/projects/project_schemas.py
+++ b/src/backend/app/projects/project_schemas.py
@@ -154,11 +154,11 @@ class BackgroundTaskStatus(BaseModel):
 class ProjectDashboard(BaseModel):
     project_name_prefix: str
     organization: str
-    organization_logo: Optional[str] = None
     total_tasks: int
-    total_submission: int
-    total_contributors: int
     created: datetime
+    organization_logo: Optional[str] = None
+    total_submission: Optional[int] = None
+    total_contributors: Optional[int] = None
     last_active: Optional[str] = None
 
     @field_validator("created", mode="before")