fix: move OdkDecrypted logic into project_deps (#1239)

* fix:commit features to db before generating task files

* refactor: convert repeated odk_cred into a function

* fix: reuse DbProject object when retrieving odk creds

---------

Co-authored-by: sujanadh <sujanadh07@gmail.com>
Co-authored-by: spwoodcock <sam.woodcock@protonmail.com>

src/backend/app/projects/project_crud.py

@@ -67,7 +67,7 @@
     split_geojson_by_task_areas,
 )
 from app.models.enums import HTTPStatus, ProjectRole
-from app.projects import project_schemas
+from app.projects import project_deps, project_schemas
 from app.s3 import add_obj_to_bucket, get_obj_from_bucket
 from app.tasks import tasks_crud
 from app.users import user_crud
@@ -143,6 +143,11 @@ async def get_project(db: Session, project_id: int):
         .filter(db_models.DbProject.id == project_id)
         .first()
     )
+    if not db_project:
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND,
+            detail=f"Project with id {project_id} does not exist",
+        )
     return db_project
 
 
@@ -1054,9 +1059,6 @@ async def upload_custom_geojson_extract(
     project = await get_project(db, project_id)
     log.debug(f"Uploading custom data extract for project: {project}")
 
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found")
-
     featcol_filtered = parse_and_filter_geojson(geojson_str)
     if not featcol_filtered:
         raise HTTPException(
@@ -1234,14 +1236,8 @@ def generate_project_files(
                 detail=f"Project with id {project_id} does not exist",
             )
 
-        # Get odk credentials from project.
-        odk_credentials = {
-            "odk_central_url": project.odk_central_url,
-            "odk_central_user": project.odk_central_user,
-            "odk_central_password": project.odk_central_password,
-        }
-
-        odk_credentials = project_schemas.ODKCentralDecrypted(**odk_credentials)
+        odk_sync = async_to_sync(project_deps.get_odk_credentials)
+        odk_credentials = odk_sync(db, project)
 
         if custom_form:
             log.debug("User provided custom XLSForm")
@@ -1664,11 +1660,7 @@ async def update_project_form(
     odk_id = project.odkid
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project.odk_central_url,
-        odk_central_user=project.odk_central_user,
-        odk_central_password=project.odk_central_password,
-    )
+    odk_credentials = await project_deps.get_odk_credentials(db, project)
 
     if form:
         xlsform = f"/tmp/custom_form.{form_type}"

src/backend/app/projects/project_deps.py

@@ -18,7 +18,7 @@
 
 """Project dependencies for use in Depends."""
 
-from typing import Optional
+from typing import Any, Optional, Union
 
 from fastapi import Depends
 from fastapi.exceptions import HTTPException
@@ -27,6 +27,7 @@
 from app.db.database import get_db
 from app.db.db_models import DbProject
 from app.models.enums import HTTPStatus
+from app.projects import project_crud, project_schemas
 
 
 async def get_project_by_id(
@@ -45,3 +46,19 @@ async def get_project_by_id(
         )
 
     return db_project
+
+
+async def get_odk_credentials(db: Session, project: Union[int, Any]):
+    """Get odk credentials of project."""
+    if isinstance(project, int):
+        db_project = await project_crud.get_project(db, project)
+    else:
+        db_project = project
+
+    odk_credentials = {
+        "odk_central_url": db_project.odk_central_url,
+        "odk_central_user": db_project.odk_central_user,
+        "odk_central_password": db_project.odk_central_password,
+    }
+
+    return project_schemas.ODKCentralDecrypted(**odk_credentials)

src/backend/app/projects/project_routes.py

@@ -234,11 +234,7 @@ async def delete_project(
         f"deletion of project {project.id}"
     )
     # Odk crendentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project.odk_central_url,
-        odk_central_user=project.odk_central_user,
-        odk_central_password=project.odk_central_password,
-    )
+    odk_credentials = await project_deps.get_odk_credentials(db, project)
     # Delete ODK Central project
     await central_crud.delete_odk_project(project.odkid, odk_credentials)
     # Delete FMTM project
@@ -619,6 +615,7 @@ async def generate_files(
 
     form_category = project.xform_title
     custom_xls_form = None
+    file_ext = None
     if xls_form_upload:
         log.debug("Validating uploaded XLS form")
 
@@ -641,7 +638,7 @@ async def generate_files(
     # Create task in db and return uuid
     log.debug(f"Creating export background task for project ID: {project_id}")
     background_task_id = await project_crud.insert_background_task_into_database(
-        db, project_id=project_id
+        db, project_id=str(project_id)
     )
 
     log.debug(f"Submitting {background_task_id} to background tasks stack")
@@ -879,8 +876,6 @@ async def download_form(
 ):
     """Download the XLSForm for a project."""
     project = await project_crud.get_project(db, project_id)
-    if not project:
-        raise HTTPException(status_code=404, detail="Project not found")
 
     headers = {
         "Content-Disposition": "attachment; filename=submission_data.xls",

src/backend/app/submissions/submission_crud.py

@@ -41,7 +41,7 @@
 from app.central.central_crud import get_odk_form, get_odk_project, list_odk_xforms
 from app.config import settings
 from app.db import db_models
-from app.projects import project_crud, project_schemas
+from app.projects import project_crud, project_deps
 from app.s3 import add_obj_to_bucket, get_obj_from_bucket
 from app.tasks import tasks_crud
 
@@ -74,12 +74,8 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None)
         )
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_sync = async_to_sync(project_deps.get_odk_credentials)
+    odk_credentials = odk_sync(db, project_info)
     xform = get_odk_form(odk_credentials)
 
     # If task id is not provided, submission for all the task are listed
@@ -144,21 +140,13 @@ def convert_to_osm(db: Session, project_id: int, task_id: int):
     get_project_sync = async_to_sync(project_crud.get_project)
     project_info = get_project_sync(db, project_id)
 
-    # Return exception if project is not found
-    if not project_info:
-        raise HTTPException(status_code=404, detail="Project not found")
-
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
     form_category = project_info.xform_title
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_sync = async_to_sync(project_deps.get_odk_credentials)
+    odk_credentials = odk_sync(db, project_info)
     # Get ODK Form with odk credentials from the project.
     xform = get_odk_form(odk_credentials)
 
@@ -227,22 +215,14 @@ def gather_all_submission_csvs(db, project_id):
     get_project_sync = async_to_sync(project_crud.get_project)
     project_info = get_project_sync(db, project_id)
 
-    # Return empty list if project is not found
-    if not project_info:
-        raise HTTPException(status_code=404, detail="Project not found")
-
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
     form_category = project_info.xform_title
     project_tasks = project_info.tasks
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_sync = async_to_sync(project_deps.get_odk_credentials)
+    odk_credentials = odk_sync(db, project_info)
     # Get ODK Form with odk credentials from the project.
     xform = get_odk_form(odk_credentials)
 
@@ -332,11 +312,8 @@ def update_submission_in_s3(
         project = get_project_sync(db, project_id)
 
         # Gather metadata
-        odk_credentials = project_schemas.ODKCentralDecrypted(
-            odk_central_url=project.odk_central_url,
-            odk_central_user=project.odk_central_user,
-            odk_central_password=project.odk_central_password,
-        )
+        odk_sync = async_to_sync(project_deps.get_odk_credentials)
+        odk_credentials = odk_sync(db, project)
         odk_forms = list_odk_xforms(project.odkid, odk_credentials, True)
 
         # Get latest submission date
@@ -429,12 +406,8 @@ def get_all_submissions_json(db: Session, project_id):
     project_info = get_project_sync(db, project_id)
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_sync = async_to_sync(project_deps.get_odk_credentials)
+    odk_credentials = odk_sync(db, project_info)
     project = get_odk_project(odk_credentials)
 
     get_task_id_list_sync = async_to_sync(tasks_crud.get_task_id_list)
@@ -491,22 +464,13 @@ async def download_submission(
     """Download submission data from ODK Central and aggregate."""
     project_info = await project_crud.get_project(db, project_id)
 
-    # Return empty list if project is not found
-    if not project_info:
-        raise HTTPException(status_code=404, detail="Project not found")
-
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
     form_category = project_info.xform_title
     project_tasks = project_info.tasks
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_credentials = await project_deps.get_odk_credentials(db, project_info)
     # Get ODK Form with odk credentials from the project.
     xform = get_odk_form(odk_credentials)
     if not export_json:
@@ -611,12 +575,7 @@ async def get_submission_points(db: Session, project_id: int, task_id: int = Non
     form_category = project_info.xform_title
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_credentials = await project_deps.get_odk_credentials(db, project_info)
     xform = get_odk_form(odk_credentials)
 
     if task_id:
@@ -666,22 +625,13 @@ async def get_submission_count_of_a_project(db: Session, project_id: int):
     """Return the total number of submissions made for a project."""
     project_info = await project_crud.get_project(db, project_id)
 
-    # Return empty list if project is not found
-    if not project_info:
-        raise HTTPException(status_code=404, detail="Project not found")
-
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
     form_category = project_info.xform_title
     project_tasks = project_info.tasks
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project_info.odk_central_url,
-        odk_central_user=project_info.odk_central_user,
-        odk_central_password=project_info.odk_central_password,
-    )
-
+    odk_credentials = await project_deps.get_odk_credentials(db, project_info)
     # Get ODK Form with odk credentials from the project.
     xform = get_odk_form(odk_credentials)
 
@@ -850,14 +800,10 @@ async def get_submission_by_task(
     Returns:
         Tuple: A tuple containing the list of submissions and the count.
     """
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project.odk_central_url,
-        odk_central_user=project.odk_central_user,
-        odk_central_password=project.odk_central_password,
-    )
+    odk_credentials = await project_deps.get_odk_credentials(db, project)
 
     xform = get_odk_form(odk_credentials)
-    data = xform.listSubmissions(project.odkid, task_id, filters)
+    data = xform.listSubmissions(project.odkid, str(task_id), filters)
     submissions = data.get("value", [])
     count = data.get("@odata.count", 0)
 

src/backend/app/submissions/submission_routes.py

@@ -371,11 +371,7 @@ async def get_submission_form_fields(
     """
     project = await project_crud.get_project(db, project_id)
     task_list = await tasks_crud.get_task_id_list(db, project_id)
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project.odk_central_url,
-        odk_central_user=project.odk_central_user,
-        odk_central_password=project.odk_central_password,
-    )
+    odk_credentials = await project_deps.get_odk_credentials(db, project)
     odk_form = central_crud.get_odk_form(odk_credentials)
     response = odk_form.form_fields(project.odkid, str(task_list[0]))
     return response

src/backend/app/tasks/tasks_routes.py

@@ -29,7 +29,7 @@
 from app.central import central_crud
 from app.db import database
 from app.models.enums import TaskStatus
-from app.projects import project_crud, project_schemas
+from app.projects import project_crud, project_deps
 from app.tasks import tasks_crud, tasks_schemas
 
 router = APIRouter(
@@ -145,11 +145,7 @@ async def task_features_count(
     project = await project_crud.get_project(db, project_id)
 
     # ODK Credentials
-    odk_credentials = project_schemas.ODKCentralDecrypted(
-        odk_central_url=project.odk_central_url,
-        odk_central_user=project.odk_central_user,
-        odk_central_password=project.odk_central_password,
-    )
+    odk_credentials = await project_deps.get_odk_credentials(db, project)
 
     odk_details = central_crud.list_odk_xforms(project.odkid, odk_credentials, True)