refactor: remove projects.xform_title completely, in favour of xform_…

…category

src/backend/app/central/central_crud.py

@@ -389,7 +389,7 @@ async def update_and_publish_form(
         form_name_prefix (str): Prefix for the form name in ODK Central.
         odk_credentials (project_schemas.ODKCentralDecrypted): ODK Central creds.
     """
-    odk_form_name = f"{form_name_prefix}_{task_id}"
+    odk_form_name = f"{form_name_prefix}_task_{task_id}"
     xform_data = await read_and_test_xform(
         xform_data,
         form_file_ext,

src/backend/app/db/db_models.py

@@ -582,8 +582,6 @@ def tasks_bad(self):
             .count()
         )
 
-    # XForm title for ODK Central (this never changes once set)
-    xform_title = cast(str, Column(String))
     # XForm category specified
     xform_category = cast(str, Column(String))
 

src/backend/app/projects/project_crud.py

@@ -269,7 +269,7 @@ async def create_project_with_project_info(
         "Creating project in FMTM database with vars: "
         f"project_user: {current_user.username} | "
         f"project_name: {project_metadata.project_info.name} | "
-        f"xform_category: {project_metadata.xform_title} | "
+        f"xform_category: {project_metadata.xform_category} | "
         f"hashtags: {project_metadata.hashtags} | "
         f"organisation_id: {project_metadata.organisation_id}"
     )
@@ -284,7 +284,6 @@ async def create_project_with_project_info(
     db_project = db_models.DbProject(
         author_id=current_user.id,
         odkid=odk_project_id,
-        xform_category=project_metadata.xform_title,
         **project_metadata.model_dump(exclude=["project_info", "outline_geojson"]),
     )
     db.add(db_project)
@@ -305,35 +304,6 @@ async def create_project_with_project_info(
     return await convert_to_app_project(db_project)
 
 
-async def upload_xlsform(
-    db: Session,
-    xlsform: str,
-    name: str,
-    category: str,
-):
-    """Upload a custom XLSForm from the user."""
-    try:
-        forms = table(
-            "xlsforms",
-            column("title"),
-            column("xls"),
-            column("xml"),
-            column("id"),
-            column("category"),
-        )
-        ins = insert(forms).values(title=name, xls=xlsform, category=category)
-        sql = ins.on_conflict_do_update(
-            constraint="xlsforms_title_key",
-            set_=dict(title=name, xls=xlsform, category=category),
-        )
-        db.execute(sql)
-        db.commit()
-        return True
-    except Exception as e:
-        log.exception(e)
-        raise HTTPException(status_code=400, detail={"message": str(e)}) from e
-
-
 async def create_tasks_from_geojson(
     db: Session,
     project_id: int,
@@ -1292,14 +1262,9 @@ def generate_project_files(
         read_xform_sync = async_to_sync(central_crud.read_and_test_xform)
         xform_data = read_xform_sync(xlsform, form_file_ext, return_form_data=True)
 
-        # Generate extra task info as dicts (to allow closing db connection)
-        task_form_name_dict = {}
-        for task_id in task_extract_dict.keys():
-            project_name = project.project_name_prefix
-            category = project.xform_title
-            task_form_name_dict[task_id] = f"{project_name}_{category}_{task_id}"
-
-        # Get ODK Project ID from database
+        # Get project name for XForm name
+        project_name = project.project_name_prefix
+        # Get ODK Project ID
         project_odk_id = project.odkid
 
         # Run with expensive task via threadpool
@@ -1317,7 +1282,7 @@ def wrap_generate_task_files(task_id):
                     project_odk_id,
                     task_id,
                     task_extract_dict[task_id],
-                    task_form_name_dict[task_id],
+                    f"{project_name}_task_{task_id}",
                     xform_data,
                     odk_credentials,
                 )

src/backend/app/projects/project_routes.py

@@ -46,7 +46,7 @@
 from sqlalchemy.sql import text
 
 from app.auth.osm import AuthUser, login_required
-from app.auth.roles import mapper, org_admin, project_admin, super_admin
+from app.auth.roles import mapper, org_admin, project_admin
 from app.central import central_crud
 from app.db import database, db_models
 from app.db.postgis_utils import (
@@ -384,29 +384,6 @@ async def project_partial_update(
     return project
 
 
-@router.post("/upload_xlsform")
-async def upload_custom_xls(
-    upload: UploadFile = File(...),
-    category: str = Form(...),
-    db: Session = Depends(database.get_db),
-    current_user: db_models.DbUser = Depends(super_admin),
-):
-    """Upload a custom XLSForm to the database.
-
-    Args:
-        upload (UploadFile): the XLSForm file
-        category (str): the category of the XLSForm.
-        db (Session): the DB session, provided automatically.
-        current_user (DbUser): Check if user is super_admin
-    """
-    content = await upload.read()  # read file content
-    name = upload.filename.split(".")[0]  # get name of file without extension
-    await project_crud.upload_xlsform(db, content, name, category)
-
-    # FIXME: fix return value
-    return {"xform_title": f"{category}"}
-
-
 @router.post("/{project_id}/upload-task-boundaries")
 async def upload_project_task_boundaries(
     project_id: int,
@@ -588,7 +565,7 @@ async def generate_files(
 
     project = org_user_dict.get("project")
 
-    xform_title = project.xform_title
+    xform_category = project.xform_category
     custom_xls_form = None
     file_ext = None
     if xls_form_upload:
@@ -621,7 +598,7 @@ async def generate_files(
         db,
         project_id,
         BytesIO(custom_xls_form) if custom_xls_form else None,
-        xform_title,
+        xform_category,
         file_ext if xls_form_upload else ".xls",
         background_task_id,
     )
@@ -853,8 +830,7 @@ async def download_form(
         "Content-Type": "application/media",
     }
     if not project.form_xls:
-        project_category = project.xform_title
-        xlsform_path = f"{xlsforms_path}/{project_category}.xls"
+        xlsform_path = f"{xlsforms_path}/{project.xform_category}.xls"
         if os.path.exists(xlsform_path):
             return FileResponse(xlsform_path, filename="form.xls")
         else:
@@ -901,13 +877,13 @@ async def update_project_form(
         with open(xlsform_path, "rb") as f:
             new_xform_data = BytesIO(f.read())
 
-    # NOTE never update xform_title as this links to ODK Central
+    # Update form category in database
     project.xform_category = category
     # Commit changes to db
     db.commit()
 
     # The reference to the form via ODK Central API (minus task_id)
-    xform_name_prefix = f"{project.project_name_prefix}_{project.xform_title}"
+    xform_name_prefix = project.project_name_prefix
 
     # Get ODK Central credentials for project
     odk_creds = await project_deps.get_odk_credentials(db, project.id)

src/backend/app/projects/project_schemas.py

@@ -136,7 +136,7 @@ class ProjectIn(BaseModel):
     """Upload new project."""
 
     project_info: ProjectInfo
-    xform_title: str
+    xform_category: str
     organisation_id: Optional[int] = None
     hashtags: Optional[List[str]] = None
     task_split_type: Optional[TaskSplitType] = None
@@ -313,7 +313,7 @@ class ProjectBase(BaseModel):
     project_info: ProjectInfo
     status: ProjectStatus
     # location_str: str
-    xform_title: Optional[str] = None
+    xform_category: Optional[str] = None
     hashtags: Optional[List[str]] = None
     organisation_id: Optional[int] = None
 

src/backend/app/submissions/submission_crud.py

@@ -62,7 +62,6 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None)
 
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
-    xform_title = project_info.xform_title
     project_tasks = project_info.tasks
 
     if not (
@@ -87,9 +86,9 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None)
 
         data = []
 
-        for id in task_list:
+        for task_list_id in task_list:
             # XML Form Id is a combination or project_name, category and task_id
-            xform_name = f"{project_name}_{xform_title}_{id}"
+            xform_name = f"{project_name}_task_{task_list_id}"
             submission_list = xform.listSubmissions(odkid, xform_name)
 
             # data.append(submission_list)
@@ -100,7 +99,7 @@ def get_submission_of_project(db: Session, project_id: int, task_id: int = None)
 
     else:
         # If task_id is provided, submission made to this particular task is returned.
-        xform_name = f"{project_name}_{xform_title}_{task_id}"
+        xform_name = f"{project_name}_task_{task_id}"
         submission_list = xform.listSubmissionBasicInfo(odkid, xform_name)
         for x in submission_list:
             x["submitted_by"] = xform_name
@@ -143,8 +142,7 @@ def convert_to_osm(db: Session, project_id: int, task_id: int):
 
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
-    xform_title = project_info.xform_title
-    xform_name = f"{project_name}_{xform_title}_{task_id}"
+    xform_name = f"{project_name}_task_{task_id}"
 
     # ODK Credentials
     odk_sync = async_to_sync(project_deps.get_odk_credentials)
@@ -153,7 +151,7 @@ def convert_to_osm(db: Session, project_id: int, task_id: int):
     xform = get_odk_form(odk_credentials)
 
     # Create a new ZIP file for the extracted files
-    final_zip_file_path = f"/tmp/{project_name}_{xform_title}_osm.zip"
+    final_zip_file_path = f"/tmp/{project_name}_osm.zip"
 
     # Remove the ZIP file if it already exists
     if os.path.exists(final_zip_file_path):
@@ -218,7 +216,6 @@ async def gather_all_submission_csvs(db, project_id):
 
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
-    xform_title = project_info.xform_title
     project_tasks = project_info.tasks
 
     # ODK Credentials
@@ -232,9 +229,9 @@ def download_submission_for_task(task_id):
             f"Thread {threading.current_thread().name} - "
             f"Downloading submission for Task ID {task_id}"
         )
-        xform_name = f"{project_name}_{xform_title}_{task_id}"
+        xform_name = f"{project_name}_form_{task_id}"
         file = xform.getSubmissionMedia(odkid, xform_name)
-        file_path = f"{project_name}_{xform_title}_submission_{task_id}.zip"
+        file_path = f"{project_name}_submission_{task_id}.zip"
         with open(file_path, "wb") as f:
             f.write(file.content)
         return file_path
@@ -295,7 +292,7 @@ def extract_files(zip_file_path):
                 )
 
     # Create a new ZIP file for the extracted files
-    final_zip_file_path = f"{project_name}_{xform_title}_submissions_final.zip"
+    final_zip_file_path = f"{project_name}_submissions_final.zip"
     with zipfile.ZipFile(final_zip_file_path, mode="w") as final_zip_file:
         for file_path in extracted_files:
             final_zip_file.write(file_path)
@@ -429,8 +426,7 @@ def get_all_submissions_json(db: Session, project_id):
     get_task_id_list_sync = async_to_sync(tasks_crud.get_task_id_list)
     task_list = get_task_id_list_sync(db, project_id)
     xform_list = [
-        f"{project_info.project_name_prefix}_{project_info.xform_title}_{task}"
-        for task in task_list
+        f"{project_info.project_name_prefix}_task_{task}" for task in task_list
     ]
 
     submissions = project.getAllSubmissions(project_info.odkid, xform_list)
@@ -487,7 +483,6 @@ async def download_submission(
 
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
-    xform_title = project_info.xform_title
     project_tasks = project_info.tasks
 
     # ODK Credentials
@@ -505,19 +500,17 @@ async def download_submission(
             task_list = [x.id for x in project_tasks]
 
             # # Create a new ZIP file for all submissions
-            # zip_file_path = f"{project_name}_{xform_title}_submissions.zip"
+            # zip_file_path = f"{project_name}_submissions.zip"
             files = []
 
             for task_list_id in task_list:
-                xform_name = f"{project_name}_{xform_title}_{task_list_id}"
+                xform_name = f"{project_name}_task_{task_list_id}"
 
                 # XML Form Id is a combination or project_name, category and task_id
                 file = xform.getSubmissionMedia(odkid, xform_name)
 
                 # Create a new output file for each submission
-                file_path = (
-                    f"{project_name}_{xform_title}_submission_{task_list_id}.zip"
-                )
+                file_path = f"{project_name}_submission_{task_list_id}.zip"
                 with open(file_path, "wb") as f:
                     f.write(file.content)
 
@@ -536,14 +529,14 @@ async def download_submission(
                     ]  # Add the extracted file paths to the list of extracted files
 
             # Create a new ZIP file for the extracted files
-            final_zip_file_path = f"{project_name}_{xform_title}_submissions_final.zip"
+            final_zip_file_path = f"{project_name}_submissions_final.zip"
             with zipfile.ZipFile(final_zip_file_path, mode="w") as final_zip_file:
                 for file_path in extracted_files:
                     final_zip_file.write(file_path)
 
             return FileResponse(final_zip_file_path)
         else:
-            xform_name = f"{project_name}_{xform_title}_{task_id}"
+            xform_name = f"{project_name}_task_{task_id}"
             file = xform.getSubmissionMedia(odkid, xform_name)
             with open(file_path, "wb") as f:
                 f.write(file.content)
@@ -559,7 +552,7 @@ async def download_submission(
         if task_id is None:
             task_list = [x.id for x in project_tasks]
             for task_list_id in task_list:
-                xform_name = f"{project_name}_{xform_title}_{task_list_id}"
+                xform_name = f"{project_name}_task_{task_list_id}"
                 file = xform.getSubmissions(odkid, xform_name, None, False, True)
                 if not file:
                     json_data = None
@@ -569,7 +562,7 @@ async def download_submission(
                     if json_data_value:
                         files.extend(json_data_value)
         else:
-            xform_name = f"{project_name}_{xform_title}_{task_id}"
+            xform_name = f"{project_name}_task_{task_id}"
             file = xform.getSubmissions(odkid, xform_name, None, False, True)
             json_data = json.loads(file)
 
@@ -594,14 +587,13 @@ async def get_submission_points(db: Session, project_id: int, task_id: int = Non
 
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
-    xform_title = project_info.xform_title
 
     # ODK Credentials
     odk_credentials = await project_deps.get_odk_credentials(db, project_id)
     xform = get_odk_form(odk_credentials)
 
     if task_id:
-        xform_name = f"{project_name}_{xform_title}_{task_id}"
+        xform_name = f"{project_name}_task_{task_id}"
         # file_path = f"{project_id}_submissions.zip"
         response_file = xform.getSubmissionMedia(odkid, xform_name)
 
@@ -647,7 +639,6 @@ async def get_submission_count_of_a_project(db: Session, project_id: int):
 
     odkid = project_info.odkid
     project_name = project_info.project_name_prefix
-    xform_title = project_info.xform_title
     project_tasks = project_info.tasks
 
     # ODK Credentials
@@ -659,7 +650,7 @@ async def get_submission_count_of_a_project(db: Session, project_id: int):
 
     task_list = [x.id for x in project_tasks]
     for task_id in task_list:
-        xform_name = f"{project_name}_{xform_title}_{task_id}"
+        xform_name = f"{project_name}_task_{task_id}"
         file = xform.getSubmissions(odkid, xform_name, None, False, True)
         if not file:
             json_data = None
@@ -823,7 +814,7 @@ async def get_submission_by_task(
     odk_credentials = await project_deps.get_odk_credentials(db, project.id)
 
     xform = get_odk_form(odk_credentials)
-    xform_name = f"{project.project_name_prefix}_{project.xform_title}_{task_id}"
+    xform_name = f"{project.project_name_prefix}_task_{task_id}"
     data = xform.listSubmissions(project.odkid, xform_name, filters)
     submissions = data.get("value", [])
     count = data.get("@odata.count", 0)