diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index f5ceccdc4..4fb882fcd 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -250,6 +250,14 @@ async def refresh_token( request: Request, user_data: AuthUser = Depends(login_required) ): """Uses the refresh token to generate a new access token.""" + if settings.DEBUG: + return JSONResponse( + status_code=HTTPStatus.OK, + content={ + "token": "debugtoken", + **user_data.model_dump(), + }, + ) try: refresh_token = extract_refresh_token_from_cookie(request) if not refresh_token: diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 1428644ae..089650983 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -24,7 +24,6 @@ from typing import Optional, Union import geojson -from defusedxml import ElementTree from fastapi import HTTPException from loguru import logger as log from osm_fieldwork.OdkCentral import OdkAppUser, OdkForm, OdkProject @@ -190,7 +189,7 @@ def create_odk_xform( odk_id: int, xform_data: BytesIO, odk_credentials: project_schemas.ODKCentralDecrypted, -) -> str: +) -> None: """Create an XForm on a remote ODK Central server. Args: @@ -198,8 +197,7 @@ def create_odk_xform( xform_data (BytesIO): XForm data to set. odk_credentials (ODKCentralDecrypted): Creds for ODK Central. - Returns: - form_name (str): ODK Central form name for the API. + Returns: None """ try: xform = get_odk_form(odk_credentials) @@ -209,25 +207,7 @@ def create_odk_xform( status_code=500, detail={"message": "Connection failed to odk central"} ) from e - xform_id = xform.createForm(odk_id, xform_data, publish=True) - if not xform_id: - namespaces = { - "h": "http://www.w3.org/1999/xhtml", - "odk": "http://www.opendatakit.org/xforms", - "xforms": "http://www.w3.org/2002/xforms", - } - # Get the form id from the XML - root = ElementTree.fromstring(xform_data.getvalue()) - xml_data = root.findall(".//xforms:data[@id]", namespaces) - extracted_name = "Not Found" - for dt in xml_data: - extracted_name = dt.get("id") - msg = f"Failed to create form on ODK Central: ({extracted_name})" - log.error(msg) - raise HTTPException( - status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg - ) from None - return xform_id + xform.createForm(odk_id, xform_data, publish=True) def delete_odk_xform( @@ -323,7 +303,10 @@ async def read_and_test_xform(input_data: BytesIO) -> None: BytesIO: the converted XML representation of the XForm. """ try: - log.debug("Parsing XLSForm --> XML data") + log.debug( + f"Parsing XLSForm --> XML data: input type {type(input_data)} | " + f"data length {input_data.getbuffer().nbytes}" + ) # NOTE pyxform.xls2xform.convert returns a ConvertResult object return BytesIO(xform_convert(input_data).xform.encode("utf-8")) except Exception as e: @@ -340,7 +323,7 @@ async def append_fields_to_user_xlsform( additional_entities: list[str] = None, task_count: int = None, existing_id: str = None, -) -> BytesIO: +) -> tuple[str, BytesIO]: """Helper to return the intermediate XLSForm prior to convert.""" log.debug("Appending mandatory FMTM fields to XLSForm") return await append_mandatory_fields( @@ -360,7 +343,7 @@ async def validate_and_update_user_xlsform( existing_id: str = None, ) -> BytesIO: """Wrapper to append mandatory fields and validate user uploaded XLSForm.""" - updated_file_bytes = await append_fields_to_user_xlsform( + xform_id, updated_file_bytes = await append_fields_to_user_xlsform( xlsform, form_category=form_category, additional_entities=additional_entities, @@ -899,12 +882,10 @@ async def get_appuser_token( xform_id: str, project_odk_id: int, odk_credentials: project_schemas.ODKCentralDecrypted, - db: Session, ): """Get the app user token for a specific project. Args: - db: The database session to use. odk_credentials: ODK credentials for the project. project_odk_id: The ODK ID of the project. xform_id: The ID of the XForm. diff --git a/src/backend/app/central/central_routes.py b/src/backend/app/central/central_routes.py index 9dc14c692..1a425bd95 100644 --- a/src/backend/app/central/central_routes.py +++ b/src/backend/app/central/central_routes.py @@ -79,9 +79,8 @@ async def refresh_appuser_token( try: odk_credentials = await project_deps.get_odk_credentials(db, project_id) project_odk_id = project.odkid - db_xform = await project_deps.get_project_xform(db, project_id) odk_token = await central_crud.get_appuser_token( - db_xform.odk_form_id, project_odk_id, odk_credentials, db + project.odk_form_id, project_odk_id, odk_credentials, db ) project.odk_token = odk_token db.commit() diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index ad8567994..ceabf66ad 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -210,23 +210,6 @@ class DbXLSForm(Base): xls = cast(bytes, Column(LargeBinary)) -class DbXForm(Base): - """XForms linked per project. - - TODO eventually we will support multiple forms per project. - TODO So the category field a stub until then. - TODO currently it's maintained under projects.xform_category. - """ - - __tablename__ = "xforms" - id = cast(int, Column(Integer, primary_key=True, autoincrement=True)) - project_id = cast( - int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True) - ) - odk_form_id = cast(str, Column(String)) - category = cast(str, Column(String)) - - class DbTaskHistory(Base): """Describes the history associated with a task.""" @@ -453,10 +436,8 @@ def tasks_bad(self): # XForm category specified xform_category = cast(str, Column(String)) - # Linked XForms - forms = relationship( - DbXForm, backref="project_xform_link", cascade="all, delete, delete-orphan" - ) + odk_form_id = cast(str, Column(String)) + xlsform_content = cast(bytes, Column(LargeBinary)) __table_args__ = ( Index("idx_geometry", outline, postgresql_using="gist"), @@ -486,13 +467,6 @@ def tasks_bad(self): odk_central_password = cast(str, Column(String)) odk_token = cast(str, Column(String, nullable=True)) - form_xls = cast( - bytes, Column(LargeBinary) - ) # XLSForm file if custom xls is uploaded - form_config_file = cast( - bytes, Column(LargeBinary) - ) # Yaml config file if custom xls is uploaded - data_extract_type = cast( str, Column(String) ) # Type of data extract (Polygon or Centroid) @@ -559,7 +533,11 @@ class DbSubmissionPhotos(Base): __tablename__ = "submission_photos" id = cast(int, Column(Integer, primary_key=True)) - project_id = cast(int, Column(Integer)) - task_id = cast(int, Column(Integer)) + project_id = cast( + int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True) + ) + task_id = cast( + int, Column(Integer, ForeignKey("tasks.id"), name="task_id", index=True) + ) submission_id = cast(str, Column(String)) s3_path = cast(str, Column(String)) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 528bce3bb..a55c4b32e 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -834,15 +834,13 @@ def flatten_dict(d, parent_key="", sep="_"): async def generate_odk_central_project_content( - project: db_models.DbProject, + project_odk_id: int, + project_odk_form_id: str, odk_credentials: project_schemas.ODKCentralDecrypted, xlsform: BytesIO, task_extract_dict: dict[int, geojson.FeatureCollection], - db: Session, ) -> str: """Populate the project in ODK Central with XForm, Appuser, Permissions.""" - project_odk_id = project.odkid - # The ODK Dataset (Entity List) must exist prior to main XLSForm entities_list = await central_crud.task_geojson_dict_to_entity_values( task_extract_dict @@ -861,33 +859,16 @@ async def generate_odk_central_project_content( # Upload survey XForm log.info("Uploading survey XForm to ODK Central") - xform_id = central_crud.create_odk_xform( + central_crud.create_odk_xform( project_odk_id, xform, odk_credentials, ) - sql = text( - """ - INSERT INTO xforms ( - project_id, odk_form_id, category - ) - VALUES ( - :project_id, :xform_id, :category - ) - """ - ) - db.execute( - sql, - { - "project_id": project.id, - "xform_id": xform_id, - "category": project.xform_category, - }, - ) - db.commit() return await central_crud.get_appuser_token( - xform_id, project_odk_id, odk_credentials, db + project_odk_form_id, + project_odk_id, + odk_credentials, ) @@ -929,13 +910,15 @@ async def generate_project_files( # Get ODK Project ID project_odk_id = project.odkid + project_xlsform = project.xlsform_content + project_odk_form_id = project.odk_form_id encrypted_odk_token = await generate_odk_central_project_content( - project, + project_odk_id, + project_odk_form_id, odk_credentials, - BytesIO(project.form_xls), + BytesIO(project_xlsform), task_extract_dict, - db, ) log.debug( f"Setting odk token for FMTM project ({project_id}) " @@ -1488,9 +1471,8 @@ async def get_dashboard_detail( """Get project details for project dashboard.""" odk_central = await project_deps.get_odk_credentials(db, project.id) xform = central_crud.get_odk_form(odk_central) - db_xform = await project_deps.get_project_xform(db, project.id) - submission_meta_data = xform.getFullDetails(project.odkid, db_xform.odk_form_id) + submission_meta_data = xform.getFullDetails(project.odkid, project.odk_form_id) project.total_submission = submission_meta_data.get("submissions", 0) project.last_active = submission_meta_data.get("lastSubmission") diff --git a/src/backend/app/projects/project_deps.py b/src/backend/app/projects/project_deps.py index 8b62d0cb1..93cb09b7a 100644 --- a/src/backend/app/projects/project_deps.py +++ b/src/backend/app/projects/project_deps.py @@ -98,28 +98,3 @@ async def get_odk_credentials(db: Session, project_id: int): odk_central_user=user, odk_central_password=password, ) - - -async def get_project_xform(db, project_id): - """Retrieve the transformation associated with a specific project. - - Args: - db: Database connection object. - project_id: The ID of the project to retrieve the transformation for. - - Returns: - The transformation record associated with the specified project. - - Raises: - None - """ - sql = text( - """ - SELECT * FROM xforms - WHERE project_id = :project_id; - """ - ) - - result = db.execute(sql, {"project_id": project_id}) - db_xform = result.first() - return db_xform diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 81fbb66bc..a16dcf86f 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -666,10 +666,14 @@ async def validate_form( ): """Basic validity check for uploaded XLSForm. - Does not append all addition values to make this a valid FMTM form for mapping. + Parses the form using ODK pyxform to check that it is valid. + + If the `debug` param is used, the form is returned for inspection. + NOTE that this debug form has additional fields appended and should + not be used for FMTM project creation. """ if debug: - updated_form = await central_crud.append_fields_to_user_xlsform( + xform_id, updated_form = await central_crud.append_fields_to_user_xlsform( xlsform, task_count=1, # NOTE this must be included to append task_filter choices ) @@ -678,14 +682,17 @@ async def validate_form( media_type=( "application/vnd.openxmlformats-" "officedocument.spreadsheetml.sheet" ), - headers={"Content-Disposition": "attachment; filename=updated_form.xlsx"}, + headers={"Content-Disposition": f"attachment; filename={xform_id}.xlsx"}, ) else: await central_crud.validate_and_update_user_xlsform( xlsform, task_count=1, # NOTE this must be included to append task_filter choices ) - return Response(status_code=HTTPStatus.OK) + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "Your form is valid"}, + ) @router.post("/{project_id}/generate-project-data") @@ -752,14 +759,21 @@ async def generate_files( with open(xlsform_path, "rb") as f: xlsform = BytesIO(f.read()) - project_xlsform = await central_crud.append_fields_to_user_xlsform( + xform_id, project_xlsform = await central_crud.append_fields_to_user_xlsform( xlsform=xlsform, form_category=form_category, task_count=task_count, additional_entities=additional_entities, ) # Write XLS form content to db - project.form_xls = project_xlsform.getvalue() + xlsform_bytes = project_xlsform.getvalue() + if not xlsform_bytes: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="There was an error with the XLSForm!", + ) + project.odk_form_id = xform_id + project.xlsform_content = xlsform_bytes db.commit() # Create task in db and return uuid @@ -979,7 +993,7 @@ async def download_form( "Content-Disposition": f"attachment; filename={project.id}_xlsform.xlsx", "Content-Type": "application/media", } - return Response(content=project.form_xls, headers=headers) + return Response(content=project.xlsform_content, headers=headers) @router.post("/update-form") @@ -1019,7 +1033,7 @@ async def update_project_form( ) # Commit changes to db - project.form_xls = xlsform.getvalue() + project.xlsform_content = xlsform.getvalue() db.commit() return project diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 670e5574f..b8628a76c 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -317,7 +317,7 @@ class ProjectBase(BaseModel): """Base project model.""" outline: Any = Field(exclude=True) - forms: Any = Field(exclude=True) + odk_form_id: Optional[str] = Field(exclude=True) id: int odkid: int @@ -360,10 +360,13 @@ def organisation_logo(self) -> Optional[str]: @computed_field @property def xform_id(self) -> Optional[str]: - """Compute the XForm ID from the linked DbXForm.""" - if not self.forms: + """Generate from odk_form_id. + + TODO this could be refactored out in future. + """ + if not self.odk_form_id: return None - return self.forms[0].odk_form_id + return self.odk_form_id class ProjectWithTasks(ProjectBase): diff --git a/src/backend/app/submissions/submission_crud.py b/src/backend/app/submissions/submission_crud.py index 1f681ab7d..9032ca742 100644 --- a/src/backend/app/submissions/submission_crud.py +++ b/src/backend/app/submissions/submission_crud.py @@ -38,7 +38,6 @@ from app.central.central_crud import ( get_odk_form, - get_odk_project, list_odk_xforms, ) from app.config import settings @@ -46,7 +45,6 @@ from app.models.enums import HTTPStatus from app.projects import project_crud, project_deps from app.s3 import add_obj_to_bucket, get_obj_from_bucket -from app.tasks import tasks_crud # async def convert_json_to_osm(file_path): # """Wrapper for osm-fieldwork json2osm.""" @@ -144,8 +142,7 @@ async def gather_all_submission_csvs(db: Session, project: db_models.DbProject): odk_credentials = await project_deps.get_odk_credentials(db, project.id) xform = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - file = xform.getSubmissionMedia(odkid, db_xform.odk_form_id) + file = xform.getSubmissionMedia(odkid, project.odk_form_id) return file.content @@ -290,28 +287,6 @@ def update_submission_in_s3( update_bg_task_sync(db, background_task_id, 2, str(e)) # 2 is FAILED -def get_all_submissions_json(db: Session, project_id): - """Get all submissions for a project in JSON format.""" - get_project_sync = async_to_sync(project_crud.get_project) - project_info = get_project_sync(db, project_id) - - # ODK Credentials - odk_sync = async_to_sync(project_deps.get_odk_credentials) - odk_credentials = odk_sync(db, project_id) - project = get_odk_project(odk_credentials) - - get_task_id_list_sync = async_to_sync(tasks_crud.get_task_id_list) - task_list = get_task_id_list_sync(db, project_id) - - # FIXME use db_xform - xform_list = [ - f"{project_info.project_name_prefix}_task_{task}" for task in task_list - ] - # FIXME use separate func - submissions = project.getAllSubmissions(project_info.odkid, xform_list) - return submissions - - async def download_submission_in_json(db: Session, project: db_models.DbProject): """Download submission data from ODK Central.""" project_name = project.project_name_prefix @@ -329,7 +304,10 @@ async def download_submission_in_json(db: Session, project: db_models.DbProject) async def get_submission_points(db: Session, project_id: int, task_id: Optional[int]): - """Get submission points for a project.""" + """Get submission points for a project. + + FIXME refactor to pass through project object via auth. + """ project_info = await project_crud.get_project_by_id(db, project_id) if not project_info: raise HTTPException(status_code=404, detail="Project not found") @@ -337,9 +315,8 @@ async def get_submission_points(db: Session, project_id: int, task_id: Optional[ odk_id = project_info.odkid odk_credentials = await project_deps.get_odk_credentials(db, project_id) xform = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project_id) - response_file = xform.getSubmissionMedia(odk_id, db_xform.odk_form_id) + response_file = xform.getSubmissionMedia(odk_id, project_info.odk_form_id) response_file_bytes = response_file.content try: @@ -382,8 +359,7 @@ async def get_submission_count_of_a_project(db: Session, project: db_models.DbPr # Get ODK Form with odk credentials from the project. xform = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - data = xform.listSubmissions(project.odkid, db_xform.odk_form_id, {}) + data = xform.listSubmissions(project.odkid, project.odk_form_id, {}) return len(data["value"]) @@ -466,40 +442,10 @@ async def get_submission_by_project( ValueError: If the submission file cannot be found. """ - db_xform = await project_deps.get_project_xform(db, project.id) odk_central = await project_deps.get_odk_credentials(db, project.id) xform = get_odk_form(odk_central) - return xform.listSubmissions(project.odkid, db_xform.odk_form_id, filters) - - -# FIXME this is not needed now it can be directly filtered from submission table -# async def get_submission_by_task( -# project: db_models.DbProject, -# task_id: int, -# filters: dict, -# db: Session, -# ): -# """Get submissions and count by task. - -# Args: -# project: The project instance. -# task_id: The ID of the task. -# filters: A dictionary of filters. -# db: The database session. - -# Returns: -# Tuple: A tuple containing the list of submissions and the count. -# """ -# odk_credentials = await project_deps.get_odk_credentials(db, project.id) - -# xform = get_odk_form(odk_credentials) -# db_xform = await project_deps.get_project_xform(db, project.id) -# data = xform.listSubmissions(project.odkid, db_xform.odk_form_id, filters) -# submissions = data.get("value", []) -# count = data.get("@odata.count", 0) - -# return submissions, count + return xform.listSubmissions(project.odkid, project.odk_form_id, filters) async def get_submission_detail( @@ -519,9 +465,8 @@ async def get_submission_detail( """ odk_credentials = await project_deps.get_odk_credentials(db, project.id) odk_form = get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) submission = json.loads( - odk_form.getSubmissions(project.odkid, db_xform.odk_form_id, submission_id) + odk_form.getSubmissions(project.odkid, project.odk_form_id, submission_id) ) return submission.get("value", [])[0] @@ -587,7 +532,6 @@ async def upload_attachment_to_s3( """ try: project = await project_deps.get_project_by_id(db, project_id) - db_xform = await project_deps.get_project_xform(db, project_id) odk_central = await project_deps.get_odk_credentials(db, project_id) xform = get_odk_form(odk_central) s3_bucket = settings.S3_BUCKET_NAME @@ -631,7 +575,7 @@ async def upload_attachment_to_s3( attachment = xform.getSubmissionPhoto( project.odkid, str(instance_id), - db_xform.odk_form_id, + project.odk_form_id, str(filename), ) if attachment: diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index a40d917ef..460fb0450 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -329,9 +329,7 @@ async def get_submission_form_fields( project = project_user.get("project") odk_credentials = await project_deps.get_odk_credentials(db, project.id) odk_form = central_crud.get_odk_form(odk_credentials) - db_xform = await project_deps.get_project_xform(db, project.id) - - return odk_form.formFields(project.odkid, db_xform.odk_form_id) + return odk_form.formFields(project.odkid, project.odk_form_id) @router.get("/submission_table") @@ -440,11 +438,10 @@ async def update_review_state( project = current_user.get("project") odk_creds = await project_deps.get_odk_credentials(db, project.id) odk_project = central_crud.get_odk_project(odk_creds) - db_xform = await project_deps.get_project_xform(db, project.id) response = odk_project.updateReviewState( project.odkid, - db_xform.odk_form_id, + project.odk_form_id, instance_id, {"reviewState": review_state}, ) diff --git a/src/backend/app/tasks/task_deps.py b/src/backend/app/tasks/task_deps.py index 4ef2d8ab3..7df1dc735 100644 --- a/src/backend/app/tasks/task_deps.py +++ b/src/backend/app/tasks/task_deps.py @@ -18,41 +18,15 @@ """Task dependencies for use in Depends.""" -from typing import Union - from fastapi import Depends from fastapi.exceptions import HTTPException from sqlalchemy.orm import Session from app.db.database import get_db -from app.db.db_models import DbProject, DbTask +from app.db.db_models import DbTask from app.models.enums import HTTPStatus -async def get_xform_name( - project: Union[int, DbProject], - task_id: int, - db: Session = Depends(get_db), -) -> str: - """Get a project xform name.""" - if isinstance(project, int): - db_project = db.query(DbProject).filter(DbProject.id == project).first() - if not db_project: - raise HTTPException( - status_code=HTTPStatus.NOT_FOUND, - detail=f"Project with ID ({project}) does not exist", - ) - else: - db_project = project - - project_name = db_project.project_name_prefix - # TODO in the future we may possibly support multiple forms per project. - # TODO to facilitate this we need to add the _{category} suffix and track. - # TODO this in the new xforms.category field/table. - form_name = project_name - return form_name - - async def get_task_by_id( project_id: int, task_id: int, diff --git a/src/backend/migrations/007-remove-xform-table.sql b/src/backend/migrations/007-remove-xform-table.sql new file mode 100644 index 000000000..5cd75299b --- /dev/null +++ b/src/backend/migrations/007-remove-xform-table.sql @@ -0,0 +1,74 @@ +-- ## Migration to: +-- * Rename projects.form_xls --> projects.xlsform_content +-- * Remove projects.form_config_file until required +-- * Add missed foreign keys to submission_photos +-- * Remove public.xforms table, moving odk_form_id to public.projects +-- Decided to remove the XForms table as projects likely always have a +-- 1:1 relationship with xforms (spwoodcock) + +-- Start a transaction +BEGIN; + +-- Add foreign keys to submission_photos table, if they don't exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'fk_project_id' + AND table_name = 'submission_photos' + ) THEN + ALTER TABLE ONLY public.submission_photos + ADD CONSTRAINT fk_project_id FOREIGN KEY (project_id) + REFERENCES public.projects (id); + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM information_schema.table_constraints + WHERE constraint_name = 'fk_tasks' + AND table_name = 'submission_photos' + ) THEN + ALTER TABLE ONLY public.submission_photos + ADD CONSTRAINT fk_tasks FOREIGN KEY ( + task_id, project_id + ) REFERENCES public.tasks (id, project_id); + END IF; +END $$; + +-- Update public.projects table +ALTER TABLE public.projects +-- Remove form_config_file column +DROP COLUMN IF EXISTS form_config_file, +-- Add odk_form_id if not exists +ADD COLUMN IF NOT EXISTS odk_form_id VARCHAR; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'projects' AND column_name = 'form_xls') THEN + ALTER TABLE public.projects + RENAME COLUMN form_xls TO xlsform_content; -- Rename form_xls to xlsform_content + END IF; +END $$; + +-- Migrate odk_form_id data from xforms to projects +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'xforms' + AND column_name = 'odk_form_id' + ) THEN + -- Perform data migration if odk_form_id exists in xforms + UPDATE public.projects p + SET odk_form_id = x.odk_form_id + FROM public.xforms x + WHERE x.project_id = p.id + AND p.odk_form_id IS NULL; -- Avoid overwriting existing values + END IF; +END $$; + +-- Drop public.xforms table if it exists +DROP TABLE IF EXISTS public.xforms; +DROP SEQUENCE IF EXISTS public.xforms_id_seq; + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 38416af3d..c556d1f5b 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -260,6 +260,8 @@ CREATE TABLE public.projects ( status public.projectstatus NOT NULL DEFAULT 'DRAFT', total_tasks integer, xform_category character varying, + xlsform_content bytea, + odk_form_id character varying, visibility public.projectvisibility NOT NULL DEFAULT 'PUBLIC', mapper_level public.mappinglevel NOT NULL DEFAULT 'INTERMEDIATE', priority public.projectpriority DEFAULT 'MEDIUM', @@ -278,8 +280,6 @@ CREATE TABLE public.projects ( odk_central_user character varying, odk_central_password character varying, odk_token character varying, - form_xls bytea, - form_config_file bytea, data_extract_type character varying, data_extract_url character varying, task_split_type public.tasksplittype, @@ -390,22 +390,24 @@ CACHE 1; ALTER TABLE public.xlsforms_id_seq OWNER TO fmtm; ALTER SEQUENCE public.xlsforms_id_seq OWNED BY public.xlsforms.id; -CREATE TABLE public.xforms ( +CREATE TABLE public.submission_photos ( id integer NOT NULL, - project_id integer, - odk_form_id character varying, - category character varying + project_id integer NOT NULL, + task_id integer NOT NULL, + submission_id character varying NOT NULL, + s3_path character varying NOT NULL ); -ALTER TABLE public.xforms OWNER TO fmtm; -CREATE SEQUENCE public.xforms_id_seq +ALTER TABLE public.submission_photos OWNER TO fmtm; +CREATE SEQUENCE public.submission_photos_id_seq AS integer START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER TABLE public.xforms_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.xforms_id_seq OWNED BY public.xforms.id; +ALTER TABLE public.submission_photos_id_seq OWNER TO fmtm; +ALTER SEQUENCE public.submission_photos_id_seq +OWNED BY public.submission_photos.id; -- nextval for primary keys (autoincrement) @@ -427,8 +429,8 @@ ALTER TABLE ONLY public.tasks ALTER COLUMN id SET DEFAULT nextval( ALTER TABLE ONLY public.xlsforms ALTER COLUMN id SET DEFAULT nextval( 'public.xlsforms_id_seq'::regclass ); -ALTER TABLE ONLY public.xforms ALTER COLUMN id SET DEFAULT nextval( - 'public.xforms_id_seq'::regclass +ALTER TABLE ONLY public.submission_photos ALTER COLUMN id SET DEFAULT nextval( + 'public.submission_photos_id_seq'::regclass ); @@ -482,8 +484,8 @@ ADD CONSTRAINT xlsforms_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.xlsforms ADD CONSTRAINT xlsforms_title_key UNIQUE (title); -ALTER TABLE ONLY public.xforms -ADD CONSTRAINT xforms_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.submission_photos +ADD CONSTRAINT submission_photos_pkey PRIMARY KEY (id); -- Indexing @@ -593,11 +595,15 @@ ADD CONSTRAINT user_roles_user_id_fkey FOREIGN KEY ( user_id ) REFERENCES public.users (id); -ALTER TABLE ONLY public.xforms +ALTER TABLE ONLY public.submission_photos ADD CONSTRAINT fk_project_id FOREIGN KEY ( project_id ) REFERENCES public.projects (id); +ALTER TABLE ONLY public.submission_photos +ADD CONSTRAINT fk_tasks FOREIGN KEY ( + task_id, project_id +) REFERENCES public.tasks (id, project_id); -- Finalise diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 3ec765848..2215fb9fe 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test", "monitoring"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:8c65c80f5e570cc8e16aad773965a60059d14e05b51e55dd334de4b4082329bb" +content_hash = "sha256:d3b1cf0233422641fe8c1e49bb518a57c17fccc8c4d74f2b35cfa55eebbeca7d" [[package]] name = "aiohttp" @@ -1579,7 +1579,7 @@ files = [ [[package]] name = "osm-fieldwork" -version = "0.16.4" +version = "0.16.5rc0" requires_python = ">=3.10" summary = "Processing field data from ODK to OpenStreetMap format." dependencies = [ @@ -1605,8 +1605,8 @@ dependencies = [ "xmltodict>=0.13.0", ] files = [ - {file = "osm-fieldwork-0.16.4.tar.gz", hash = "sha256:0285313d3e4bd99df0cccd91b8706b6d3f66ae427bab259250df19b07c51d31b"}, - {file = "osm_fieldwork-0.16.4-py3-none-any.whl", hash = "sha256:595afcf05a0a3fda035e5c2c342b5a5c1bcfa2e21002098f6c670c7e502baf93"}, + {file = "osm-fieldwork-0.16.5rc0.tar.gz", hash = "sha256:34efa14be5bfb111f8227809867d8c73bbdf3893e472d5c239643a727fe1c769"}, + {file = "osm_fieldwork-0.16.5rc0-py3-none-any.whl", hash = "sha256:a879a8b0dce9273d7c72d03ec6d704152305ccc8be72bc9404b95877737eec67"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 1829caf20..f0dd0d334 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -42,10 +42,9 @@ dependencies = [ "asgiref==3.8.1", "sozipfile==0.3.2", "cryptography>=42.0.8", - "defusedxml>=0.7.1", "pyjwt>=2.8.0", "async-lru>=2.0.4", - "osm-fieldwork>=0.16.4", + "osm-fieldwork>=0.16.5rc0", "osm-login-python==2.0.0", "osm-rawdata==0.3.2", "fmtm-splitter==1.3.1",