From bd1276b14c3e0a1280fb56a1f1c71e8909d44bda Mon Sep 17 00:00:00 2001 From: Sujan Adhikari <109404840+Sujanadh@users.noreply.github.com> Date: Sun, 22 Sep 2024 05:04:24 +0545 Subject: [PATCH 1/3] feat(backend): endpoint to retreive submission photos (#1794) * feat: created an endpoint to return submission photo * refactor: proper docstring with return type * return image url in list for the possibility of multiple images * refactor: small refactor to use path params --------- Co-authored-by: spwoodcock --- .../app/submissions/submission_routes.py | 117 ++++++++---------- src/frontend/src/views/SubmissionDetails.tsx | 6 +- 2 files changed, 50 insertions(+), 73 deletions(-) diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index 50b326199..a40d917ef 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -27,6 +27,7 @@ from fastapi.concurrency import run_in_threadpool from fastapi.responses import FileResponse, JSONResponse, Response from loguru import logger as log +from sqlalchemy import text from sqlalchemy.orm import Session from app.auth.auth_schemas import AuthUser, ProjectUserDict @@ -413,74 +414,7 @@ async def submission_table( return response -# FIXME remove it since separate endpoint is not required now. -# @router.get("/task_submissions/{project_id}") -# async def task_submissions( -# task_id: int, -# project: db_models.DbProject = Depends(project_deps.get_project_by_id), -# page: int = Query(1, ge=1), -# limit: int = Query(13, le=100), -# submission_id: Optional[str] = None, -# submitted_by: Optional[str] = None, -# review_state: Optional[str] = None, -# submitted_date: Optional[str] = Query( -# None, title="Submitted Date", description="Date in format (e.g., 'YYYY-MM-DD')" -# ), -# db: Session = Depends(database.get_db), -# current_user: AuthUser = Depends(mapper), -# ): -# """This api returns the submission table of a project. - -# It takes two parameter: project_id and task_id. - -# project_id: The ID of the project. - -# task_id: The ID of the task. -# """ -# skip = (page - 1) * limit -# filters = { -# "$top": limit, -# "$skip": skip, -# "$count": True, -# "$wkt": True, -# } - -# if submitted_date: -# filters["$filter"] = ( -# "__system/submissionDate ge {}T00:00:00+00:00 " -# "and __system/submissionDate le {}T23:59:59.999+00:00" -# ).format(submitted_date, submitted_date) - -# if submitted_by: -# if "$filter" in filters: -# filters["$filter"] += f"and (username eq '{submitted_by}')" -# else: -# filters["$filter"] = f"username eq '{submitted_by}'" - -# if review_state: -# if "$filter" in filters: -# filters["$filter"] += f" and (__system/reviewState eq '{review_state}')" -# else: -# filters["$filter"] = f"__system/reviewState eq '{review_state}'" - -# data, count = await submission_crud.get_submission_by_task( -# project, task_id, filters, db -# ) -# pagination = await project_crud.get_pagination(page, count, limit, count) -# response = submission_schemas.PaginatedSubmissions( -# results=data, -# pagination=submission_schemas.PaginationInfo(**pagination.model_dump()), -# ) -# if submission_id: -# submission_detail = await submission_crud.get_submission_detail( -# project, task_id, submission_id, db -# ) -# response = submission_detail.get("value", [])[0] - -# return response - - -@router.get("/submission-detail") +@router.get("/{submission_id}") async def submission_detail( submission_id: str, db: Session = Depends(database.get_db), @@ -594,3 +528,50 @@ async def conflate_geojson( raise HTTPException( status_code=500, detail=f"Failed to process conflation: {str(e)}" ) from e + + +@router.get("/{submission_id}/photos") +async def submission_photo( + submission_id: str, + db: Session = Depends(database.get_db), +) -> dict: + """Get submission photos. + + Retrieves the S3 paths of the submission photos for the given submission ID. + + Args: + submission_id (str): The ID of the submission. + db (Session): The database session. + + Returns: + dict: A dictionary containing the S3 path of the submission photo. + If no photo is found, + the dictionary will contain a None value for the S3 path. + + Raises: + HTTPException: If an error occurs while retrieving the submission photo. + """ + try: + sql = text(""" + SELECT + s3_path + FROM + submission_photos + WHERE + submission_id = :submission_id; + """) + results = db.execute(sql, {"submission_id": submission_id}).fetchall() + + # Extract the s3_path from each result and return as a list + s3_paths = [result.s3_path for result in results] if results else [] + + return {"image_urls": s3_paths} + + except Exception as e: + log.warning( + f"Failed to get submission photos for submission {submission_id}: {e}" + ) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to get submission photos", + ) from e diff --git a/src/frontend/src/views/SubmissionDetails.tsx b/src/frontend/src/views/SubmissionDetails.tsx index 3eba42f0b..680ddb2aa 100644 --- a/src/frontend/src/views/SubmissionDetails.tsx +++ b/src/frontend/src/views/SubmissionDetails.tsx @@ -109,11 +109,7 @@ const SubmissionDetails = () => { useEffect(() => { dispatch( - SubmissionService( - `${ - import.meta.env.VITE_API_URL - }/submission/submission-detail?submission_id=${paramsInstanceId}&project_id=${projectId}`, - ), + SubmissionService(`${import.meta.env.VITE_API_URL}/submission/${paramsInstanceId}?project_id=${projectId}`), ); }, [projectId, paramsInstanceId]); From f6efdd6ea1149ded0870bf7256b7584a43ff3d81 Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Sun, 22 Sep 2024 00:20:37 +0100 Subject: [PATCH 2/3] feat(backend): endpoint to create additional Entity lists on a project (#1799) * fix(frontend): download xlsform with .xlsx format (updated from .xls) * fix(backend): fix javarosa conversion for all geom types (LineString) * refactor(backend): restructure code related to entity creation * feat(backend): add endpoint /additional-entity for a project * refactor(backend): rename features_geojson param --> geojson * docs: add note to /additional-entity about filename usage * fix(backend): correct error message if coordinates are invalid * test(backend): fix test for /projects/task-split endpoint * test(frontend): only select playwright test project to avoid errors * refactor: add traceback info to generate project files exceptions * fix(backend): geojson geom --> javarosa format for all types --- src/backend/app/central/central_crud.py | 78 ++++++++++++- src/backend/app/central/central_schemas.py | 45 ++++++- src/backend/app/db/postgis_utils.py | 123 +++++++++----------- src/backend/app/helpers/helper_routes.py | 4 +- src/backend/app/projects/project_crud.py | 50 ++++---- src/backend/app/projects/project_routes.py | 44 ++++++- src/backend/app/projects/project_schemas.py | 37 ------ src/backend/tests/test_projects_routes.py | 26 ++++- src/frontend/e2e/02-mapper-flow.spec.ts | 9 +- src/frontend/e2e/helpers.ts | 9 +- src/frontend/src/api/Project.ts | 2 +- 11 files changed, 281 insertions(+), 146 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index cfe1728a2..1428644ae 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -19,6 +19,7 @@ import csv import json +from asyncio import gather from io import BytesIO, StringIO from typing import Optional, Union @@ -32,7 +33,7 @@ from sqlalchemy import text from sqlalchemy.orm import Session -from app.central import central_deps +from app.central import central_deps, central_schemas from app.config import encrypt_value, settings from app.db.postgis_utils import ( geojson_to_javarosa_geom, @@ -519,6 +520,81 @@ async def convert_odk_submission_json_to_geojson( return geojson.FeatureCollection(features=all_features) +async def feature_geojson_to_entity_dict( + feature: geojson.Feature, +) -> central_schemas.EntityDict: + """Convert a single GeoJSON to an Entity dict for upload.""" + if not isinstance(feature, (dict, geojson.Feature)): + log.error(f"Feature not in correct format: {feature}") + raise ValueError(f"Feature not in correct format: {type(feature)}") + + feature_id = feature.get("id") + + geometry = feature.get("geometry", {}) + if not geometry: + msg = "'geometry' data field is mandatory" + log.debug(msg) + raise ValueError(msg) + + javarosa_geom = await geojson_to_javarosa_geom(geometry) + + # NOTE all properties MUST be string values for Entities, convert + properties = { + str(key): str(value) for key, value in feature.get("properties", {}).items() + } + # Set to TaskStatus enum READY value (0) + properties["status"] = "0" + + task_id = properties.get("task_id") + entity_label = f"Task {task_id} Feature {feature_id}" + + return {"label": entity_label, "data": {"geometry": javarosa_geom, **properties}} + + +async def task_geojson_dict_to_entity_values( + task_geojson_dict: dict[int, geojson.Feature], +) -> list[central_schemas.EntityDict]: + """Convert a dict of task GeoJSONs into data for ODK Entity upload.""" + log.debug("Converting dict of task GeoJSONs to Entity upload format") + + asyncio_tasks = [] + for _, geojson_dict in task_geojson_dict.items(): + # Extract the features list and pass each Feature through + features = geojson_dict.get("features", []) + asyncio_tasks.extend( + [feature_geojson_to_entity_dict(feature) for feature in features if feature] + ) + + return await gather(*asyncio_tasks) + + +async def create_entity_list( + odk_creds: project_schemas.ODKCentralDecrypted, + odk_id: int, + dataset_name: str = "features", + properties: list[str] = None, + entities_list: list[central_schemas.EntityDict] = None, +) -> None: + """Create a new Entity list in ODK.""" + if properties is None: + # Get the default properties for FMTM project + properties = central_schemas.entity_fields_to_list() + log.debug(f"Using default FMTM properties for Entity creation: {properties}") + + async with central_deps.get_odk_dataset(odk_creds) as odk_central: + # Step 1: create the Entity list, with properties + await odk_central.createDataset( + odk_id, datasetName=dataset_name, properties=properties + ) + # Step 2: populate the Entities + if entities_list: + await odk_central.createEntities( + odk_id, + dataset_name, + entities_list, + ) + + async def get_entities_geojson( odk_creds: project_schemas.ODKCentralDecrypted, odk_id: int, diff --git a/src/backend/app/central/central_schemas.py b/src/backend/app/central/central_schemas.py index 2f62e1c5f..456f75d77 100644 --- a/src/backend/app/central/central_schemas.py +++ b/src/backend/app/central/central_schemas.py @@ -17,8 +17,9 @@ # """Schemas for returned ODK Central objects.""" +from dataclasses import dataclass from enum import Enum -from typing import Optional +from typing import Optional, TypedDict from geojson_pydantic import Feature, FeatureCollection from pydantic import BaseModel, Field, ValidationInfo, computed_field @@ -27,6 +28,48 @@ from app.models.enums import TaskStatus +@dataclass +class NameTypeMapping: + """A simple dataclass mapping field name to field type.""" + + name: str + type: str + + +ENTITY_FIELDS: list[NameTypeMapping] = [ + NameTypeMapping(name="geometry", type="geopoint"), + NameTypeMapping(name="project_id", type="string"), + NameTypeMapping(name="task_id", type="string"), + NameTypeMapping(name="osm_id", type="string"), + NameTypeMapping(name="tags", type="string"), + NameTypeMapping(name="version", type="string"), + NameTypeMapping(name="changeset", type="string"), + NameTypeMapping(name="timestamp", type="datetime"), + NameTypeMapping(name="status", type="string"), +] + + +def entity_fields_to_list() -> list[str]: + """Converts a list of Field objects to a list of field names.""" + return [field.name for field in ENTITY_FIELDS] + + +# Dynamically generate EntityPropertyDict using ENTITY_FIELDS +def create_entity_property_dict() -> dict[str, type]: + """Dynamically create a TypedDict using the defined fields.""" + return {field.name: str for field in ENTITY_FIELDS} + + +EntityPropertyDict = TypedDict("EntityPropertyDict", create_entity_property_dict()) + + +class EntityDict(TypedDict): + """Dict of Entity label and data.""" + + label: str + data: EntityPropertyDict + + class CentralBase(BaseModel): """ODK Central return.""" diff --git a/src/backend/app/db/postgis_utils.py b/src/backend/app/db/postgis_utils.py index 67a6961e7..7d7b850b8 100644 --- a/src/backend/app/db/postgis_utils.py +++ b/src/backend/app/db/postgis_utils.py @@ -19,7 +19,6 @@ import json import logging -from asyncio import gather from datetime import datetime, timezone from io import BytesIO from random import getrandbits @@ -84,7 +83,7 @@ def featcol_to_wkb_geom( features = featcol.get("features", []) if len(features) > 1 and features[0].get("type") == "MultiPolygon": - featcol = multipolygon_to_polygon(featcol) + featcol = multigeom_to_singlegeom(featcol) features = featcol.get("features", []) geometry = features[0].get("geometry") @@ -347,7 +346,7 @@ async def split_geojson_by_task_areas( log.error("Attempted geojson task splitting failed") return None - if feature_collections: + if feature_collections and len(feature_collections[0]) > 1: # NOTE the feature collections are nested in a tuple, first remove task_geojson_dict = { record[0]: geojson.loads(json.dumps(record[1])) @@ -407,11 +406,11 @@ def add_required_geojson_properties( def normalise_featcol(featcol: geojson.FeatureCollection) -> geojson.FeatureCollection: - """Normalise a FeatureCollection into a standadised format. + """Normalise a FeatureCollection into a standardised format. The final FeatureCollection will only contain: - Polygon - - Polyline + - LineString - Point Processed: @@ -441,7 +440,7 @@ def normalise_featcol(featcol: geojson.FeatureCollection) -> geojson.FeatureColl coords.pop() # Convert MultiPolygon type --> individual Polygons - return multipolygon_to_polygon(featcol) + return multigeom_to_singlegeom(featcol) def geojson_to_featcol(geojson_obj: dict) -> geojson.FeatureCollection: @@ -498,7 +497,7 @@ def featcol_keep_dominant_geom_type( def get_featcol_dominant_geom_type(featcol: geojson.FeatureCollection) -> str: """Get the predominant geometry type in a FeatureCollection.""" - geometry_counts = {"Polygon": 0, "Point": 0, "Polyline": 0} + geometry_counts = {"Polygon": 0, "Point": 0, "LineString": 0} for feature in featcol.get("features", []): geometry_type = feature.get("geometry", {}).get("type", "") @@ -552,6 +551,10 @@ def is_valid_coordinate(coord): first_coordinate = coordinates coordinates = coordinates[0] + error_message = ( + "ERROR: The coordinates within the GeoJSON file are not valid. " + "Is the file empty?" + ) if not is_valid_coordinate(first_coordinate): log.error(error_message) raise HTTPException(status_code=400, detail=error_message) @@ -629,20 +632,30 @@ async def geojson_to_javarosa_geom(geojson_geometry: dict) -> str: if geojson_geometry is None: return "" - coordinates = [] - if geojson_geometry["type"] in ["Point", "LineString", "MultiPoint"]: - coordinates = [[geojson_geometry.get("coordinates", [])]] - elif geojson_geometry["type"] in ["Polygon", "MultiLineString"]: - coordinates = geojson_geometry.get("coordinates", []) - elif geojson_geometry["type"] == "MultiPolygon": - # Flatten the list structure to get coordinates of all polygons - coordinates = sum(geojson_geometry.get("coordinates", []), []) + coordinates = geojson_geometry.get("coordinates", []) + geometry_type = geojson_geometry["type"] + + # Normalise single geometries into the same structure as multi-geometries + # We end up with three levels of nesting for the processing below + if geometry_type == "Point": + # Format [x, y] + coordinates = [[coordinates]] + elif geometry_type in ["LineString", "MultiPoint"]: + # Format [[x, y], [x, y]] + coordinates = [coordinates] + elif geometry_type in ["Polygon", "MultiLineString"]: + # Format [[[x, y], [x, y]]] + pass + elif geometry_type == "MultiPolygon": + # Format [[[[x, y], [x, y]]]], flatten coords + coordinates = [coord for poly in coordinates for coord in poly] else: - raise ValueError("Unsupported GeoJSON geometry type") + raise ValueError(f"Unsupported GeoJSON geometry type: {geometry_type}") + # Prepare the JavaRosa format by iterating over coordinates javarosa_geometry = [] - for polygon in coordinates: - for lon, lat in polygon: + for polygon_or_line in coordinates: + for lon, lat in polygon_or_line: javarosa_geometry.append(f"{lat} {lon} 0.0 0.0") return ";".join(javarosa_geometry) @@ -664,7 +677,7 @@ async def javarosa_to_geojson_geom(javarosa_geom_string: str, geom_type: str) -> if geom_type == "Point": lat, lon, _, _ = map(float, javarosa_geom_string.split()) geojson_geometry = {"type": "Point", "coordinates": [lon, lat]} - elif geom_type == "Polyline": + elif geom_type == "LineString": coordinates = [ [float(coord) for coord in reversed(point.split()[:2])] for point in javarosa_geom_string.split(";") @@ -685,56 +698,26 @@ async def javarosa_to_geojson_geom(javarosa_geom_string: str, geom_type: str) -> return geojson_geometry -async def feature_geojson_to_entity_dict( - feature: dict, -) -> dict: - """Convert a single GeoJSON to an Entity dict for upload.""" - feature_id = feature.get("id") - - geometry = feature.get("geometry", {}) - if not geometry: - msg = "'geometry' data field is mandatory" - log.debug(msg) - raise ValueError(msg) - - javarosa_geom = await geojson_to_javarosa_geom(geometry) - - # NOTE all properties MUST be string values for Entities, convert - properties = { - str(key): str(value) for key, value in feature.get("properties", {}).items() - } - # Set to TaskStatus enum READY value (0) - properties["status"] = "0" - - task_id = properties.get("task_id") - entity_label = f"Task {task_id} Feature {feature_id}" - - return {"label": entity_label, "data": {"geometry": javarosa_geom, **properties}} - - -async def task_geojson_dict_to_entity_values(task_geojson_dict): - """Convert a dict of task GeoJSONs into data for ODK Entity upload.""" - asyncio_tasks = [] - for _, geojson_dict in task_geojson_dict.items(): - features = geojson_dict.get("features", []) - asyncio_tasks.extend( - [feature_geojson_to_entity_dict(feature) for feature in features if feature] - ) - - return await gather(*asyncio_tasks) - - -def multipolygon_to_polygon( +def multigeom_to_singlegeom( featcol: geojson.FeatureCollection, ) -> geojson.FeatureCollection: - """Converts a GeoJSON FeatureCollection of MultiPolygons to Polygons. + """Converts any Multi(xxx) geometry types to individual geometries. Args: - featcol : A GeoJSON FeatureCollection containing MultiPolygons/Polygons. + featcol : A GeoJSON FeatureCollection of geometries. Returns: - geojson.FeatureCollection: A GeoJSON FeatureCollection containing Polygons. + geojson.FeatureCollection: A GeoJSON FeatureCollection containing + single geometry types only: Polygon, LineString, Point. """ + + def split_multigeom(geom, properties): + """Splits multi-geometries into individual geometries.""" + return [ + geojson.Feature(geometry=mapping(single_geom), properties=properties) + for single_geom in geom.geoms + ] + final_features = [] for feature in featcol.get("features", []): @@ -745,12 +728,16 @@ def multipolygon_to_polygon( log.warning(f"Geometry is not valid, so was skipped: {feature['geometry']}") continue - if geom.geom_type == "Polygon": - final_features.append(geojson.Feature(geometry=geom, properties=properties)) - elif geom.geom_type == "MultiPolygon": - final_features.extend( - geojson.Feature(geometry=polygon_coords, properties=properties) - for polygon_coords in geom.geoms + if geom.geom_type.startswith("Multi"): + # Handle all MultiXXX types + final_features.extend(split_multigeom(geom, properties)) + else: + # Handle single geometry types + final_features.append( + geojson.Feature( + geometry=mapping(geom), + properties=properties, + ) ) return geojson.FeatureCollection(final_features) diff --git a/src/backend/app/helpers/helper_routes.py b/src/backend/app/helpers/helper_routes.py index 76b397db0..7c1f66b7d 100644 --- a/src/backend/app/helpers/helper_routes.py +++ b/src/backend/app/helpers/helper_routes.py @@ -48,7 +48,7 @@ add_required_geojson_properties, featcol_keep_dominant_geom_type, javarosa_to_geojson_geom, - multipolygon_to_polygon, + multigeom_to_singlegeom, parse_geojson_file_to_featcol, ) from app.models.enums import GeometryType, HTTPStatus, XLSFormType @@ -264,7 +264,7 @@ async def flatten_multipolygons_to_polygons( raise HTTPException( status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail="No geometries present" ) - multi_to_single_polygons = multipolygon_to_polygon(featcol) + multi_to_single_polygons = multigeom_to_singlegeom(featcol) if multi_to_single_polygons: headers = { diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 89665ef8e..528bce3bb 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -22,6 +22,7 @@ from asyncio import gather from io import BytesIO from pathlib import Path +from traceback import extract_tb from typing import List, Optional, Union import geojson @@ -40,7 +41,7 @@ from sqlalchemy import and_, column, func, select, table, text from sqlalchemy.orm import Session -from app.central import central_crud, central_deps +from app.central import central_crud from app.config import settings from app.db import db_models from app.db.postgis_utils import ( @@ -52,7 +53,6 @@ merge_polygons, parse_geojson_file_to_featcol, split_geojson_by_task_areas, - task_geojson_dict_to_entity_values, wkb_geom_to_feature, ) from app.models.enums import HTTPStatus, ProjectRole, ProjectVisibility, XLSFormType @@ -748,17 +748,17 @@ async def upload_custom_fgb_extract( async def get_data_extract_type(featcol: FeatureCollection) -> str: """Determine predominant geometry type for extract.""" geom_type = get_featcol_dominant_geom_type(featcol) - if geom_type not in ["Polygon", "Polyline", "Point"]: + if geom_type not in ["Polygon", "LineString", "Point"]: msg = ( "Extract does not contain valid geometry types, from 'Polygon' " - ", 'Polyline' and 'Point'." + ", 'LineString' and 'Point'." ) log.error(msg) raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg) geom_name_map = { "Polygon": "polygon", "Point": "centroid", - "Polyline": "line", + "LineString": "line", } data_extract_type = geom_name_map.get(geom_type, "polygon") @@ -837,29 +837,24 @@ async def generate_odk_central_project_content( project: db_models.DbProject, odk_credentials: project_schemas.ODKCentralDecrypted, xlsform: BytesIO, - task_extract_dict: dict, + task_extract_dict: dict[int, geojson.FeatureCollection], db: Session, ) -> str: """Populate the project in ODK Central with XForm, Appuser, Permissions.""" project_odk_id = project.odkid # The ODK Dataset (Entity List) must exist prior to main XLSForm - entities_list = await task_geojson_dict_to_entity_values(task_extract_dict) - fields_list = project_schemas.entity_fields_to_list() - - async with central_deps.get_odk_dataset(odk_credentials) as odk_central: - await odk_central.createDataset( - project_odk_id, datasetName="features", properties=fields_list - ) - await odk_central.createEntities( - project_odk_id, - "features", - entities_list, - ) + entities_list = await central_crud.task_geojson_dict_to_entity_values( + task_extract_dict + ) - # TODO add here additional upload of Entities - # TODO add code here - # additional_entities = ["roads"] + log.debug("Creating main ODK Entity list for project: features") + await central_crud.create_entity_list( + odk_credentials, + project_odk_id, + dataset_name="features", + entities_list=entities_list, + ) # Do final check of XLSForm validity + return parsed XForm xform = await central_crud.read_and_test_xform(xlsform) @@ -967,6 +962,19 @@ async def generate_project_files( ) # 4 is COMPLETED except Exception as e: + # Get the traceback details for easier debugging + tb = extract_tb(e.__traceback__) + if tb: + last_entry = tb[-1] + function_name = last_entry.name + line_number = last_entry.lineno + file_name = last_entry.filename + + log.warning( + f"Error occurred in function {function_name} | line {line_number}" + f" | file {file_name}" + ) + log.warning(str(e)) if background_task_id: diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index eae5363b2..81fbb66bc 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -54,6 +54,7 @@ flatgeobuf_to_featcol, merge_polygons, parse_geojson_file_to_featcol, + split_geojson_by_task_areas, wkb_geom_to_feature, ) from app.models.enums import ( @@ -624,8 +625,7 @@ async def task_split( """Split a task into subtasks. Args: - project_geojson (UploadFile): The geojson to split. - Should be a FeatureCollection. + project_geojson (UploadFile): The geojson (AOI) to split. extract_geojson (UploadFile, optional): Custom data extract geojson containing osm features (should be a FeatureCollection). If not included, an extract is generated automatically. @@ -782,6 +782,42 @@ async def generate_files( ) +@router.post("/{project_id}/additional-entity") +async def add_additional_entity_list( + geojson: UploadFile = File(...), + db: Session = Depends(database.get_db), + project_user_dict: ProjectUserDict = Depends(project_manager), +): + """Add an additional Entity list for the project in ODK. + + Note that the Entity list will be named from the filename + of the GeoJSON uploaded. + """ + project = project_user_dict.get("project") + project_id = project.id + project_odk_id = project.odkid + odk_credentials = await project_deps.get_odk_credentials(db, project_id) + # NOTE the Entity name is extracted from the filename (without extension) + entity_name = Path(geojson.filename).stem + + # Parse geojson + divide by task + # (not technically required, but also appends properties in correct format) + featcol = parse_geojson_file_to_featcol(await geojson.read()) + feature_split_by_task = await split_geojson_by_task_areas(db, featcol, project_id) + entities_list = await central_crud.task_geojson_dict_to_entity_values( + feature_split_by_task + ) + + await central_crud.create_entity_list( + odk_credentials, + project_odk_id, + dataset_name=entity_name, + entities_list=entities_list, + ) + + return Response(status_code=HTTPStatus.OK) + + @router.get("/categories/") async def get_categories(current_user: AuthUser = Depends(login_required)): """Get api for fetching all the categories. @@ -802,7 +838,7 @@ async def get_categories(current_user: AuthUser = Depends(login_required)): @router.post("/preview-split-by-square/") async def preview_split_by_square( project_geojson: UploadFile = File(...), - extract_geojson: UploadFile = File(None), + extract_geojson: Optional[UploadFile] = File(None), dimension: int = Form(100), ): """Preview splitting by square. @@ -940,7 +976,7 @@ async def download_form( project = project_user.get("project") headers = { - "Content-Disposition": "attachment; filename=submission_data.xls", + "Content-Disposition": f"attachment; filename={project.id}_xlsform.xlsx", "Content-Type": "application/media", } return Response(content=project.form_xls, headers=headers) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index e69dc2b60..670e5574f 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -18,7 +18,6 @@ """Pydantic schemas for Projects.""" import uuid -from dataclasses import dataclass from datetime import datetime from typing import Any, List, Optional, Union @@ -437,39 +436,3 @@ def get_last_active(self, value, values): return f'{days_difference} day{"s" if days_difference > 1 else ""} ago' else: return last_active.strftime("%d %b %Y") - - -@dataclass -class Field: - """A data class representing a field with a name and type. - - Args: - name (str): The name of the field. - type (str): The type of the field. - - Returns: - None - """ - - name: str - type: str - - -def entity_fields_to_list() -> List[str]: - """Converts a list of Field objects to a list of field names. - - Returns: - List[str]: A list of fields. - """ - fields: List[Field] = [ - Field(name="geometry", type="geopoint"), - Field(name="project_id", type="string"), - Field(name="task_id", type="string"), - Field(name="osm_id", type="string"), - Field(name="tags", type="string"), - Field(name="version", type="string"), - Field(name="changeset", type="string"), - Field(name="timestamp", type="datetime"), - Field(name="status", type="string"), - ] - return [field.name for field in fields] diff --git a/src/backend/tests/test_projects_routes.py b/src/backend/tests/test_projects_routes.py index 19a45b1b7..08e00bbc7 100644 --- a/src/backend/tests/test_projects_routes.py +++ b/src/backend/tests/test_projects_routes.py @@ -552,13 +552,31 @@ def compare_entities(response_entity, expected_entity): def test_project_task_split(client): - """Test project task split.""" - with open(f"{test_data_path}/data_extract_kathmandu.geojson", "rb") as project_file: - project_geojson = project_file.read() + """Test project AOI splitting into tasks.""" + aoi_geojson = json.dumps( + { + "type": "Polygon", + "coordinates": [ + [ + [85.317028828, 27.7052522097], + [85.317028828, 27.7041424888], + [85.318844411, 27.7041424888], + [85.318844411, 27.7052522097], + [85.317028828, 27.7052522097], + ] + ], + } + ).encode("utf-8") + aoi_geojson_file = { + "project_geojson": ( + "kathmandu_aoi.geojson", + BytesIO(aoi_geojson).read(), + ) + } response = client.post( "/projects/task-split", - files={"project_geojson": ("data_extract_kathmandu.geojson", project_geojson)}, + files=aoi_geojson_file, data={"no_of_buildings": 40}, ) diff --git a/src/frontend/e2e/02-mapper-flow.spec.ts b/src/frontend/e2e/02-mapper-flow.spec.ts index 48ea6062a..b6cc32a7e 100644 --- a/src/frontend/e2e/02-mapper-flow.spec.ts +++ b/src/frontend/e2e/02-mapper-flow.spec.ts @@ -3,7 +3,7 @@ import { test, expect } from '@playwright/test'; -import { tempLogin, openFirstProject } from './helpers'; +import { tempLogin, openTestProject } from './helpers'; test.describe('mapper flow', () => { test('task actions', async ({ browserName, page }) => { @@ -13,7 +13,7 @@ test.describe('mapper flow', () => { // 0. Temp Login await tempLogin(page); - await openFirstProject(page); + await openTestProject(page); // 1. Click on task area on map await page.locator('canvas').click({ @@ -59,6 +59,7 @@ test.describe('mapper flow', () => { // 3. Mark task as fully mapped await page.getByRole('button', { name: 'MARK AS FULLY MAPPED' }).click(); + // Required again for the confirmation dialog (0/4 features mapped) await page.getByRole('button', { name: 'MARK AS FULLY MAPPED' }).click(); await page.waitForSelector('div:has-text("updated status to MAPPED"):nth-of-type(1)'); await expect( @@ -108,7 +109,7 @@ test.describe('mapper flow', () => { // 0. Temp Login await tempLogin(page); - await openFirstProject(page); + await openTestProject(page); // 1. Click on task area on map // click on task & assert task popup visibility @@ -177,7 +178,7 @@ test.describe('mapper flow', () => { // 0. Temp Login await tempLogin(page); - await openFirstProject(page); + await openTestProject(page); await page.locator('canvas').click({ position: { diff --git a/src/frontend/e2e/helpers.ts b/src/frontend/e2e/helpers.ts index cbd3a6db9..5c5a3d26d 100644 --- a/src/frontend/e2e/helpers.ts +++ b/src/frontend/e2e/helpers.ts @@ -6,8 +6,11 @@ export async function tempLogin(page: Page) { await page.getByText('Temporary Account').click(); } -export async function openFirstProject(page: Page) { - // click first project card on the home page - await page.locator('.MuiCardContent-root').first().click(); +export async function openTestProject(page: Page) { + // open project card with regex text 'Project Create Playwright xxx' + await page + .getByText(/^Project Create Playwright/) + .first() + .click(); await page.waitForTimeout(4000); } diff --git a/src/frontend/src/api/Project.ts b/src/frontend/src/api/Project.ts index 1f5718f71..9e346e4aa 100755 --- a/src/frontend/src/api/Project.ts +++ b/src/frontend/src/api/Project.ts @@ -90,7 +90,7 @@ export const DownloadProjectForm = (url: string, downloadType: 'form' | 'geojson const a = document.createElement('a'); a.href = window.URL.createObjectURL(response.data); a.download = `${ - downloadType === 'form' ? `project_form_${projectId}.xls` : `task_polygons_${projectId}.geojson` + downloadType === 'form' ? `project_form_${projectId}.xlsx` : `task_polygons_${projectId}.geojson` }`; a.click(); dispatch(ProjectActions.SetDownloadProjectFormLoading({ type: downloadType, loading: false })); From 35e7240cc78d62c288f0ea60aef0eb7bde8beaed Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 22 Sep 2024 00:47:52 +0100 Subject: [PATCH 3/3] build: upgrade fmtm-splitter --> v1.3.1 for latest fixes --- src/backend/pdm.lock | 8 ++++---- src/backend/pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 0af91a8e8..3ec765848 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test", "monitoring"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:a02ce45e28be043d654f512ec57a6db6adf4cba528cb2c937191e3d025ca5490" +content_hash = "sha256:8c65c80f5e570cc8e16aad773965a60059d14e05b51e55dd334de4b4082329bb" [[package]] name = "aiohttp" @@ -599,7 +599,7 @@ files = [ [[package]] name = "fmtm-splitter" -version = "1.3.0" +version = "1.3.1" requires_python = ">=3.10" summary = "A utility for splitting an AOI into multiple tasks." dependencies = [ @@ -610,8 +610,8 @@ dependencies = [ "shapely>=1.8.1", ] files = [ - {file = "fmtm-splitter-1.3.0.tar.gz", hash = "sha256:047ecc3d71234b8949ddb9422039ef900a128b2417d6f5c8e22c5e14eab84ae4"}, - {file = "fmtm_splitter-1.3.0-py3-none-any.whl", hash = "sha256:194ea5dca1e5ffd0d8944ae9566c7c752c8bb95a648bca19cea3df7d4a6e0487"}, + {file = "fmtm-splitter-1.3.1.tar.gz", hash = "sha256:90b739df69c1ab8ad18d615423ef230665e0b43b94c3e6c1ce345f8e4021e18f"}, + {file = "fmtm_splitter-1.3.1-py3-none-any.whl", hash = "sha256:409795cbb6c2d261544e2dcf6314aabdd0b63c47af7996a26958356003b345fe"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 6a5d4b411..1829caf20 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "osm-fieldwork>=0.16.4", "osm-login-python==2.0.0", "osm-rawdata==0.3.2", - "fmtm-splitter==1.3.0", + "fmtm-splitter==1.3.1", ] requires-python = ">=3.11" readme = "../../README.md"