diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 2bdf66677c..6900ac34e8 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -629,139 +629,3 @@ def convert_csv( csvin.finishGeoJson() return True - - -def create_odk_xform_for_janakpur( - project_id: int, - xform_id: str, - filespec: str, - odk_credentials: project_schemas.ODKCentral = None, - create_draft: bool = False, - upload_media=True, - convert_to_draft_when_publishing=True, -): - """Create an XForm on a remote ODK Central server.""" - title = os.path.basename(os.path.splitext(filespec)[0]) - # result = xform.createForm(project_id, title, filespec, True) - # Pass odk credentials of project in xform - - if not odk_credentials: - odk_credentials = project_schemas.ODKCentral( - odk_central_url=settings.ODK_CENTRAL_URL, - odk_central_user=settings.ODK_CENTRAL_USER, - odk_central_password=settings.ODK_CENTRAL_PASSWD, - ) - try: - xform = get_odk_form(odk_credentials) - except Exception as e: - log.error(e) - raise HTTPException( - status_code=500, detail={"message": "Connection failed to odk central"} - ) from e - - result = xform.createForm(project_id, xform_id, filespec, create_draft) - - if result != 200 and result != 409: - return result - - # This modifies an existing published XForm to be in draft mode. - # An XForm must be in draft mode to upload an attachment. - if upload_media: - # Upload buildings file - building_file = f"/tmp/buildings_{title}.geojson" - - result = xform.uploadMedia( - project_id, title, building_file, convert_to_draft_when_publishing - ) - - # Upload roads file - road_file = f"/tmp/roads_{title}.geojson" - result = xform.uploadMedia( - project_id, title, road_file, convert_to_draft_when_publishing - ) - - result = xform.publishForm(project_id, title) - return result - - -def generate_updated_xform_for_janakpur( - xlsform: str, - xform: str, - form_type: str, -): - """Update the version in an XForm so it's unique.""" - name = os.path.basename(xform).replace(".xml", "") - - log.debug(f"Name in form = {name}") - - outfile = xform - if form_type != "xml": - try: - xls2xform_convert(xlsform_path=xlsform, xform_path=outfile, validate=False) - except Exception as e: - log.error(f"Couldn't convert {xlsform} to an XForm!", str(e)) - raise HTTPException(status_code=400, detail=str(e)) from e - - if os.path.getsize(outfile) <= 0: - log.warning(f"{outfile} is empty!") - raise HTTPException(status=400, detail=f"{outfile} is empty!") from None - - xls = open(outfile, "r") - data = xls.read() - xls.close() - else: - xls = open(xlsform, "r") - data = xls.read() - xls.close() - - tmp = name.split("_") - tmp[0] - tmp[1] - id = tmp[2].split(".")[0] - - buildings_extract = f"jr://file/buildings_{name}.geojson" - roads_extract = f"jr://file/roads_{name}.geojson" - - namespaces = { - "h": "http://www.w3.org/1999/xhtml", - "odk": "http://www.opendatakit.org/xforms", - "xforms": "http://www.w3.org/2002/xforms", - } - - import xml.etree.ElementTree as ET - - root = ET.fromstring(data) - head = root.find("h:head", namespaces) - model = head.find("xforms:model", namespaces) - instances = model.findall("xforms:instance", namespaces) - - index = 0 - for inst in instances: - try: - if "src" in inst.attrib: - print("SRC = Present") - if (inst.attrib["src"]) == "jr://file/buildings.geojson": # FIXME - print("INST attribs = ", inst.attrib["src"]) - inst.attrib["src"] = buildings_extract - - if (inst.attrib["src"]) == "jr://file/roads.geojson": # FIXME - inst.attrib["src"] = roads_extract - - # Looking for data tags - data_tags = inst.findall("xforms:data", namespaces) - if data_tags: - for dt in data_tags: - dt.attrib["id"] = id - except Exception: - continue - index += 1 - - # Save the modified XML - newxml = ET.tostring(root) - - # write the updated XML file - outxml = open(outfile, "w") - outxml.write(newxml.decode()) - outxml.close() - - return outfile diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 9a234e266b..a708a9ad2f 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -2491,254 +2491,6 @@ async def convert_geojson_to_osm(geojson_file: str): return json2osm(geojson_file) -def generate_appuser_files_for_janakpur( - db: Session, - project_id: int, - form: str, - building_extracts_contents: str, - road_extracts_contents: str, - category: str, - form_type: str, - background_task_id: uuid.UUID, -): - project_log = log.bind(task="create_project", project_id=project_id) - - project_log.info(f"Starting generate_appuser_files for project {project_id}") - - # Get the project table contents. - project = table( - "projects", - column("project_name_prefix"), - column("xform_title"), - column("id"), - column("odkid"), - column("odk_central_url"), - column("odk_central_user"), - column("odk_central_password"), - column("outline"), - ) - - where = f"id={project_id}" - sql = select( - project.c.project_name_prefix, - project.c.xform_title, - project.c.id, - project.c.odkid, - project.c.odk_central_url, - project.c.odk_central_user, - project.c.odk_central_password, - geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label("outline"), - ).where(text(where)) - result = db.execute(sql) - - # There should only be one match - if result.rowcount != 1: - log.warning(str(sql)) - if result.rowcount < 1: - raise HTTPException(status_code=400, detail="Project not found") - else: - raise HTTPException(status_code=400, detail="Multiple projects found") - - one = result.first() - - if one: - prefix = one.project_name_prefix - - # Get odk credentials from project. - odk_credentials = { - "odk_central_url": one.odk_central_url, - "odk_central_user": one.odk_central_user, - "odk_central_password": one.odk_central_password, - } - - odk_credentials = project_schemas.ODKCentral(**odk_credentials) - - xform_title = one.xform_title if one.xform_title else None - - if form: - xlsform = f"/tmp/custom_form.{form_type}" - contents = form - with open(xlsform, "wb") as f: - f.write(contents) - else: - xlsform = f"{xlsforms_path}/{xform_title}.xls" - - category = xform_title - - # FIXME: Need to figure out this step. - # Data Extracts - if building_extracts_contents is not None: - project_log.info("Uploading data extracts") - upload_custom_data_extracts(db, project_id, building_extracts_contents) - - if road_extracts_contents is not None: - project_log.info("Uploading roads data") - upload_custom_data_extracts( - db, project_id, road_extracts_contents, "highways" - ) - - # Generating QR Code, XForm and uploading OSM Extracts to the form. - # Creating app users and updating the role of that usegenerate_updated_xformr. - tasks_list = tasks_crud.get_task_lists(db, project_id) - - project_name = prefix - odk_id = one.odkid - project_obj = get_project(db, project_id) - - for task_id in tasks_list: - # Generate taskFiles - name = f"{project_name}_{category}_{task_id}" - - appuser = central_crud.create_appuser(odk_id, name, odk_credentials) - - # If app user could not be created, raise an exception. - if not appuser: - project_log.error("Couldn't create appuser for project") - return False - - # prefix should be sent instead of name - project_log.info(f"Creating qr code for task_id {task_id}") - create_qr = create_qrcode( - db, - odk_id, - appuser.json()["token"], - project_name, - odk_credentials.odk_central_url, - ) - - task = tasks_crud.get_task(db, task_id) - task.qr_code_id = create_qr["qr_code_id"] - db.commit() - db.refresh(task) - - # This file will store xml contents of an xls form. - xform = f"/tmp/{name}.xml" - - buildings_extracts = ( - f"/tmp/buildings_{name}.geojson" # This file will store osm extracts - ) - roads_extracts = ( - f"/tmp/roads_{name}.geojson" # This file will store osm extracts - ) - - # xform_id_format - xform_id = f"{name}".split("_")[2] - - # Get the features for this task. - # Postgis query to filter task inside this task outline and of this project - # Update those features and set task_id - query = text( - f"""UPDATE features - SET task_id={task_id} - WHERE id IN ( - SELECT id - FROM features - WHERE project_id={project_id} - AND ST_IsValid(geometry) - AND ST_IsValid('{task.outline}'::Geometry) - AND ST_Contains('{task.outline}'::Geometry, ST_Centroid(geometry)) - )""" - ) - result = db.execute(query) - - # Get the geojson of those features for this task. - buildings_query = text( - f"""SELECT jsonb_build_object( - 'type', 'FeatureCollection', - 'features', jsonb_agg(feature) - ) - FROM ( - SELECT jsonb_build_object( - 'type', 'Feature', - 'id', id, - 'geometry', ST_AsGeoJSON(geometry)::jsonb, - 'properties', properties - ) AS feature - FROM features - WHERE project_id={project_id} and task_id={task_id} and category_title='buildings' - ) features;""" - ) - result = db.execute(buildings_query) - features = result.fetchone()[0] - - highway_query = text( - f"""SELECT jsonb_build_object( - 'type', 'FeatureCollection', - 'features', jsonb_agg(feature) - ) - FROM ( - SELECT jsonb_build_object( - 'type', 'Feature', - 'id', id, - 'geometry', ST_AsGeoJSON(geometry)::jsonb, - 'properties', properties - ) AS feature - FROM features - WHERE project_id={project_id} and category_title='highways' - ) features;""" - ) - highway_result = db.execute(highway_query) - highway_features = highway_result.fetchone()[0] - - # upload_media = False if features["features"] is None else True - upload_media = True - - # Update outfile containing osm extracts with the new geojson contents containing title in the properties. - with open(buildings_extracts, "w") as jsonfile: - jsonfile.truncate(0) # clear the contents of the file - dump(features, jsonfile) - - # Update outfile containing osm extracts with the new geojson contents containing title in the properties. - with open(roads_extracts, "w") as jsonfile: - jsonfile.truncate(0) # clear the contents of the file - dump(highway_features, jsonfile) - - project_log.info(f"Generating xform for task {task_id}") - outfile = central_crud.generate_updated_xform_for_janakpur( - xlsform, xform, form_type - ) - - # Create an odk xform - project_log.info(f"Uploading media in {task_id}") - result = central_crud.create_odk_xform_for_janakpur( - odk_id, task, outfile, odk_credentials, False, upload_media - ) - - project_log.info(f"Updating role for app user in task {task_id}") - # Update the user role for the created xform. - try: - # Pass odk credentials - if odk_credentials: - url = odk_credentials.odk_central_url - user = odk_credentials.odk_central_user - pw = odk_credentials.odk_central_password - - else: - log.debug("ODKCentral connection variables not set in function") - log.debug("Attempting extraction from environment variables") - url = settings.ODK_CENTRAL_URL - user = settings.ODK_CENTRAL_USER - pw = settings.ODK_CENTRAL_PASSWD - - odk_app = OdkAppUser(url, user, pw) - - odk_app.updateRole( - projectId=odk_id, xform=xform_id, actorId=appuser.json()["id"] - ) - - except Exception as e: - log.warning(str(e)) - - project_obj.extract_completed_count += 1 - db.commit() - db.refresh(project_obj) - - # Update background task status to COMPLETED - update_background_task_status_in_database( - db, background_task_id, 4 - ) # 4 is COMPLETED - - def get_address_from_lat_lon(latitude, longitude): """Get address using Nominatim, using lat,lon.""" base_url = "https://nominatim.openstreetmap.org/reverse" diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index bd652ff53b..bde0248aa6 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -1175,91 +1175,6 @@ async def project_centroid( return result_dict_list -@router.post("/{project_id}/generate_files_for_janakpur") -async def generate_files_janakpur( - background_tasks: BackgroundTasks, - project_id: int, - buildings_file: UploadFile, - roads_file: UploadFile, - form: UploadFile, - db: Session = Depends(database.get_db), -): - """Generate required media files tasks in the project based on the provided params.""" - log.debug(f"Generating media files tasks for project: {project_id}") - xform_title = None - - project = project_crud.get_project(db, project_id) - if not project: - raise HTTPException( - status_code=428, detail=f"Project with id {project_id} does not exist" - ) - - project.data_extract_type = "polygon" - db.commit() - - if form: - log.debug("Validating uploaded XLS file") - # Validating for .XLS File. - file_name = os.path.splitext(form.filename) - file_ext = file_name[1] - allowed_extensions = [".xls", ".xlsx", ".xml"] - if file_ext not in allowed_extensions: - raise HTTPException(status_code=400, detail="Provide a valid .xls file") - xform_title = file_name[0] - project.form_xls = await form.read() - db.commit() - - if buildings_file: - log.debug("Validating uploaded buildings geojson file") - # Validating for .geojson File. - data_extracts_file_name = os.path.splitext(buildings_file.filename) - extracts_file_ext = data_extracts_file_name[1] - if extracts_file_ext != ".geojson": - raise HTTPException(status_code=400, detail="Provide a valid geojson file") - try: - buildings_extracts_contents = await buildings_file.read() - json.loads(buildings_extracts_contents) - except json.JSONDecodeError: - raise HTTPException(status_code=400, detail="Provide a valid geojson file") - - if roads_file: - log.debug("Validating uploaded roads geojson file") - # Validating for .geojson File. - road_extracts_file_name = os.path.splitext(roads_file.filename) - road_extracts_file_ext = road_extracts_file_name[1] - if road_extracts_file_ext != ".geojson": - raise HTTPException(status_code=400, detail="Provide a valid geojson file") - try: - road_extracts_contents = await roads_file.read() - json.loads(road_extracts_contents) - except json.JSONDecodeError: - raise HTTPException(status_code=400, detail="Provide a valid geojson file") - - # Create task in db and return uuid - log.debug( - "Creating generate_files_for_janakpur" - f"background task for project ID: {project_id}" - ) - background_task_id = await project_crud.insert_background_task_into_database( - db, project_id=project_id - ) - - log.debug(f"Submitting {background_task_id} to background tasks stack") - background_tasks.add_task( - project_crud.generate_appuser_files_for_janakpur, - db, - project_id, - contents, - buildings_extracts_contents if buildings_file else None, - road_extracts_contents if roads_file else None, - xform_title, - file_ext[1:] if form else "xls", - background_task_id, - ) - - return {"Message": project_id, "task_id": background_task_id} - - @router.get("/task-status/{uuid}", response_model=project_schemas.BackgroundTaskStatus) async def get_task_status( background_tasks: BackgroundTasks,