Skip to content

Commit

Permalink
refactor: remove endpoints specific to janakpur project (#1012)
Browse files Browse the repository at this point in the history 
* remove functions to generate files for janakpur project

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
@nrjadkry @pre-commit-ci
nrjadkry and pre-commit-ci[bot] authored Nov 28, 2023
1 parent 9cb8a60 commit 4106a01
Show file tree
Hide file tree
Showing 3 changed files with 0 additions and 469 deletions.
136 changes: 0 additions & 136 deletions src/backend/app/central/central_crud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,139 +629,3 @@ def convert_csv(
csvin.finishGeoJson()

return True


def create_odk_xform_for_janakpur(
project_id: int,
xform_id: str,
filespec: str,
odk_credentials: project_schemas.ODKCentral = None,
create_draft: bool = False,
upload_media=True,
convert_to_draft_when_publishing=True,
):
"""Create an XForm on a remote ODK Central server."""
title = os.path.basename(os.path.splitext(filespec)[0])
# result = xform.createForm(project_id, title, filespec, True)
# Pass odk credentials of project in xform

if not odk_credentials:
odk_credentials = project_schemas.ODKCentral(
odk_central_url=settings.ODK_CENTRAL_URL,
odk_central_user=settings.ODK_CENTRAL_USER,
odk_central_password=settings.ODK_CENTRAL_PASSWD,
)
try:
xform = get_odk_form(odk_credentials)
except Exception as e:
log.error(e)
raise HTTPException(
status_code=500, detail={"message": "Connection failed to odk central"}
) from e

result = xform.createForm(project_id, xform_id, filespec, create_draft)

if result != 200 and result != 409:
return result

# This modifies an existing published XForm to be in draft mode.
# An XForm must be in draft mode to upload an attachment.
if upload_media:
# Upload buildings file
building_file = f"/tmp/buildings_{title}.geojson"

result = xform.uploadMedia(
project_id, title, building_file, convert_to_draft_when_publishing
)

# Upload roads file
road_file = f"/tmp/roads_{title}.geojson"
result = xform.uploadMedia(
project_id, title, road_file, convert_to_draft_when_publishing
)

result = xform.publishForm(project_id, title)
return result


def generate_updated_xform_for_janakpur(
xlsform: str,
xform: str,
form_type: str,
):
"""Update the version in an XForm so it's unique."""
name = os.path.basename(xform).replace(".xml", "")

log.debug(f"Name in form = {name}")

outfile = xform
if form_type != "xml":
try:
xls2xform_convert(xlsform_path=xlsform, xform_path=outfile, validate=False)
except Exception as e:
log.error(f"Couldn't convert {xlsform} to an XForm!", str(e))
raise HTTPException(status_code=400, detail=str(e)) from e

if os.path.getsize(outfile) <= 0:
log.warning(f"{outfile} is empty!")
raise HTTPException(status=400, detail=f"{outfile} is empty!") from None

xls = open(outfile, "r")
data = xls.read()
xls.close()
else:
xls = open(xlsform, "r")
data = xls.read()
xls.close()

tmp = name.split("_")
tmp[0]
tmp[1]
id = tmp[2].split(".")[0]

buildings_extract = f"jr://file/buildings_{name}.geojson"
roads_extract = f"jr://file/roads_{name}.geojson"

namespaces = {
"h": "http://www.w3.org/1999/xhtml",
"odk": "http://www.opendatakit.org/xforms",
"xforms": "http://www.w3.org/2002/xforms",
}

import xml.etree.ElementTree as ET

root = ET.fromstring(data)
head = root.find("h:head", namespaces)
model = head.find("xforms:model", namespaces)
instances = model.findall("xforms:instance", namespaces)

index = 0
for inst in instances:
try:
if "src" in inst.attrib:
print("SRC = Present")
if (inst.attrib["src"]) == "jr://file/buildings.geojson": # FIXME
print("INST attribs = ", inst.attrib["src"])
inst.attrib["src"] = buildings_extract

if (inst.attrib["src"]) == "jr://file/roads.geojson": # FIXME
inst.attrib["src"] = roads_extract

# Looking for data tags
data_tags = inst.findall("xforms:data", namespaces)
if data_tags:
for dt in data_tags:
dt.attrib["id"] = id
except Exception:
continue
index += 1

# Save the modified XML
newxml = ET.tostring(root)

# write the updated XML file
outxml = open(outfile, "w")
outxml.write(newxml.decode())
outxml.close()

return outfile
248 changes: 0 additions & 248 deletions src/backend/app/projects/project_crud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2491,254 +2491,6 @@ async def convert_geojson_to_osm(geojson_file: str):
return json2osm(geojson_file)


def generate_appuser_files_for_janakpur(
db: Session,
project_id: int,
form: str,
building_extracts_contents: str,
road_extracts_contents: str,
category: str,
form_type: str,
background_task_id: uuid.UUID,
):
project_log = log.bind(task="create_project", project_id=project_id)

project_log.info(f"Starting generate_appuser_files for project {project_id}")

# Get the project table contents.
project = table(
"projects",
column("project_name_prefix"),
column("xform_title"),
column("id"),
column("odkid"),
column("odk_central_url"),
column("odk_central_user"),
column("odk_central_password"),
column("outline"),
)

where = f"id={project_id}"
sql = select(
project.c.project_name_prefix,
project.c.xform_title,
project.c.id,
project.c.odkid,
project.c.odk_central_url,
project.c.odk_central_user,
project.c.odk_central_password,
geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label("outline"),
).where(text(where))
result = db.execute(sql)

# There should only be one match
if result.rowcount != 1:
log.warning(str(sql))
if result.rowcount < 1:
raise HTTPException(status_code=400, detail="Project not found")
else:
raise HTTPException(status_code=400, detail="Multiple projects found")

one = result.first()

if one:
prefix = one.project_name_prefix

# Get odk credentials from project.
odk_credentials = {
"odk_central_url": one.odk_central_url,
"odk_central_user": one.odk_central_user,
"odk_central_password": one.odk_central_password,
}

odk_credentials = project_schemas.ODKCentral(**odk_credentials)

xform_title = one.xform_title if one.xform_title else None

if form:
xlsform = f"/tmp/custom_form.{form_type}"
contents = form
with open(xlsform, "wb") as f:
f.write(contents)
else:
xlsform = f"{xlsforms_path}/{xform_title}.xls"

category = xform_title

# FIXME: Need to figure out this step.
# Data Extracts
if building_extracts_contents is not None:
project_log.info("Uploading data extracts")
upload_custom_data_extracts(db, project_id, building_extracts_contents)

if road_extracts_contents is not None:
project_log.info("Uploading roads data")
upload_custom_data_extracts(
db, project_id, road_extracts_contents, "highways"
)

# Generating QR Code, XForm and uploading OSM Extracts to the form.
# Creating app users and updating the role of that usegenerate_updated_xformr.
tasks_list = tasks_crud.get_task_lists(db, project_id)

project_name = prefix
odk_id = one.odkid
project_obj = get_project(db, project_id)

for task_id in tasks_list:
# Generate taskFiles
name = f"{project_name}_{category}_{task_id}"

appuser = central_crud.create_appuser(odk_id, name, odk_credentials)

# If app user could not be created, raise an exception.
if not appuser:
project_log.error("Couldn't create appuser for project")
return False

# prefix should be sent instead of name
project_log.info(f"Creating qr code for task_id {task_id}")
create_qr = create_qrcode(
db,
odk_id,
appuser.json()["token"],
project_name,
odk_credentials.odk_central_url,
)

task = tasks_crud.get_task(db, task_id)
task.qr_code_id = create_qr["qr_code_id"]
db.commit()
db.refresh(task)

# This file will store xml contents of an xls form.
xform = f"/tmp/{name}.xml"

buildings_extracts = (
f"/tmp/buildings_{name}.geojson" # This file will store osm extracts
)
roads_extracts = (
f"/tmp/roads_{name}.geojson" # This file will store osm extracts
)

# xform_id_format
xform_id = f"{name}".split("_")[2]

# Get the features for this task.
# Postgis query to filter task inside this task outline and of this project
# Update those features and set task_id
query = text(
f"""UPDATE features
SET task_id={task_id}
WHERE id IN (
SELECT id
FROM features
WHERE project_id={project_id}
AND ST_IsValid(geometry)
AND ST_IsValid('{task.outline}'::Geometry)
AND ST_Contains('{task.outline}'::Geometry, ST_Centroid(geometry))
)"""
)
result = db.execute(query)

# Get the geojson of those features for this task.
buildings_query = text(
f"""SELECT jsonb_build_object(
'type', 'FeatureCollection',
'features', jsonb_agg(feature)
)
FROM (
SELECT jsonb_build_object(
'type', 'Feature',
'id', id,
'geometry', ST_AsGeoJSON(geometry)::jsonb,
'properties', properties
) AS feature
FROM features
WHERE project_id={project_id} and task_id={task_id} and category_title='buildings'
) features;"""
)
result = db.execute(buildings_query)
features = result.fetchone()[0]

highway_query = text(
f"""SELECT jsonb_build_object(
'type', 'FeatureCollection',
'features', jsonb_agg(feature)
)
FROM (
SELECT jsonb_build_object(
'type', 'Feature',
'id', id,
'geometry', ST_AsGeoJSON(geometry)::jsonb,
'properties', properties
) AS feature
FROM features
WHERE project_id={project_id} and category_title='highways'
) features;"""
)
highway_result = db.execute(highway_query)
highway_features = highway_result.fetchone()[0]

# upload_media = False if features["features"] is None else True
upload_media = True

# Update outfile containing osm extracts with the new geojson contents containing title in the properties.
with open(buildings_extracts, "w") as jsonfile:
jsonfile.truncate(0) # clear the contents of the file
dump(features, jsonfile)

# Update outfile containing osm extracts with the new geojson contents containing title in the properties.
with open(roads_extracts, "w") as jsonfile:
jsonfile.truncate(0) # clear the contents of the file
dump(highway_features, jsonfile)

project_log.info(f"Generating xform for task {task_id}")
outfile = central_crud.generate_updated_xform_for_janakpur(
xlsform, xform, form_type
)

# Create an odk xform
project_log.info(f"Uploading media in {task_id}")
result = central_crud.create_odk_xform_for_janakpur(
odk_id, task, outfile, odk_credentials, False, upload_media
)

project_log.info(f"Updating role for app user in task {task_id}")
# Update the user role for the created xform.
try:
# Pass odk credentials
if odk_credentials:
url = odk_credentials.odk_central_url
user = odk_credentials.odk_central_user
pw = odk_credentials.odk_central_password

else:
log.debug("ODKCentral connection variables not set in function")
log.debug("Attempting extraction from environment variables")
url = settings.ODK_CENTRAL_URL
user = settings.ODK_CENTRAL_USER
pw = settings.ODK_CENTRAL_PASSWD

odk_app = OdkAppUser(url, user, pw)

odk_app.updateRole(
projectId=odk_id, xform=xform_id, actorId=appuser.json()["id"]
)

except Exception as e:
log.warning(str(e))

project_obj.extract_completed_count += 1
db.commit()
db.refresh(project_obj)

# Update background task status to COMPLETED
update_background_task_status_in_database(
db, background_task_id, 4
) # 4 is COMPLETED


def get_address_from_lat_lon(latitude, longitude):
"""Get address using Nominatim, using lat,lon."""
base_url = "https://nominatim.openstreetmap.org/reverse"
Expand Down
Loading

0 comments on commit 4106a01

Please sign in to comment.