From c66f04cae6b8f483891dd038987b90872e1d5c42 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari <41701707+nrjadkry@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:24:16 +0545 Subject: [PATCH] feat: cache submissions in s3 (#1035) * api to update submission in cache in s3 * feat: api to download submission from cache * transition from POST to GET for download submission from cache endpoint * added submissions in a zip file as a sozip file in s3 * added sozipfile module * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * feat: postgis func to convert geojson to flatgeobuf * docs: extra comments for geosjon_to_flatgeobuf * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * build: recrease pdm.lock with sozipfile * fix: replace all instances of zipfile with sozipfile * added metadata json file alongside submission.json in s3 bucket * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refactor: rename submissions crud functions for clarity * refactor: simplify update_submission_in_s3 * refactor: reorder submission logic to check for last_submission * check the last submission date from submission in s3 and compare with submission from odk * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix:submission path * update submission download logic with passing task_id as a parameter * remove: download submission from cache endpoint * updated parameter task_id to background_task_id * updated endpoint name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated get data from metadata json * metadata json added outside of zipfile in s3 bucket * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refactor: reuse metadata_s3_path variable in submission upload --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: spwoodcock --- src/backend/app/db/postgis_utils.py | 36 +++++ src/backend/app/projects/project_crud.py | 5 +- src/backend/app/submission/submission_crud.py | 126 +++++++++++++++-- .../app/submission/submission_routes.py | 62 +++++++- src/backend/pdm.lock | 132 ++++++++++-------- src/backend/pyproject.toml | 1 + src/backend/tests/test_projects_routes.py | 2 +- 7 files changed, 282 insertions(+), 82 deletions(-) diff --git a/src/backend/app/db/postgis_utils.py b/src/backend/app/db/postgis_utils.py index d1a2b2bc4b..d3c9dd5e6c 100644 --- a/src/backend/app/db/postgis_utils.py +++ b/src/backend/app/db/postgis_utils.py @@ -21,8 +21,11 @@ from geoalchemy2 import Geometry from geoalchemy2.shape import to_shape +from geojson import FeatureCollection from geojson_pydantic import Feature from shapely.geometry import mapping +from sqlalchemy import text +from sqlalchemy.orm import Session def timestamp(): @@ -61,3 +64,36 @@ def get_centroid(geometry: Geometry, properties: str = {}, id: int = None): } return Feature(**geojson) return {} + + +def geojson_to_flatgeobuf(db: Session, geojson: FeatureCollection): + """From a given FeatureCollection, return a memory flatgeobuf obj.""" + sql = f""" + DROP TABLE IF EXISTS public.temp_features CASCADE; + + CREATE TABLE IF NOT EXISTS public.temp_features( + id serial PRIMARY KEY, + geom geometry + ); + + WITH data AS (SELECT '{geojson}'::json AS fc) + INSERT INTO public.temp_features (geom) + SELECT + ST_AsText(ST_GeomFromGeoJSON(feat->>'geometry')) AS geom + FROM ( + SELECT json_array_elements(fc->'features') AS feat + FROM data + ) AS f; + + WITH thegeom AS + (SELECT * FROM public.temp_features) + SELECT ST_AsFlatGeobuf(thegeom.*) + FROM thegeom; + """ + # Run the SQL + result = db.execute(text(sql)) + # Get a memoryview object, then extract to Bytes + flatgeobuf = result.fetchone()[0].tobytes() + # Cleanup table + db.execute(text("DROP TABLE IF EXISTS public.temp_features CASCADE;")) + return flatgeobuf diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index a90e296baf..aa261ea629 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -22,12 +22,10 @@ import os import time import uuid -import zipfile from asyncio import gather from concurrent.futures import ThreadPoolExecutor, wait from io import BytesIO from typing import List, Optional, Union -from zipfile import ZipFile import geoalchemy2 import geojson @@ -35,6 +33,7 @@ import requests import segno import shapely.wkb as wkblib +import sozipfile.sozipfile as zipfile import sqlalchemy from asgiref.sync import async_to_sync from fastapi import File, HTTPException, UploadFile @@ -764,7 +763,7 @@ async def update_project_with_zip( detail=f"File must be a zip. Uploaded file was {uploaded_zip.content_type}", ) - with ZipFile(io.BytesIO(uploaded_zip.file.read()), "r") as zip: + with zipfile.ZipFile(io.BytesIO(uploaded_zip.file.read()), "r") as zip: # verify valid zip file bad_file = zip.testzip() if bad_file: diff --git a/src/backend/app/submission/submission_crud.py b/src/backend/app/submission/submission_crud.py index dbadecc4b7..94be8e9c15 100644 --- a/src/backend/app/submission/submission_crud.py +++ b/src/backend/app/submission/submission_crud.py @@ -21,10 +21,13 @@ import json import os import threading -import zipfile +import uuid from asyncio import gather +from datetime import datetime +from io import BytesIO from pathlib import Path +import sozipfile.sozipfile as zipfile from asgiref.sync import async_to_sync from fastapi import HTTPException, Response from fastapi.responses import FileResponse @@ -32,8 +35,11 @@ from osm_fieldwork.json2osm import JsonDump from sqlalchemy.orm import Session -from ..central.central_crud import get_odk_form, get_odk_project -from ..projects import project_crud, project_schemas +from app.central.central_crud import get_odk_form, get_odk_project, list_odk_xforms +from app.config import settings +from app.projects import project_crud, project_schemas +from app.s3 import add_obj_to_bucket, get_obj_from_bucket +from app.tasks import tasks_crud def get_submission_of_project(db: Session, project_id: int, task_id: int = None): @@ -300,7 +306,7 @@ def convert_to_osm(db: Session, project_id: int, task_id: int): submission = xform.getSubmissions(odkid, task_id, None, False, True) submission = (json.loads(submission))["value"] else: - submission = get_all_submissions(db, project_id) + submission = get_all_submissions_json(db, project_id) if not submission: raise HTTPException(status_code=404, detail="Submission not found") @@ -342,8 +348,12 @@ def convert_to_osm(db: Session, project_id: int, task_id: int): return FileResponse(final_zip_file_path) -def download_submission_for_project(db, project_id): - log.info(f"Downloading all submissions for a project {project_id}") +def gather_all_submission_csvs(db, project_id): + """Gather all of the submission CSVs for a project. + + Generate a single zip with all submissions. + """ + log.info(f"Downloading all CSV submissions for project {project_id}") get_project_sync = async_to_sync(project_crud.get_project) project_info = get_project_sync(db, project_id) @@ -436,7 +446,103 @@ def extract_files(zip_file_path): return final_zip_file_path -def get_all_submissions(db: Session, project_id): +def update_submission_in_s3( + db: Session, project_id: int, background_task_id: uuid.UUID +): + try: + # Get Project + get_project_sync = async_to_sync(project_crud.get_project) + project = get_project_sync(db, project_id) + + # Gather metadata + odk_credentials = project_schemas.ODKCentral( + odk_central_url=project.odk_central_url, + odk_central_user=project.odk_central_user, + odk_central_password=project.odk_central_password, + ) + odk_forms = list_odk_xforms(project.odkid, odk_credentials, True) + + # Get latest submission date + valid_datetimes = [ + form["lastSubmission"] + for form in odk_forms + if form["lastSubmission"] is not None + ] + last_submission = max( + valid_datetimes, key=lambda x: datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ") + ) + + # Check if the file already exists in s3 + s3_project_path = f"/{project.organisation_id}/{project_id}" + metadata_s3_path = f"/{s3_project_path}/submissions.meta.json" + try: + # Get the last submission date from the metadata + file = get_obj_from_bucket(settings.S3_BUCKET_NAME, metadata_s3_path) + zip_file_last_submission = (json.loads(file.getvalue()))["last_submission"] + if last_submission <= zip_file_last_submission: + # Update background task status to COMPLETED + update_bg_task_sync = async_to_sync( + project_crud.update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 4) # 4 is COMPLETED + return + + except Exception as e: + log.warning(str(e)) + pass + + # Zip file is outdated, regenerate + metadata = { + "last_submission": last_submission, + } + + # Get submissions from ODK Central + submissions = get_all_submissions_json(db, project_id) + + submissions_zip = BytesIO() + # Create a sozipfile with metadata and submissions + with zipfile.ZipFile( + submissions_zip, + "w", + compression=zipfile.ZIP_DEFLATED, + chunk_size=zipfile.SOZIP_DEFAULT_CHUNK_SIZE, + ) as myzip: + myzip.writestr("submissions.json", json.dumps(submissions)) + + # Add zipfile to the s3 bucket + add_obj_to_bucket( + settings.S3_BUCKET_NAME, + submissions_zip, + f"/{s3_project_path}/submission.zip", + ) + + # Upload metadata to s3 + metadata_obj = BytesIO(json.dumps(metadata).encode()) + add_obj_to_bucket( + settings.S3_BUCKET_NAME, + metadata_obj, + metadata_s3_path, + ) + + # Update background task status to COMPLETED + update_bg_task_sync = async_to_sync( + project_crud.update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 4) # 4 is COMPLETED + + return True + + except Exception as e: + log.warning(str(e)) + # Update background task status to FAILED + update_bg_task_sync = async_to_sync( + project_crud.update_background_task_status_in_database + ) + update_bg_task_sync(db, background_task_id, 2, str(e)) # 2 is FAILED + + +def get_all_submissions_json(db: Session, project_id): + """Get all submissions for a project in JSON format.""" get_project_sync = async_to_sync(project_crud.get_project) project_info = get_project_sync(db, project_id) @@ -449,9 +555,9 @@ def get_all_submissions(db: Session, project_id): project = get_odk_project(odk_credentials) - get_task_id_list_sync = async_to_sync(get_task_id_list) - task_lists = get_task_id_list_sync(db, project_id) - submissions = project.getAllSubmissions(project_info.odkid, task_lists) + get_task_id_list_sync = async_to_sync(tasks_crud.get_task_id_list) + task_list = get_task_id_list_sync(db, project_id) + submissions = project.getAllSubmissions(project_info.odkid, task_list) return submissions diff --git a/src/backend/app/submission/submission_routes.py b/src/backend/app/submission/submission_routes.py index d20014c429..f6a1d31283 100644 --- a/src/backend/app/submission/submission_routes.py +++ b/src/backend/app/submission/submission_routes.py @@ -17,16 +17,19 @@ # import json import os +from typing import Optional -from fastapi import APIRouter, Depends, Response +from fastapi import APIRouter, BackgroundTasks, Depends, Response from fastapi.concurrency import run_in_threadpool -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, JSONResponse from osm_fieldwork.odk_merge import OdkMerge from osm_fieldwork.osmfile import OsmFile from sqlalchemy.orm import Session -from ..db import database -from ..projects import project_crud +from app.config import settings +from app.db import database +from app.projects import project_crud, project_schemas + from . import submission_crud router = APIRouter( @@ -104,7 +107,7 @@ async def download_submission( """ if not (task_id or export_json): - file = submission_crud.download_submission_for_project(db, project_id) + file = submission_crud.gather_all_submission_csvs(db, project_id) return FileResponse(file) return await submission_crud.download_submission( @@ -164,7 +167,7 @@ async def conflate_osm_data( # All Submissions JSON # NOTE runs in separate thread using run_in_threadpool submission = await run_in_threadpool( - lambda: submission_crud.get_all_submissions(db, project_id) + lambda: submission_crud.get_all_submissions_json(db, project_id) ) # Data extracta file @@ -214,6 +217,51 @@ async def conflate_osm_data( return [] +@router.post("/download-submission") +async def download_submission_json( + background_tasks: BackgroundTasks, + project_id: int, + background_task_id: Optional[str] = None, + db: Session = Depends(database.get_db), +): + # Get Project + project = await project_crud.get_project(db, project_id) + + # Return existing export if complete + if background_task_id: + # Get the backgrund task status + task_status, task_message = await project_crud.get_background_task_status( + background_task_id, db + ) + + if task_status != 4: + return project_schemas.BackgroundTaskStatus( + status=task_status.name, message=task_message or "" + ) + + bucket_root = f"{settings.S3_DOWNLOAD_ROOT}/{settings.S3_BUCKET_NAME}" + return JSONResponse( + status_code=200, + content=f"{bucket_root}/{project.organisation_id}/{project_id}/submission.zip", + ) + + # Create task in db and return uuid + background_task_id = await project_crud.insert_background_task_into_database( + db, "sync_submission", project_id + ) + + background_tasks.add_task( + submission_crud.update_submission_in_s3, db, project_id, background_task_id + ) + return JSONResponse( + status_code=200, + content={ + "Message": "Submission update process initiated", + "task_id": str(background_task_id), + }, + ) + + @router.get("/get_osm_xml/{project_id}") async def get_osm_xml( project_id: int, @@ -229,7 +277,7 @@ async def get_osm_xml( # All Submissions JSON # NOTE runs in separate thread using run_in_threadpool submission = await run_in_threadpool( - lambda: submission_crud.get_all_submissions(db, project_id) + lambda: submission_crud.get_all_submissions_json(db, project_id) ) # Write the submission to a file diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index 90d9a9422b..e17351c8e0 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -6,7 +6,7 @@ groups = ["default", "debug", "dev", "docs", "test"] cross_platform = true static_urls = false lock_version = "4.3" -content_hash = "sha256:2025602908bcd27e4e1952cb194dcd3f2d55c631e2af0a0b40382fd23bd6fa5c" +content_hash = "sha256:e038b9453bf68f42caf71d86d0c3c5c6ca8756964eb70a22249c4acfc628a332" [[package]] name = "annotated-types" @@ -601,38 +601,38 @@ files = [ [[package]] name = "greenlet" -version = "3.0.1" +version = "3.0.2" requires_python = ">=3.7" summary = "Lightweight in-process concurrent programming" files = [ - {file = "greenlet-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f89e21afe925fcfa655965ca8ea10f24773a1791400989ff32f467badfe4a064"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28e89e232c7593d33cac35425b58950789962011cc274aa43ef8865f2e11f46d"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8ba29306c5de7717b5761b9ea74f9c72b9e2b834e24aa984da99cbfc70157fd"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19bbdf1cce0346ef7341705d71e2ecf6f41a35c311137f29b8a2dc2341374565"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:599daf06ea59bfedbec564b1692b0166a0045f32b6f0933b0dd4df59a854caf2"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b641161c302efbb860ae6b081f406839a8b7d5573f20a455539823802c655f63"}, - {file = "greenlet-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d57e20ba591727da0c230ab2c3f200ac9d6d333860d85348816e1dca4cc4792e"}, - {file = "greenlet-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5805e71e5b570d490938d55552f5a9e10f477c19400c38bf1d5190d760691846"}, - {file = "greenlet-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:52e93b28db27ae7d208748f45d2db8a7b6a380e0d703f099c949d0f0d80b70e9"}, - {file = "greenlet-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f7bfb769f7efa0eefcd039dd19d843a4fbfbac52f1878b1da2ed5793ec9b1a65"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91e6c7db42638dc45cf2e13c73be16bf83179f7859b07cfc139518941320be96"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1757936efea16e3f03db20efd0cd50a1c86b06734f9f7338a90c4ba85ec2ad5a"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19075157a10055759066854a973b3d1325d964d498a805bb68a1f9af4aaef8ec"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9d21aaa84557d64209af04ff48e0ad5e28c5cca67ce43444e939579d085da72"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2847e5d7beedb8d614186962c3d774d40d3374d580d2cbdab7f184580a39d234"}, - {file = "greenlet-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:97e7ac860d64e2dcba5c5944cfc8fa9ea185cd84061c623536154d5a89237884"}, - {file = "greenlet-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b2c02d2ad98116e914d4f3155ffc905fd0c025d901ead3f6ed07385e19122c94"}, - {file = "greenlet-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:22f79120a24aeeae2b4471c711dcf4f8c736a2bb2fabad2a67ac9a55ea72523c"}, - {file = "greenlet-3.0.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:100f78a29707ca1525ea47388cec8a049405147719f47ebf3895e7509c6446aa"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60d5772e8195f4e9ebf74046a9121bbb90090f6550f81d8956a05387ba139353"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:daa7197b43c707462f06d2c693ffdbb5991cbb8b80b5b984007de431493a319c"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea6b8aa9e08eea388c5f7a276fabb1d4b6b9d6e4ceb12cc477c3d352001768a9"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d11ebbd679e927593978aa44c10fc2092bc454b7d13fdc958d3e9d508aba7d0"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dbd4c177afb8a8d9ba348d925b0b67246147af806f0b104af4d24f144d461cd5"}, - {file = "greenlet-3.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20107edf7c2c3644c67c12205dc60b1bb11d26b2610b276f97d666110d1b511d"}, - {file = "greenlet-3.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8bef097455dea90ffe855286926ae02d8faa335ed8e4067326257cb571fc1445"}, - {file = "greenlet-3.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:b2d3337dcfaa99698aa2377c81c9ca72fcd89c07e7eb62ece3f23a3fe89b2ce4"}, - {file = "greenlet-3.0.1.tar.gz", hash = "sha256:816bd9488a94cba78d93e1abb58000e8266fa9cc2aa9ccdd6eb0696acb24005b"}, + {file = "greenlet-3.0.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9acd8fd67c248b8537953cb3af8787c18a87c33d4dcf6830e410ee1f95a63fd4"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:339c0272a62fac7e602e4e6ec32a64ff9abadc638b72f17f6713556ed011d493"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38878744926cec29b5cc3654ef47f3003f14bfbba7230e3c8492393fe29cc28b"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b3f0497db77cfd034f829678b28267eeeeaf2fc21b3f5041600f7617139e6773"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1a8a08de7f68506a38f9a2ddb26bbd1480689e66d788fcd4b5f77e2d9ecfcc"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89a6f6ddcbef4000cda7e205c4c20d319488ff03db961d72d4e73519d2465309"}, + {file = "greenlet-3.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c1f647fe5b94b51488b314c82fdda10a8756d650cee8d3cd29f657c6031bdf73"}, + {file = "greenlet-3.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9560c580c896030ff9c311c603aaf2282234643c90d1dec738a1d93e3e53cd51"}, + {file = "greenlet-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2e9c5423046eec21f6651268cb674dfba97280701e04ef23d312776377313206"}, + {file = "greenlet-3.0.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1fd25dfc5879a82103b3d9e43fa952e3026c221996ff4d32a9c72052544835d"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfdc950dd25f25d6582952e58521bca749cf3eeb7a9bad69237024308c8196"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edf7a1daba1f7c54326291a8cde58da86ab115b78c91d502be8744f0aa8e3ffa"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4cf532bf3c58a862196b06947b1b5cc55503884f9b63bf18582a75228d9950e"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e79fb5a9fb2d0bd3b6573784f5e5adabc0b0566ad3180a028af99523ce8f6138"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:006c1028ac0cfcc4e772980cfe73f5476041c8c91d15d64f52482fc571149d46"}, + {file = "greenlet-3.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fefd5eb2c0b1adffdf2802ff7df45bfe65988b15f6b972706a0e55d451bffaea"}, + {file = "greenlet-3.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c0fdb8142742ee68e97c106eb81e7d3e883cc739d9c5f2b28bc38a7bafeb6d1"}, + {file = "greenlet-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:8f8d14a0a4e8c670fbce633d8b9a1ee175673a695475acd838e372966845f764"}, + {file = "greenlet-3.0.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:654b84c9527182036747938b81938f1d03fb8321377510bc1854a9370418ab66"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd5bc4fde0842ff2b9cf33382ad0b4db91c2582db836793d58d174c569637144"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c27b142a9080bdd5869a2fa7ebf407b3c0b24bd812db925de90e9afe3c417fd6"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0df7eed98ea23b20e9db64d46eb05671ba33147df9405330695bcd81a73bb0c9"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb5d60805057d8948065338be6320d35e26b0a72f45db392eb32b70dd6dc9227"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0e28f5233d64c693382f66d47c362b72089ebf8ac77df7e12ac705c9fa1163d"}, + {file = "greenlet-3.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e4bfa752b3688d74ab1186e2159779ff4867644d2b1ebf16db14281f0445377"}, + {file = "greenlet-3.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c42bb589e6e9f9d8bdd79f02f044dff020d30c1afa6e84c0b56d1ce8a324553c"}, + {file = "greenlet-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:b2cedf279ca38ef3f4ed0d013a6a84a7fc3d9495a716b84a5fc5ff448965f251"}, + {file = "greenlet-3.0.2.tar.gz", hash = "sha256:1c1129bc47266d83444c85a8e990ae22688cf05fb20d7951fd2866007c2ba9bc"}, ] [[package]] @@ -1293,7 +1293,7 @@ files = [ [[package]] name = "pandas" -version = "2.1.3" +version = "2.1.4" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" dependencies = [ @@ -1305,25 +1305,25 @@ dependencies = [ "tzdata>=2022.1", ] files = [ - {file = "pandas-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:acf08a73b5022b479c1be155d4988b72f3020f308f7a87c527702c5f8966d34f"}, - {file = "pandas-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3cc4469ff0cf9aa3a005870cb49ab8969942b7156e0a46cc3f5abd6b11051dfb"}, - {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35172bff95f598cc5866c047f43c7f4df2c893acd8e10e6653a4b792ed7f19bb"}, - {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dfe0e65a2f3988e940224e2a70932edc964df79f3356e5f2997c7d63e758b4"}, - {file = "pandas-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0296a66200dee556850d99b24c54c7dfa53a3264b1ca6f440e42bad424caea03"}, - {file = "pandas-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:465571472267a2d6e00657900afadbe6097c8e1dc43746917db4dfc862e8863e"}, - {file = "pandas-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04d4c58e1f112a74689da707be31cf689db086949c71828ef5da86727cfe3f82"}, - {file = "pandas-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fa2ad4ff196768ae63a33f8062e6838efed3a319cf938fdf8b95e956c813042"}, - {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4441ac94a2a2613e3982e502ccec3bdedefe871e8cea54b8775992485c5660ef"}, - {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ded6ff28abbf0ea7689f251754d3789e1edb0c4d0d91028f0b980598418a58"}, - {file = "pandas-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca5680368a5139d4920ae3dc993eb5106d49f814ff24018b64d8850a52c6ed2"}, - {file = "pandas-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:de21e12bf1511190fc1e9ebc067f14ca09fccfb189a813b38d63211d54832f5f"}, - {file = "pandas-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a5d53c725832e5f1645e7674989f4c106e4b7249c1d57549023ed5462d73b140"}, - {file = "pandas-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cf4cf26042476e39394f1f86868d25b265ff787c9b2f0d367280f11afbdee6d"}, - {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72c84ec1b1d8e5efcbff5312abe92bfb9d5b558f11e0cf077f5496c4f4a3c99e"}, - {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f539e113739a3e0cc15176bf1231a553db0239bfa47a2c870283fd93ba4f683"}, - {file = "pandas-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fc77309da3b55732059e484a1efc0897f6149183c522390772d3561f9bf96c00"}, - {file = "pandas-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:08637041279b8981a062899da0ef47828df52a1838204d2b3761fbd3e9fcb549"}, - {file = "pandas-2.1.3.tar.gz", hash = "sha256:22929f84bca106921917eb73c1521317ddd0a4c71b395bcf767a106e3494209f"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, ] [[package]] @@ -1338,12 +1338,12 @@ files = [ [[package]] name = "pathspec" -version = "0.11.2" -requires_python = ">=3.7" +version = "0.12.1" +requires_python = ">=3.8" summary = "Utility library for gitignore style pattern matching of file paths." files = [ - {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, - {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] [[package]] @@ -1389,8 +1389,8 @@ files = [ [[package]] name = "pre-commit" -version = "3.5.0" -requires_python = ">=3.8" +version = "3.6.0" +requires_python = ">=3.9" summary = "A framework for managing and maintaining multi-language pre-commit hooks." dependencies = [ "cfgv>=2.0.0", @@ -1400,8 +1400,8 @@ dependencies = [ "virtualenv>=20.10.0", ] files = [ - {file = "pre_commit-3.5.0-py2.py3-none-any.whl", hash = "sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660"}, - {file = "pre_commit-3.5.0.tar.gz", hash = "sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32"}, + {file = "pre_commit-3.6.0-py2.py3-none-any.whl", hash = "sha256:c255039ef399049a5544b6ce13d135caba8f2c28c3b4033277a788f434308376"}, + {file = "pre_commit-3.6.0.tar.gz", hash = "sha256:d30bad9abf165f7785c15a21a1f46da7d0677cb00ee7ff4c579fd38922efe15d"}, ] [[package]] @@ -2141,6 +2141,16 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "sozipfile" +version = "0.3.2" +requires_python = ">=3.8" +summary = "Fork of Python zipfile module, adding generation of sozip optimization" +files = [ + {file = "sozipfile-0.3.2-py3-none-any.whl", hash = "sha256:ee7f7e7ddd0ff49543084883958961620d313fb91aac3e021d756c576d5c3825"}, + {file = "sozipfile-0.3.2.tar.gz", hash = "sha256:a06a981ac3ca5389a6e88fdd3633285f012918fc083f50b680d539ce460fa2ad"}, +] + [[package]] name = "sqlalchemy" version = "2.0.23" @@ -2274,12 +2284,12 @@ files = [ [[package]] name = "typing-extensions" -version = "4.8.0" +version = "4.9.0" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] [[package]] diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index b3624536cf..2c628de614 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "minio==7.2.0", "pyproj==3.6.1", "asgiref==3.7.2", + "sozipfile==0.3.2", "osm-login-python==1.0.1", "osm-fieldwork==0.4.0", "osm-rawdata==0.1.7", diff --git a/src/backend/tests/test_projects_routes.py b/src/backend/tests/test_projects_routes.py index 42ff028e3f..482be458ad 100644 --- a/src/backend/tests/test_projects_routes.py +++ b/src/backend/tests/test_projects_routes.py @@ -21,10 +21,10 @@ import json import os import uuid -import zipfile from unittest.mock import Mock, patch import pytest +import sozipfile.sozipfile as zipfile from fastapi.concurrency import run_in_threadpool from geoalchemy2.elements import WKBElement from loguru import logger as log