Skip to content

Commit

Permalink
Preprocessing pipeline
Browse files Browse the repository at this point in the history
* Change defaults & usage filter_by/value in get_vectorfile_ign
* Add simili pipeline for preprocessing datasets (national AdminExpress with/without urban districts, living areas
* Store preprocessed datasets on MinIO
  • Loading branch information
tgrandje committed Oct 22, 2023
1 parent ff970f9 commit 21caa2a
Show file tree
Hide file tree
Showing 3 changed files with 371 additions and 267 deletions.
6 changes: 4 additions & 2 deletions cartiflette/public/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ def get_vectorfile_ign(
territory: str = "*",
borders: str = "COMMUNE",
crs: str = "*",
filter_by: str = "origin",
value: str = "raw",
bucket=cartiflette.BUCKET,
path_within_bucket=cartiflette.PATH_WITHIN_BUCKET,
type_download: str = "https",
Expand Down Expand Up @@ -237,8 +239,8 @@ def get_vectorfile_ign(
"year": year,
"borders": borders,
"crs": crs,
"filter_by": None,
"value": None,
"filter_by": filter_by,
"value": value,
"provider": "cartiflette",
"dataset_family": dataset_family,
"source": source,
Expand Down
5 changes: 0 additions & 5 deletions cartiflette/public/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,6 @@ def download_file_single(
)

with tempfile.TemporaryDirectory() as tdir:
print("=" * 50)
print(tdir)
if type_download == "bucket":
try:
if not fs.exists(url):
Expand All @@ -161,9 +159,6 @@ def download_file_single(

else:
local_path = f"{tdir}/{os.path.basename(url)}"
print("-" * 50)
print(url)
print(local_path)
fs.download(url, local_path)

else:
Expand Down
Loading

0 comments on commit 21caa2a

Please sign in to comment.