Skip to content

Commit b1ee5ea

Browse files
tgrandjetgrandje
committed
init demoday
* note : rename `vectorfile_format` argument into `file_format` in order to reuse the same functions for dataframe and geodataframe
1 parent 7c5b55f commit b1ee5ea

21 files changed

+979
-530
lines changed

‎README.qmd

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ normandie = cartiflette.s3.download_vectorfile_url_all(
4141
crs = 4326,
4242
values = "11",
4343
borders="COMMUNE",
44-
vectorfile_format="topojson",
44+
file_format="topojson",
4545
filter_by="REGION",
4646
source="EXPRESS-COG-CARTO-TERRITOIRE",
4747
year=2022)
@@ -57,7 +57,7 @@ regions = cartiflette.s3.download_vectorfile_url_all(
5757
crs = 4326,
5858
values = ["11","27","28"],
5959
borders="COMMUNE",
60-
vectorfile_format="topojson",
60+
file_format="topojson",
6161
filter_by="REGION",
6262
source="EXPRESS-COG-CARTO-TERRITOIRE",
6363
year=2022)
@@ -71,7 +71,7 @@ regions = cartiflette.s3.download_vectorfile_url_all(
7171
crs = 4326,
7272
values = ["11","27","28"],
7373
borders="COMMUNE_ARRONDISSEMENT",
74-
vectorfile_format="topojson",
74+
file_format="topojson",
7575
filter_by="DEPARTEMENT",
7676
source="EXPRESS-COG-CARTO-TERRITOIRE",
7777
year=2022)

‎cartiflette/config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
load_dotenv()
77

88
BUCKET = "projet-cartiflette"
9-
PATH_WITHIN_BUCKET = "diffusion/shapefiles-test4"
9+
PATH_WITHIN_BUCKET = "diffusion/shapefiles-test-demoday"
1010
ENDPOINT_URL = "https://minio.lab.sspcloud.fr"
1111

1212
kwargs = {}

‎cartiflette/download/dataset.py

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,15 +133,26 @@ def _get_last_md5(self) -> None:
133133
self.md5 = md5
134134
except Exception as e:
135135
logger.debug(e)
136-
logger.debug("file not referenced in md5 json")
136+
logger.info(f"{self} not referenced in md5 json")
137137

138138
@pebble.synchronized
139139
def update_json_md5(self, md5: str) -> bool:
140140
"Mise à jour du json des md5"
141+
logger.info(f"updating {self} md5 with {md5} value")
141142
md5 = {
142-
self.provider: {
143-
self.dataset_family: {
144-
self.source: {self.territory: {str(self.year): md5}}
143+
self.provider
144+
if self.provider
145+
else "null": {
146+
self.dataset_family
147+
if self.dataset_family
148+
else "null": {
149+
self.source
150+
if self.source
151+
else "null": {
152+
self.territory
153+
if self.territory
154+
else "null": {str(self.year): md5}
155+
}
145156
}
146157
}
147158
}

‎cartiflette/download/download.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,13 @@ def _upload_raw_dataset_to_s3(
6060
6161
Ex: {
6262
'CHEF_LIEU': [
63-
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
63+
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
6464
],
6565
'COMMUNE': [
66-
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
66+
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
6767
],
6868
'ARRONDISSEMENT': [
69-
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
69+
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
7070
]
7171
}
7272
@@ -93,7 +93,7 @@ def _upload_raw_dataset_to_s3(
9393
"crs": layer.crs,
9494
"filter_by": None,
9595
"value": None,
96-
"vectorfile_format": layer.format,
96+
"file_format": layer.format,
9797
"provider": layer.provider,
9898
"dataset_family": layer.dataset_family,
9999
"source": layer.source,
@@ -201,13 +201,13 @@ def _download_sources(
201201
'downloaded': True,
202202
'paths': {
203203
'CHEF_LIEU': [
204-
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
204+
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/CHEF_LIEU.shp'
205205
],
206206
'COMMUNE': [
207-
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
207+
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=martinique/COMMUNE.shp'
208208
],
209209
'ARRONDISSEMENT': [
210-
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/vectorfile_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
210+
'projet-cartiflette/diffusion/shapefiles-test4/year=2017/administrative_level=None/crs=4326/None=None/file_format=shp/provider=IGN/dataset_family=BDTOPO/source=ROOT/territory=metropole/ARRONDISSEMENT.shp'
211211
]
212212
}
213213
}

0 commit comments

Comments
 (0)