add data_path to support mbtile rasters with an extra data layer

Author: hambsch

server/guppy/db/models.py

@@ -15,6 +15,7 @@ class LayerMetadata(Base):
     layer_name = Column(String, nullable=False, unique=True)
     label = Column(String, nullable=False)
     file_path = Column(String, nullable=False)
+    data_path = Column(String, nullable=False)
     is_rgb = Column(Boolean, nullable=False, default=False, server_default=text('FALSE'))
     is_mbtile = Column(Boolean, nullable=False, default=False, server_default=text('FALSE'))
 

server/guppy/db/schemas.py

@@ -21,6 +21,7 @@ class LayerMetadataSchema(CamelModel):
     layer_name: str
     label: Optional[str] = None
     file_path: str
+    data_path: Optional[str] = None
     is_rgb: Optional[bool] = False
     is_mbtile: Optional[bool] = False
 
@@ -199,6 +200,7 @@ class LayerMetadataBody(CamelModel):
     layer_name: str
     label: str
     file_path: str
+    data_path: Optional[str] = None
     is_rgb: Optional[bool] = False
     is_mbtile: Optional[bool] = False
 

server/guppy/endpoints/endpoint_utils.py

@@ -204,6 +204,8 @@ def validate_layer_and_get_file_path(db: Session, layer_name: str) -> str:
         file_path = layer.file_path
         if not os.path.exists(file_path):
             raise HTTPException(status_code=404, detail=f"File not found: {file_path}")
+        if file_path.endswith('.mbtiles') and os.path.exists(layer.data_path):
+            file_path = layer.data_path
         layer_data_chache[layer_name] = file_path
     return layer_data_chache[layer_name]
 
@@ -243,7 +245,7 @@ def sample_coordinates_window(coords_dict, layer_models, bounds, round_val=None)
 
     """
     result_all = []
-    path = layer_models[0].file_path
+    path = layer_models[0].file_path if not layer_models[0].is_mbtile else layer_models[0].data_path
     coords = []
     for k, v in coords_dict.items():
         coords.extend(v)

server/guppy/endpoints/endpoints.py

@@ -40,7 +40,7 @@ def get_stats_for_bbox(db: Session, layer_name: str, bbox_left: float, bbox_bott
     t = time.time()
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
     if layer_model:
-        path = layer_model.file_path
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and bbox_left and bbox_bottom and bbox_right and bbox_top:
             with rasterio.open(path) as src:
                 target_srs = src.crs.to_epsg()
@@ -74,7 +74,7 @@ def get_data_for_wkt(db: Session, layer_name: str, body: s.GeometryBody):
     t = time.time()
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
     if layer_model:
-        path = layer_model.file_path
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and body:
             geom = wkt.loads(body.geometry)
             with rasterio.open(path) as src:
@@ -107,7 +107,7 @@ def get_stats_for_wkt(db: Session, layer_name: str, body: s.GeometryBody, native
     t = time.time()
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
     if layer_model:
-        path = layer_model.file_path
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and body:
             geom = wkt.loads(body.geometry)
             with rasterio.open(path) as src:
@@ -150,7 +150,7 @@ def get_stats_for_model(layer_model, native, geom, srs):
     Returns:
         The statistics response if successful, otherwise None.
     """
-    path = layer_model.file_path
+    path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
     if os.path.exists(path) and geom:
         with rasterio.open(path) as src:
             target_srs = src.crs.to_epsg()
@@ -205,7 +205,7 @@ def get_line_data_for_wkt(db: Session, layer_name: str, body: s.LineGeometryBody
     t = time.time()
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
     if layer_model:
-        path = layer_model.file_path
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and body:
             line = wkt.loads(body.geometry)
             with rasterio.open(path) as src:
@@ -296,8 +296,9 @@ def get_multi_line_data_list_for_wkt(db: Session, body: s.MultiLineGeometryListB
 def get_point_value_from_layer(db: Session, layer_name: str, x: float, y: float):
     t = time.time()
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
-    if layer_model and not layer_model.is_mbtile:
-        path = layer_model.file_path
+    is_raster = not layer_model.is_mbtile or layer_model.data_path != None
+    if layer_model and is_raster:
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and x and y:
             with rasterio.open(path) as src:
                 nodata = src.nodata
@@ -308,8 +309,8 @@ def get_point_value_from_layer(db: Session, layer_name: str, x: float, y: float)
                     logger.info(f'get_point_value_from_raster 200 {time.time() - t}')
                     return s.PointResponse(type='point value', layer_name=layer_name, value=None if math.isclose(float(v[0]), nodata) else float(v[0]))
         logger.warning(f'file not found {path}')
-    elif layer_model and layer_model.is_mbtile:
-        path = layer_model.file_path
+    elif layer_model:
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and x and y:
             tile_z, tile_x, tile_y = latlon_to_tilexy(x, y, 14)
             tile = get_tile_data(layer_name=layer_name, mb_file=path, z=tile_z, x=tile_x, y=tile_y)
@@ -362,7 +363,8 @@ def get_line_object_list_for_wkt(db: Session, layer_name: str, body: s.LineObjec
         if layer_model.file_path.endswith('.pkl'):
             input_file_df = pd.read_pickle(layer_model.file_path)
         else:
-            input_file_df = gpd.read_file(layer_model.file_path)
+
+            input_file_df = gpd.read_file(layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path)
         input_file_df.to_crs(crs='epsg:3857', inplace=True)
         input_file_df = input_file_df[~pd.isna(input_file_df.geometry)]
         input_file_df = input_file_df[~input_file_df.is_empty]
@@ -391,7 +393,7 @@ def get_classification_for_wkt(db: Session, layer_name: str, body: s.GeometryBod
     t = time.time()
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
     if layer_model:
-        path = layer_model.file_path
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and body:
             geom = wkt.loads(body.geometry)
             with rasterio.open(path) as src:
@@ -433,7 +435,7 @@ def get_combine_layers(db: Session, body: s.CombineLayersGeometryBody):
         geom = wkt.loads(body.geometry)
         layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_item.layer_name).first()
         if layer_model:
-            path = layer_model.file_path
+            path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
             if os.path.exists(path) and body:
                 with rasterio.open(path) as src:
                     target_srs = src.crs.to_epsg()
@@ -465,7 +467,8 @@ def get_combine_layers(db: Session, body: s.CombineLayersGeometryBody):
 
 
 def get_layer_contour(layer):
-    with rasterio.open(layer.file_path) as src:
+    file_path = layer.file_path if not layer.is_mbtile else layer.data_path
+    with rasterio.open(file_path) as src:
         contour_geojson = list(dataset_features(src, bidx=1, as_mask=True, precision=1, band=False, geographic=False))
         return {'layerName': layer.layer_name, 'geometry': contour_geojson}
 
@@ -489,7 +492,7 @@ def get_quantiles_for_wkt(db: Session, layer_name: str, body: s.QuantileBody, na
     layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_name).first()
     if layer_model:
         quantiles = body.quantiles
-        path = layer_model.file_path
+        path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
         if os.path.exists(path) and body:
             geom = wkt.loads(body.geometry)
             with rasterio.open(path) as src:

server/guppy/endpoints/endpoints_admin.py

@@ -27,6 +27,8 @@ def delete_layer_mapping(db: Session, layer_name: str):
     if layer_model:
         if os.path.exists(layer_model.file_path):
             os.remove(layer_model.file_path)
+        if os.path.exists(layer_model.data_path):
+            os.remove(layer_model.data_path)
         db.delete(layer_model)
         db.commit()
         logger.info(f'delete_layer_mapping 200 {time.time() - t}')
@@ -35,7 +37,7 @@ def delete_layer_mapping(db: Session, layer_name: str):
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
-def update_layer_mapping(db: Session, layer_name: str, label: str, file_path: str, is_rgb: bool, is_mbtile: bool):
+def update_layer_mapping(db: Session, layer_name: str, label: str, file_path: str, data_path: str, is_rgb: bool, is_mbtile: bool):
     """
     Updates the mapping of a layer in the database.
 
@@ -55,6 +57,7 @@ def update_layer_mapping(db: Session, layer_name: str, label: str, file_path: st
     if layer_model:
         layer_model.label = label
         layer_model.file_path = file_path
+        layer_model.data_path = data_path
         layer_model.is_rgb = is_rgb
         layer_model.is_mbtile = is_mbtile
         db.commit()
@@ -64,10 +67,9 @@ def update_layer_mapping(db: Session, layer_name: str, label: str, file_path: st
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
-def insert_layer_mapping(db: Session, layer_name: str, label: str, file_path: str, is_rgb: bool, is_mbtile: bool):
+def insert_layer_mapping(db: Session, layer_name: str, label: str, file_path: str, data_path: str, is_rgb: bool, is_mbtile: bool):
     """
     Inserts a layer mapping into the database.
-
     Args:
         db: The session object for the database connection.
         layer_name: The name of the layer.
@@ -80,7 +82,7 @@ def insert_layer_mapping(db: Session, layer_name: str, label: str, file_path: st
 
     """
     t = time.time()
-    layer_model = m.LayerMetadata(layer_name=layer_name, label=label, file_path=file_path, is_rgb=is_rgb, is_mbtile=is_mbtile)
+    layer_model = m.LayerMetadata(layer_name=layer_name, label=label, file_path=file_path, data_path=data_path, is_rgb=is_rgb, is_mbtile=is_mbtile)
     db.add(layer_model)
     db.commit()
     logger.info(f'insert_layer_mapping 201 {time.time() - t}')

server/guppy/endpoints/endpoints_upload.py

@@ -15,18 +15,23 @@
 logger = logging.getLogger(__name__)
 
 
-def upload_file(layer_name: str, label: str, file: UploadFile, db: Session, is_rgb: bool = False, max_zoom: int = 17):
+def upload_file(layer_name: str, label: str, file: UploadFile, data: UploadFile | None, db: Session, is_rgb: bool = False, max_zoom: int = 17):
     """
     Args:
         layer_name (str): The name of the layer.
         label (str): The label of the layer.
         file (UploadFile): The file to upload.
+        data (UploadFile): The data to upload (if the input is a mbtile raster, we need the data geotiff aswell for sampling functions).
         db (Session): The database session.
         is_rgb (bool, optional): Indicates whether the file is in RGB format.
 
     """
-
+    data_location = None
     filename_without_extension, ext = os.path.splitext(file.filename)
+    if data:
+        dataname_without_extension, d_ext = os.path.splitext(data.filename)
+        sanitized_dataname = sanitize_input_str(filename_without_extension)
+        validate_file_input(d_ext, data, dataname_without_extension, layer_name)
 
     validate_file_input(ext, file, filename_without_extension, layer_name)
 
@@ -35,13 +40,16 @@ def upload_file(layer_name: str, label: str, file: UploadFile, db: Session, is_r
 
     check_layer_exists(layer_name=f"{sanitized_layer_name}", db=db)
 
-    file_location, tmp_file_location = create_location_paths_and_check_if_exists(ext, sanitized_filename, sanitized_layer_name)
-
+    file_location, tmp_file_location = create_location_paths_and_check_if_exists(ext, sanitized_filename, sanitized_layer_name, is_raster=True if data else False)
     write_input_file_to_disk(file, tmp_file_location)
 
+    if data:
+        data_location, tmp_data_location = create_location_paths_and_check_if_exists(d_ext, sanitized_dataname, sanitized_layer_name, is_raster=True)
+        write_input_file_to_disk(data, tmp_data_location)
+
     is_mbtile = create_preprocessed_layer_file(ext, file_location, sanitized_filename, sanitized_layer_name, tmp_file_location, max_zoom)
 
-    insert_into_layer_metadata(layer_uuid=sanitized_layer_name, label=label, file_path=file_location, db=db, is_rgb=is_rgb, is_mbtile=is_mbtile)
+    insert_into_layer_metadata(layer_uuid=sanitized_layer_name, label=label, file_path=file_location, data_path=data_location, db=db, is_rgb=is_rgb, is_mbtile=is_mbtile)
     return f"Upload successful: Layer {sanitized_layer_name} uploaded with label {label}."
 
 

server/guppy/endpoints/raster_calc_utils.py

@@ -465,7 +465,7 @@ def fill_path_and_argument_lists(arguments_list, layer_list, db, nodata, path_li
     for layer_item in layer_list:
         layer_model = db.query(m.LayerMetadata).filter_by(layer_name=layer_item.layer_name).first()
         if layer_model:
-            path = layer_model.file_path
+            path = layer_model.file_path if not layer_model.is_mbtile else layer_model.data_path
             path_list.append(path)
             if os.path.exists(path):
                 with rasterio.open(path) as src:

server/guppy/endpoints/upload_utils.py

@@ -157,14 +157,15 @@ def create_preprocessed_layer_file(ext: str, file_location: str, sanitized_filen
     return is_mbtile
 
 
-def create_location_paths_and_check_if_exists(ext: str, sanitized_filename: str, sanitized_layer_name: str) -> tuple[str, str]:
+def create_location_paths_and_check_if_exists(ext: str, sanitized_filename: str, sanitized_layer_name: str, is_raster=False) -> tuple[str, str]:
     """
     Creates file paths for the uploaded file and checks if the file already exists.
 
     Args:
         ext (str): The file extension.
         sanitized_filename (str): The sanitized name of the file.
         sanitized_layer_name (str): The sanitized name of the layer.
+        is_raster (bool): Optional. Indicates whether the file is a raster file. Default is False.
 
     Returns:
         tuple: A tuple containing the file location and temporary file location.
@@ -179,7 +180,11 @@ def create_location_paths_and_check_if_exists(ext: str, sanitized_filename: str,
         if os.path.exists(file_location):
             raise create_error(message=f"Upload failed: File {sanitized_layer_name}_{sanitized_filename}.tif already exists.", code=400)
     elif ext.lower() in ['.mbtiles']:
-        file_location = f"{cfg.deploy.content}/shapefiles/uploaded/{sanitized_layer_name}_{sanitized_filename}.mbtiles"
+        if is_raster:
+            folder = 'tifs'
+        else:
+            folder = 'shapefiles'
+        file_location = f"{cfg.deploy.content}/{folder}/uploaded/{sanitized_layer_name}_{sanitized_filename}.mbtiles"
         tmp_file_location = file_location
         if os.path.exists(file_location):
             raise create_error(message=f"Upload failed: File {sanitized_layer_name}_{sanitized_filename}.mbtiles already exists.", code=400)
@@ -309,7 +314,7 @@ def save_geotif_tiled_overviews(input_file: str, output_file: str, nodata: int)
     return output_file
 
 
-def insert_into_layer_metadata(layer_uuid: str, label: str, file_path: str, db: Session, is_rgb: bool = False, is_mbtile: bool = False):
+def insert_into_layer_metadata(layer_uuid: str, label: str, file_path: str, data_path: str, db: Session, is_rgb: bool = False, is_mbtile: bool = False):
     """
     Inserts a record into the layer_metadata table.
 
@@ -319,7 +324,7 @@ def insert_into_layer_metadata(layer_uuid: str, label: str, file_path: str, db:
         db: The database connection object.
         is_rgb: Optional. Indicates whether the layer is an RGB layer. Default is False.
     """
-    new_layer = LayerMetadata(layer_name=layer_uuid, label=label, file_path=file_path, is_rgb=is_rgb, is_mbtile=is_mbtile)
+    new_layer = LayerMetadata(layer_name=layer_uuid, label=label, file_path=file_path, data_path=data_path, is_rgb=is_rgb, is_mbtile=is_mbtile)
     db.add(new_layer)
     db.commit()
     logger.info("Record inserted into layer metadata")

server/guppy/html/layers.html

@@ -91,6 +91,7 @@ <h1>Layer Metadata</h1>
     <input type="text" id="layer_name" name="layer_name" placeholder="Layer Name" required>
     <input type="text" id="label" name="label" placeholder="Layer Label" required>
     <input type="text" id="file_path" name="file_path" placeholder="File Path" required>
+    <input type="text" id="data_path" name="data_path" placeholder="Data Path" required>
     <label>
         <input type="checkbox" id="is_rgb" name="is_rgb"> Is RGB
     </label>
@@ -110,6 +111,7 @@ <h1>Layer Metadata</h1>
         const layerName = document.getElementById('layer_name').value;
         const label = document.getElementById('label').value;
         const filePath = document.getElementById('file_path').value;
+        const dataPath = document.getElementById('data_path').value;
         const isRgb = document.getElementById('is_rgb').checked;
         const isMbtile = document.getElementById('is_mbtile').checked;
 
@@ -118,7 +120,7 @@ <h1>Layer Metadata</h1>
             headers: {
                 'Content-Type': 'application/json',
             },
-            body: JSON.stringify({layerName, label, filePath, isRgb, isMbtile}),
+            body: JSON.stringify({layerName, label, filePath,dataPath, isRgb, isMbtile}),
         });
 
         if (response.ok) {
@@ -144,6 +146,7 @@ <h1>Layer Metadata</h1>
                 <th>Name</th>
                 <th>Label</th>
                 <th>Path</th>
+                <th>Data Path</th>
                 <th>Is RGB</th>
                 <th>Is MBTile</th>
                 <th>Actions</th>
@@ -159,15 +162,17 @@ <h1>Layer Metadata</h1>
             const nameCell = row.insertCell(0);
             const labelCell = row.insertCell(1);
             const pathCell = row.insertCell(2);
-            const rgbCell = row.insertCell(3);
-            const mbtileCell = row.insertCell(4);
-            const deleteCell = row.insertCell(5); // Cell for the delete button
-            const statsCell = row.insertCell(6); // Cell for the stats button
-            const downloadCell = row.insertCell(7); // Cell for the download button
+            const dataCell = row.insertCell(3);
+            const rgbCell = row.insertCell(4);
+            const mbtileCell = row.insertCell(5);
+            const deleteCell = row.insertCell(6); // Cell for the delete button
+            const statsCell = row.insertCell(7); // Cell for the stats button
+            const downloadCell = row.insertCell(8); // Cell for the download button
 
             nameCell.textContent = layer.layerName;
             labelCell.textContent = layer.effectiveLabel;
             pathCell.textContent = layer.filePath;
+            dataCell.textContent = layer.dataPath;
             rgbCell.textContent = layer.isRgb ? 'Yes' : 'No';
             mbtileCell.textContent = layer.isMbtile ? 'Yes' : 'No';