From dd2f342300936627d62c56bc555403a72aa40770 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 15 May 2024 11:37:03 +0200 Subject: [PATCH 01/68] skeleton fixed removed the double "save and remove lines" keep the best candidates for extremities --- skeleton.py | 319 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 skeleton.py diff --git a/skeleton.py b/skeleton.py new file mode 100644 index 00000000..0397559d --- /dev/null +++ b/skeleton.py @@ -0,0 +1,319 @@ +from itertools import product + +import geopandas as gpd +from geopandas.geodataframe import GeoDataFrame +import pandas as pd +import numpy as np + +from shapely import LineString +from shapely.geometry import CAP_STYLE +from shapely.validation import make_valid +from shapely.ops import voronoi_diagram, linemerge + +import logging + +MASK_GEOJSON_1 = "/home/MDaab/data/squelette_test/MaskHydro_merge 1.geojson" +MASK_GEOJSON_2 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm2.geojson" +MASK_GEOJSON_3 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm3.geojson" +MASK_GEOJSON_4 = "/home/MDaab/data/squelette_test/ecoulement_brut_lg1.geojson" + +MIDDLE_FILE = "squelette_filtrer.geojson" +SAVE_FILE = "squelette_hydrographique.geojson" +WATER_MIN_SIZE = 20 + + + +# def simplify_geometry(polygon): +# """ simplifier certaines formes pour faciliter les calculs +# """ +# ## Buffer positif de la moitié de la taille de la résolution du masque HYDRO (1m de résolution) +# _geom = polygon.buffer(1, cap_style=CAP_STYLE.square) +# geom = _geom.buffer(-0.5, cap_style=CAP_STYLE.square) +# return geom + + +def fix_invalid_geometry(geometry): + """ Fixer les géoémtries invalides + """ + if not geometry.is_valid: + return make_valid(geometry) + else: + return geometry + + +def check_geometry(initial_gdf): + """Vérifier la topologie des polygones + """ + # Obtenir des géométries simples + gdf_simple = initial_gdf.explode(ignore_index=True) # Le paramètre ignore_index est utilisé pour obtenir une incrémentation d'index mise à jour. + # Supprimer les géométries en double si il y en a + gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) + # Identifier les géométries invalides et ne garder que les géométries valides + gdf_valid = gdf_without_duplicates.copy() + gdf_valid.geometry = gdf_valid.geometry.apply( + lambda geom: fix_invalid_geometry(geom) + ) + return gdf_valid + + +def filter_branch(df, crs): + """Filter branches from voronoi_lines to extract hydrographic section + + Args: + -df (GeoDataframe) : Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique + - crs (str): code EPSG + + Returns : + out (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique avec leurs occurences + """ + ### Étape 1 : Obtenez les limites de chaque ligne + boundaries = df.boundary + # df.reset_index() + df["id"] = df.index + + ### Étape 2 : Explosez les limites pour avoir une ligne par limite + exploded_boundaries = boundaries.explode(index_parts=False, ignore_index=False) + # Créez un DataFrame à partir des limites explosées avec une colonne 'geometry' + boundary_df = gpd.GeoDataFrame(geometry=exploded_boundaries) + + ### Étape 3 : Utilisez groupby pour compter le nombre d'occurrences de chaque points + boundary_df["id"] = boundary_df.index + boundary_df["wkt"] = boundary_df["geometry"].to_wkt() + occurences_df = boundary_df.groupby(["geometry"], dropna=True, as_index=False).agg({'id': [lambda x: x.iloc[0], 'count']}) + occurences_df.columns = ['geometry', 'id', 'occurences'] + + ### Étape 4 : Obtenir les occurences de chaque lignes de Voronoi + # Merger les points par occurences avec leur geometry + points_merge = pd.merge(boundary_df, occurences_df, on='geometry').drop(columns=['wkt', 'id_y']).drop_duplicates() + points_merge.columns = ['geometry', 'id', 'occurences'] + points_merge = gpd.GeoDataFrame(points_merge, crs=crs) + # Intersecter les points avec les lignes de Voronoi + points_intersect_lines = df.sjoin(points_merge, predicate='intersects') + # Grouper les lignes par index "id_left" et calculer les occurences "minimum" et "maximum" + out = points_intersect_lines.groupby('id_left', dropna=True).agg({'occurences': ['min'], 'geometry': [lambda x: x.iloc[0]]}) + out.columns = ['occurence_min', 'geometry'] + out = gpd.GeoDataFrame(out, crs=crs) + + return out + + +def can_line_be_removed(line, vertices_dict): + if line.length > WATER_MIN_SIZE: + return False + + point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] + if len(vertices_dict[point_a]) > 1 and len(vertices_dict[point_b]) > 1: + return False + return True + +def remove_extra_lines(lines:GeoDataFrame): + + # prepare a vertice dict, containing all the lines joining on a same vertex + vertices_dict = {} + for index in lines.index : + line = lines.iloc[index]['geometry'] + point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] + try : + vertices_dict[point_a].append(line) # ATTENTION (les points peuvent peut-être être légèrement différents) + except KeyError: + vertices_dict[point_a] = [line] + try : + vertices_dict[point_b].append(line) + except KeyError: + vertices_dict[point_b] = [line] + + # get all the lines to remove from the list + lines_to_remove = [] + for vertex, line_list in vertices_dict.items(): + # print(len(line_list)) + if len(line_list) == 1 : + continue + + # if len(line_list) != 1, then it's probably 3 ; Can't be 2 because of the previous merge, very unlikely to be 4+ + lines_to_keep = [] + lines_that_can_be_removed = [] + for line in line_list: + if can_line_be_removed(line, vertices_dict): + lines_that_can_be_removed.append(line) + else: + lines_to_keep.append(line) + + # if no line can be removed, we continue + if not lines_that_can_be_removed: + continue + + # strange case where 3+ lines are together, but isolated from the rest of the world + if len(lines_to_keep) == 0: + lines_to_remove += lines_that_can_be_removed + continue + + # there is only 1 line that can be removed and 2+ to be kept, so we remove it + if len(lines_that_can_be_removed) == 1: + lines_to_remove.append(lines_that_can_be_removed[0]) + continue + + # at least 2 lines can be removed, we will keep the one forming the straightest line with + # a line to keep, we assume it's the most likely to "continue" the river + min_dot_product = 1 + line_that_should_not_be_removed = None + for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): + # if vertex == line_to_keep.boundary.geoms[0]: + # vector_to_keep = np.array((line_to_keep.boundary.geoms[1] - vertex).coords[0]) + # else: + # vector_to_keep = np.array((line_to_keep.boundary.geoms[0] - vertex).coords[0]) + + # if vertex == line_that_can_be_removed.boundary.geoms[0]: + # vector_to_remove = np.array((line_that_can_be_removed.boundary.geoms[1] - vertex).coords[0]) + # else: + # vector_to_remove = np.array((line_that_can_be_removed.boundary.geoms[0] - vertex).coords[0]) + + if vertex == line_to_keep.boundary.geoms[0]: + vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + else: + vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + + if vertex == line_that_can_be_removed.boundary.geoms[0]: + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + else: + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + + length_to_keep = np.linalg.norm(vector_to_keep) + length_to_remove = np.linalg.norm(vector_to_remove) + if not length_to_keep or not length_to_remove: # length = 0, probably won't happen + continue + + vector_to_keep = vector_to_keep / length_to_keep + vector_to_remove = vector_to_remove / length_to_remove + dot_product = np.dot(vector_to_keep, vector_to_remove) + # the smallest dot product (which should be negative) is the straightest couple of lines + if dot_product < min_dot_product: + min_dot_product = dot_product + line_that_should_not_be_removed = line_that_can_be_removed + + lines_that_can_be_removed.remove(line_that_should_not_be_removed) + lines_to_remove += lines_that_can_be_removed + + # return the lines without the lines to remove + return lines[~lines['geometry'].isin(lines_to_remove)] + +def line_merge(voronoi_lines): + """Fusionner tous les LINESTRING en un seul objet MultiLineString + + Args: + - voronoi_lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique + + Returns: + + lines (GeoDataframe): Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique + + """ + # Fusionner tous les LINESTRING en un seul objet MultiLineString + merged_line = voronoi_lines.geometry.unary_union + # Appliquer un algo de fusion des lignes (ex. "ST_LineMerge") sur l'objet MultiLineString + merged_line = linemerge(merged_line) + + # turn multipart geometries into multiple single geometries + geometry = gpd.GeoSeries(merged_line.geoms, crs=crs).explode(index_parts=False) + lines = gpd.GeoDataFrame(geometry=geometry, crs=crs) + + return lines + + +def create_voronoi_lines(mask_hydro, crs): + """ Créer les lignes de Voronoi + + Args: + -mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) + - crs (str): code EPSG + + Returns : + out lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique + """ + # Vérfier quye la Geometrie n'est pas vide + if not mask_hydro['geometry'].dtype == 'geometry': + return + + ###### Calculer le diagram de Voronoi ###### + # "Segmentize" le masque HYDRO : + # Renvoie une geometry/geography modifiée dont aucun segment n'est plus long que max_segment_length. + segmentize_geom = (mask_hydro['geometry'].unary_union).segmentize(max_segment_length=2) + # Calculer le diagrame de Voronoi, et ne renvoyer que les polylignes + regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) + + ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### + geometry = gpd.GeoSeries(regions.geoms, crs=crs).explode(index_parts=False) + df = gpd.GeoDataFrame(geometry=geometry, crs=crs) + lines_filter = df.sjoin(mask_hydro, predicate='within') # Opération spatiale "Within" + # Enregistrer le diagram de Voronoi (Squelette) + lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index + lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' + + return gpd.GeoDataFrame(lines_filter, crs=crs) + + +def run(mask_hydro, crs): + """Calculer le squelette hydrographique + + Args: + - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) + - crs (str): code EPSG + """ + gdf = check_geometry(mask_hydro) # Vérifier et corriger les géométries non valides + ### Diagram Voronoi ### + voronoi_lines = create_voronoi_lines(gdf, crs) + + ### Mettre une occurence au squelette pour obtenir un tronçon hydrographique ### + # Fusionner tous les LINESTRING en un seul objet MultiLineString + lines = line_merge(voronoi_lines) + old_number_of_lines = len(lines) + while True: + lines = remove_extra_lines(lines) + lines = line_merge(lines) + + if len(lines) != old_number_of_lines: + old_number_of_lines = len(lines) + else: + break + + # pruned_lines = remove_extra_lines(lines) + # merged_pruned_lines = line_merge(pruned_lines) + # super_pruned_lines = remove_extra_lines(merged_pruned_lines) + gpd.GeoDataFrame(lines, crs=crs).to_file("test_5.geojson", driver='GeoJSON') + + + + + # gdf = filter_branch(lines, crs) + + # gdf.to_file(MIDDLE_FILE, driver='GeoJSON') # sauvegarder le résultat + # ## Filtrer les occurences ## + # # Filtrer les lignes pour ne conserver que celles dont les "occurences > 1" + # filtered_gdf = gdf.query('occurence_min > 1') + # filtered_gdf.drop(columns=['occurence_min']) + # # Merge lines pour des occurences > 1 + # lines_occurences = line_merge(filtered_gdf) + # ## 2e calcul des occurences filtrées ## + # gdf2 = filter_branch(lines_occurences, crs) + # ## Filtrer les occurences ## + # # Filtrer les lignes pour ne conserver que celles dont les "occurences > 1" + # filtered_gdf2 = gdf2.query('occurence_min > 1') + # # print(len(filtered_gdf)*0.1) + # # print(len(filtered_gdf2)) + # # Eviter les squelettes discontinues + # if len(filtered_gdf2) < len(filtered_gdf)*0.1: + # gdf2.to_file(SAVE_FILE, driver='GeoJSON') # sauvegarder le résultat + # else: + # filtered_gdf2.to_file(SAVE_FILE, driver='GeoJSON') # sauvegarder le résultat + + +if __name__ == "__main__": + # Example usage + # Calcule du squelette + mask_hydro = gpd.read_file(MASK_GEOJSON_1) + crs = mask_hydro.crs # Load a crs from input + # Simplifier geometrie + # buffer_geom = mask_hydro.apply(simplify_geometry) ## Appliquer un buffer + simplify_geom = mask_hydro.simplify(tolerance=2, preserve_topology=True) # simplifier les géométries avec Douglas-Peucker + df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) + # Squelette + run(df, crs) From 271836ccaaf9e9376dd00ca4b99cf08f325f2de8 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 21 May 2024 10:49:48 +0200 Subject: [PATCH 02/68] before pruning lines --- skeleton.py | 223 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 86 deletions(-) diff --git a/skeleton.py b/skeleton.py index 0397559d..db9fe63f 100644 --- a/skeleton.py +++ b/skeleton.py @@ -1,15 +1,15 @@ from itertools import product +from typing import Dict, List, Tuple import geopandas as gpd from geopandas.geodataframe import GeoDataFrame import pandas as pd import numpy as np -from shapely import LineString +from shapely import LineString, Point, geometry from shapely.geometry import CAP_STYLE from shapely.validation import make_valid -from shapely.ops import voronoi_diagram, linemerge - +from shapely.ops import voronoi_diagram, linemerge, split, snap import logging MASK_GEOJSON_1 = "/home/MDaab/data/squelette_test/MaskHydro_merge 1.geojson" @@ -20,20 +20,12 @@ MIDDLE_FILE = "squelette_filtrer.geojson" SAVE_FILE = "squelette_hydrographique.geojson" WATER_MIN_SIZE = 20 - - - -# def simplify_geometry(polygon): -# """ simplifier certaines formes pour faciliter les calculs -# """ -# ## Buffer positif de la moitié de la taille de la résolution du masque HYDRO (1m de résolution) -# _geom = polygon.buffer(1, cap_style=CAP_STYLE.square) -# geom = _geom.buffer(-0.5, cap_style=CAP_STYLE.square) -# return geom +BRIDGE_MAX_WIDTH = 50 +VORONOI_MAX_LENGTH = 2 def fix_invalid_geometry(geometry): - """ Fixer les géoémtries invalides + """ Fixer les géométries invalides """ if not geometry.is_valid: return make_valid(geometry) @@ -42,13 +34,12 @@ def fix_invalid_geometry(geometry): def check_geometry(initial_gdf): - """Vérifier la topologie des polygones + """garanty polygons' validity """ - # Obtenir des géométries simples - gdf_simple = initial_gdf.explode(ignore_index=True) # Le paramètre ignore_index est utilisé pour obtenir une incrémentation d'index mise à jour. - # Supprimer les géométries en double si il y en a + # create simple geometry + gdf_simple = initial_gdf.explode(ignore_index=True) # ignore_index makes the resulting index multi-indexed gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) - # Identifier les géométries invalides et ne garder que les géométries valides + # remove invalid polygons gdf_valid = gdf_without_duplicates.copy() gdf_valid.geometry = gdf_valid.geometry.apply( lambda geom: fix_invalid_geometry(geom) @@ -106,30 +97,38 @@ def can_line_be_removed(line, vertices_dict): return False return True -def remove_extra_lines(lines:GeoDataFrame): - - # prepare a vertice dict, containing all the lines joining on a same vertex +def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: + """ + get a dictionary of vertices listing all the lines having a specific vertex as one of its extremities + Args: + - gdf_lines:geodataframe containing a list of lines + return: + - a dict with the vertices are the keys and the values are lists of lines + """ + # prepare a vertice dict, containing all the lines connected on a same vertex vertices_dict = {} - for index in lines.index : - line = lines.iloc[index]['geometry'] + for index in gdf_lines.index : + line = gdf_lines.iloc[index]['geometry'] point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] try : - vertices_dict[point_a].append(line) # ATTENTION (les points peuvent peut-être être légèrement différents) + vertices_dict[point_a].append(line) except KeyError: vertices_dict[point_a] = [line] try : vertices_dict[point_b].append(line) except KeyError: vertices_dict[point_b] = [line] + return vertices_dict + +def remove_extra_lines(gdf_lines:GeoDataFrame)->GeoDataFrame: + vertices_dict = get_vertices_dict(gdf_lines) - # get all the lines to remove from the list lines_to_remove = [] for vertex, line_list in vertices_dict.items(): - # print(len(line_list)) if len(line_list) == 1 : continue - # if len(line_list) != 1, then it's probably 3 ; Can't be 2 because of the previous merge, very unlikely to be 4+ + # if len(line_list) !is not 1, then it's probably 3; Can't be 2 because of the previous merge, very unlikely to be 4+ lines_to_keep = [] lines_that_can_be_removed = [] for line in line_list: @@ -157,16 +156,6 @@ def remove_extra_lines(lines:GeoDataFrame): min_dot_product = 1 line_that_should_not_be_removed = None for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): - # if vertex == line_to_keep.boundary.geoms[0]: - # vector_to_keep = np.array((line_to_keep.boundary.geoms[1] - vertex).coords[0]) - # else: - # vector_to_keep = np.array((line_to_keep.boundary.geoms[0] - vertex).coords[0]) - - # if vertex == line_that_can_be_removed.boundary.geoms[0]: - # vector_to_remove = np.array((line_that_can_be_removed.boundary.geoms[1] - vertex).coords[0]) - # else: - # vector_to_remove = np.array((line_that_can_be_removed.boundary.geoms[0] - vertex).coords[0]) - if vertex == line_to_keep.boundary.geoms[0]: vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) else: @@ -194,7 +183,7 @@ def remove_extra_lines(lines:GeoDataFrame): lines_to_remove += lines_that_can_be_removed # return the lines without the lines to remove - return lines[~lines['geometry'].isin(lines_to_remove)] + return gdf_lines[~gdf_lines['geometry'].isin(lines_to_remove)] def line_merge(voronoi_lines): """Fusionner tous les LINESTRING en un seul objet MultiLineString @@ -203,7 +192,6 @@ def line_merge(voronoi_lines): - voronoi_lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique Returns: - lines (GeoDataframe): Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique """ @@ -229,15 +217,13 @@ def create_voronoi_lines(mask_hydro, crs): Returns : out lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique """ - # Vérfier quye la Geometrie n'est pas vide + # we don't work on an empty geometry if not mask_hydro['geometry'].dtype == 'geometry': return - - ###### Calculer le diagram de Voronoi ###### - # "Segmentize" le masque HYDRO : - # Renvoie une geometry/geography modifiée dont aucun segment n'est plus long que max_segment_length. - segmentize_geom = (mask_hydro['geometry'].unary_union).segmentize(max_segment_length=2) - # Calculer le diagrame de Voronoi, et ne renvoyer que les polylignes + + # divide geometry into segments no longer than max_segment_length + segmentize_geom = (mask_hydro['geometry'].unary_union).segmentize(max_segment_length=VORONOI_MAX_LENGTH) + # Create the voronoi diagram and only keep polygon regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### @@ -250,6 +236,90 @@ def create_voronoi_lines(mask_hydro, crs): return gpd.GeoDataFrame(lines_filter, crs=crs) +def get_prior_point(vertex, line) -> Tuple[float, float]: + """ return the point right before an extremity vertex in a line + """ + if vertex == line.boundary.geoms[0]: + prior_point = line.coords[2] + else: + prior_point = line.coords[-2] + return prior_point + +def connect_extremities(gdf_lines:GeoDataFrame)->GeoDataFrame: + """ + connect extremities of rivers, as we assume they are disconnected because of bridges + Args: + - gdf_lines: geodataframe containing a list of lines + return: + The same geodataframe with connection between extremities + """ + # garanty that all lines have at least 3 segments, because the connexion is not + # done on the last point, but the previous one. Therefore we need enough points + for index in gdf_lines.index : + line = gdf_lines.iloc[index]['geometry'] + if len(line.coords) < 4: # 3 segments, so 4 points + gdf_lines.loc[index] = line.segmentize(line.length/3) + + vertices_dict = get_vertices_dict(gdf_lines) + # we remove all vertices that are not extremities (an extremity is connected to only one line) + keys_to_remove = [] + for vertex, line_list in vertices_dict.items(): + if len(line_list) != 1: + keys_to_remove.append(vertex) + for vertex in keys_to_remove: + vertices_dict.pop(vertex, None) + + best_candidate = {} + for vertex, [line] in vertices_dict.items(): + prior_point = get_prior_point(vertex, line) + + # get the perpendicular line passing throught vertex, equation ax + by + c = 0 + a, b = vertex.coords[0][0] - prior_point[0], vertex.coords[0][1] - prior_point[1] + c = - a * vertex.coords[0][0] - b * vertex.coords[0][1] + + # make sure the prior_point is below the line (if above, we multiply the equation by -1 to change it) + if not a * prior_point[0] + b * prior_point[1] + c <= 0: + a, b, c = -a, -b, -c + + # search best vertex candidate to join this vertex i.e: + # the closest, not too far, vertex above the perpendicular line (to be "in front" of the river) + for other_vertex, [other_line] in vertices_dict.items(): + if other_vertex == vertex: + continue + # have to be above the perpendicular line + if a * other_vertex.coords[0][0] + b * other_vertex.coords[0][1] + c <= 0: + continue + # should not be too far: + squared_distance = (vertex.coords[0][0] - other_vertex.coords[0][0])**2 + (vertex.coords[0][1] - other_vertex.coords[0][1])**2 + if squared_distance > BRIDGE_MAX_WIDTH**2: + continue + other_prior_point = get_prior_point(other_vertex, other_line) + best_candidate[prior_point] = other_prior_point + + already_done_prior_points= [] + # if 2 prior_points have each other as best candidate, we create a line between them + for prior_point, other_prior_point in best_candidate.items(): + if best_candidate[other_prior_point] == prior_point and other_prior_point not in already_done_prior_points: + already_done_prior_points.append(prior_point) + gdf_lines.loc[len(gdf_lines)] = [LineString([prior_point, other_prior_point])] + return gdf_lines + +def split_into_basic_segments(gdf_lines:GeoDataFrame)->GeoDataFrame: + def split_line_by_point(line, point, tolerance: float=1.0e-12): + return split(snap(line, point, tolerance), point) + result = ( + gdf_lines + .assign(geometry=gdf_lines.apply( + lambda x: split_line_by_point( + x.geometry, + geometry.Point(x.geometry.coords[1]) + ), axis=1 + )) + .explode() + .reset_index(drop=True) + ) + return result + def run(mask_hydro, crs): """Calculer le squelette hydrographique @@ -259,52 +329,31 @@ def run(mask_hydro, crs): - crs (str): code EPSG """ gdf = check_geometry(mask_hydro) # Vérifier et corriger les géométries non valides - ### Diagram Voronoi ### + ### Voronoi Diagram ### voronoi_lines = create_voronoi_lines(gdf, crs) - ### Mettre une occurence au squelette pour obtenir un tronçon hydrographique ### - # Fusionner tous les LINESTRING en un seul objet MultiLineString - lines = line_merge(voronoi_lines) - old_number_of_lines = len(lines) + # merge all lines that can be merged without doubt (when only 2 lines are connected) + # remove lines we don't want, then repeat until it doesn't change anymore + gdf_lines = line_merge(voronoi_lines) + old_number_of_lines = len(gdf_lines) while True: - lines = remove_extra_lines(lines) - lines = line_merge(lines) + gdf_lines = remove_extra_lines(gdf_lines) + gdf_lines = line_merge(gdf_lines) - if len(lines) != old_number_of_lines: - old_number_of_lines = len(lines) + if len(gdf_lines) != old_number_of_lines: + old_number_of_lines = len(gdf_lines) else: break - # pruned_lines = remove_extra_lines(lines) - # merged_pruned_lines = line_merge(pruned_lines) - # super_pruned_lines = remove_extra_lines(merged_pruned_lines) - gpd.GeoDataFrame(lines, crs=crs).to_file("test_5.geojson", driver='GeoJSON') - - - - - # gdf = filter_branch(lines, crs) - - # gdf.to_file(MIDDLE_FILE, driver='GeoJSON') # sauvegarder le résultat - # ## Filtrer les occurences ## - # # Filtrer les lignes pour ne conserver que celles dont les "occurences > 1" - # filtered_gdf = gdf.query('occurence_min > 1') - # filtered_gdf.drop(columns=['occurence_min']) - # # Merge lines pour des occurences > 1 - # lines_occurences = line_merge(filtered_gdf) - # ## 2e calcul des occurences filtrées ## - # gdf2 = filter_branch(lines_occurences, crs) - # ## Filtrer les occurences ## - # # Filtrer les lignes pour ne conserver que celles dont les "occurences > 1" - # filtered_gdf2 = gdf2.query('occurence_min > 1') - # # print(len(filtered_gdf)*0.1) - # # print(len(filtered_gdf2)) - # # Eviter les squelettes discontinues - # if len(filtered_gdf2) < len(filtered_gdf)*0.1: - # gdf2.to_file(SAVE_FILE, driver='GeoJSON') # sauvegarder le résultat - # else: - # filtered_gdf2.to_file(SAVE_FILE, driver='GeoJSON') # sauvegarder le résultat + gdf_lines = connect_extremities(gdf_lines) + gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs + # connecting extremities creates lines we don't want (because of where we connect the extremities), + # we remove them one last time + gdf_lines = remove_extra_lines(gdf_lines) + # gdf_lines = line_merge(gdf_lines) + + gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_7.geojson", driver='GeoJSON') if __name__ == "__main__": # Example usage @@ -315,5 +364,7 @@ def run(mask_hydro, crs): # buffer_geom = mask_hydro.apply(simplify_geometry) ## Appliquer un buffer simplify_geom = mask_hydro.simplify(tolerance=2, preserve_topology=True) # simplifier les géométries avec Douglas-Peucker df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) + df.to_file("test_0.geojson", driver='GeoJSON') # Squelette - run(df, crs) + # run(df, crs) + run(mask_hydro, crs) From 75cc05a62c4985ee4410f48c20644906f67c1ff8 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 27 May 2024 11:40:47 +0200 Subject: [PATCH 03/68] works but too slow --- skeleton.py | 171 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 135 insertions(+), 36 deletions(-) diff --git a/skeleton.py b/skeleton.py index db9fe63f..69e9fdf6 100644 --- a/skeleton.py +++ b/skeleton.py @@ -7,15 +7,16 @@ import numpy as np from shapely import LineString, Point, geometry -from shapely.geometry import CAP_STYLE +from shapely.geometry import CAP_STYLE, Polygon, MultiPoint from shapely.validation import make_valid -from shapely.ops import voronoi_diagram, linemerge, split, snap +from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate import logging MASK_GEOJSON_1 = "/home/MDaab/data/squelette_test/MaskHydro_merge 1.geojson" MASK_GEOJSON_2 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm2.geojson" MASK_GEOJSON_3 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm3.geojson" MASK_GEOJSON_4 = "/home/MDaab/data/squelette_test/ecoulement_brut_lg1.geojson" +MASK_GEOJSON_5 = "/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson" MIDDLE_FILE = "squelette_filtrer.geojson" SAVE_FILE = "squelette_hydrographique.geojson" @@ -97,6 +98,35 @@ def can_line_be_removed(line, vertices_dict): return False return True +def un_loop(loop): + polygon_loop = Polygon(loop.coords) + geometry = gpd.GeoSeries(polygon_loop, crs=2154) + gpd_loop = gpd.GeoDataFrame(geometry=geometry, crs=2154) + gpd_loop.to_file("loop.geojson", driver='GeoJSON') + # new_lines = triangulate(loop, 0, True) + # polygon_loop = MultiPoint(loop.coords) + # new_lines = voronoi_diagram(polygon_loop, envelope=polygon_loop, tolerance=0, edges=True) + + voronoi = create_voronoi_lines(gpd_loop, 2154) + + voronoi.to_file("voronoi.geojson", driver='GeoJSON') + + # coords_combine = loop.coords + # for index in voronoi.index : + # coords_combine = coords_combine + voronoi.iloc[index]['geometry'].coords + + voronoi.loc[len(voronoi.index)] = loop + + # combine_point = MultiPoint(coords_combine) + + # combine_point = MultiPoint(list(loop.coords) + [x.coords for x in voronoi['geometry']]) + triangulate_combine_points = triangulate(voronoi.unary_union, 0, True) + gpd.GeoDataFrame(geometry=triangulate_combine_points, crs=2154).to_file("triangulate_combine.geojson", driver='GeoJSON') + # geometry = gpd.GeoSeries(new_lines, crs=2154).explode(index_parts=False) + # geometry = gpd.GeoSeries(new_lines.geoms, crs=2154) + # gpd.GeoDataFrame(geometry=geometry, crs=2154).to_file("new_lines.geojson", driver='GeoJSON') + pass + def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: """ get a dictionary of vertices listing all the lines having a specific vertex as one of its extremities @@ -109,7 +139,11 @@ def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: vertices_dict = {} for index in gdf_lines.index : line = gdf_lines.iloc[index]['geometry'] + if line.is_ring: # it's a loop : no extremity + continue + point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] + # point_a, point_b = line.coords[0], line.coords[-1] try : vertices_dict[point_a].append(line) except KeyError: @@ -121,6 +155,8 @@ def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: return vertices_dict def remove_extra_lines(gdf_lines:GeoDataFrame)->GeoDataFrame: + + gpd.GeoDataFrame(gdf_lines, crs=2154).to_file("test_ring.geojson", driver='GeoJSON') vertices_dict = get_vertices_dict(gdf_lines) lines_to_remove = [] @@ -233,7 +269,7 @@ def create_voronoi_lines(mask_hydro, crs): # Enregistrer le diagram de Voronoi (Squelette) lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' - + return gpd.GeoDataFrame(lines_filter, crs=crs) def get_prior_point(vertex, line) -> Tuple[float, float]: @@ -245,7 +281,7 @@ def get_prior_point(vertex, line) -> Tuple[float, float]: prior_point = line.coords[-2] return prior_point -def connect_extremities(gdf_lines:GeoDataFrame)->GeoDataFrame: +def connect_extremities(gdf_lines:GeoDataFrame, crs)->GeoDataFrame: """ connect extremities of rivers, as we assume they are disconnected because of bridges Args: @@ -255,7 +291,7 @@ def connect_extremities(gdf_lines:GeoDataFrame)->GeoDataFrame: """ # garanty that all lines have at least 3 segments, because the connexion is not # done on the last point, but the previous one. Therefore we need enough points - for index in gdf_lines.index : + for index in gdf_lines.index: line = gdf_lines.iloc[index]['geometry'] if len(line.coords) < 4: # 3 segments, so 4 points gdf_lines.loc[index] = line.segmentize(line.length/3) @@ -269,6 +305,7 @@ def connect_extremities(gdf_lines:GeoDataFrame)->GeoDataFrame: for vertex in keys_to_remove: vertices_dict.pop(vertex, None) + best_candidate = {} for vertex, [line] in vertices_dict.items(): prior_point = get_prior_point(vertex, line) @@ -296,30 +333,82 @@ def connect_extremities(gdf_lines:GeoDataFrame)->GeoDataFrame: other_prior_point = get_prior_point(other_vertex, other_line) best_candidate[prior_point] = other_prior_point - already_done_prior_points= [] + new_lines = [] + already_done_prior_points = set() # if 2 prior_points have each other as best candidate, we create a line between them for prior_point, other_prior_point in best_candidate.items(): if best_candidate[other_prior_point] == prior_point and other_prior_point not in already_done_prior_points: - already_done_prior_points.append(prior_point) - gdf_lines.loc[len(gdf_lines)] = [LineString([prior_point, other_prior_point])] - return gdf_lines - -def split_into_basic_segments(gdf_lines:GeoDataFrame)->GeoDataFrame: - def split_line_by_point(line, point, tolerance: float=1.0e-12): - return split(snap(line, point, tolerance), point) - result = ( - gdf_lines - .assign(geometry=gdf_lines.apply( - lambda x: split_line_by_point( - x.geometry, - geometry.Point(x.geometry.coords[1]) - ), axis=1 - )) - .explode() - .reset_index(drop=True) - ) - return result + already_done_prior_points.add(prior_point) + already_done_prior_points.add(other_prior_point) + # gdf_lines.loc[len(gdf_lines)] = [LineString([prior_point, other_prior_point])] + new_lines.append(LineString([prior_point, other_prior_point])) + for index in gdf_lines.index: + line = gdf_lines.iloc[index]['geometry'] + start = 0 + end = len(line.coords) + if line.coords[2] in already_done_prior_points: + start = 2 + # new_lines.append(LineString(line.coords[0:2])) + if line.coords[-2] in already_done_prior_points: + end = -2 + # new_lines.append(LineString(line.coords[-2:])) + new_lines.append(LineString(line.coords[start:end])) + + # gdf_cutlines = gpd.GeoDataFrame(geometry=cut_lines) + + return gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) + # gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs + # return + +def get_lines_partially_on_loops(gdf_lines:GeoDataFrame, gdf_divided_lines:GeoDataFrame, crs:int)->GeoDataFrame: + # get all loops + gdf_loops = gdf_lines[gdf_lines['geometry'].is_ring].reset_index() + + # gets all points on loops + points_on_loops = [] + for index in gdf_loops.index: + points_on_loops += list(gdf_loops.iloc[index]['geometry'].coords) + mp = MultiPoint(points_on_loops) + + # get lines with at least 1 point on a loop + commun_lines = gdf_divided_lines[~gdf_divided_lines['geometry'].disjoint(mp)].reset_index() + + # get lines with one point, and one point only, on loops + partially_on_loop_lines = [] + for index in commun_lines.index: + coords = commun_lines.iloc[index]['geometry'].coords + completely_covered = False + for index_loop in gdf_loops.index: + for begin_loop, end_loop in zip(gdf_loops.iloc[index_loop]['geometry'].coords[:-1], gdf_loops.iloc[index_loop]['geometry'].coords[1:]): + if (begin_loop in coords) and (end_loop in coords): + completely_covered = True + break + if completely_covered: + break + if not completely_covered: + partially_on_loop_lines.append(commun_lines.iloc[index]['geometry']) + + return gpd.GeoDataFrame(geometry=partially_on_loop_lines, crs=crs) + # gdf_connected_lines.to_file("test_connected_lines.geojson", driver='GeoJSON') + + # assembled = gpd.GeoDataFrame(pd.concat([gdf_connected_lines, gdf_lines], ignore_index=True)) + # assembled.to_file("test_assembled.geojson", driver='GeoJSON') # premerge + + # test = commun_lines[any(commun_lines['geometry'].covered_by(gdf_lines))] + # new_lines_excluded = [] + + # test = gdf_lines.unary_union + + # for index in commun_lines.index: + # if not any(commun_lines.iloc[index]['geometry'].covered_by(gdf_lines)): + # new_lines_excluded.append(commun_lines.iloc[index]['geometry']) + + + # geometry = gpd.GeoSeries(regions.geoms, crs=crs).explode(index_parts=False) + # df = gpd.GeoDataFrame(geometry=geometry, crs=crs) + + # gpd.GeoDataFrame(new_lines_excluded, crs=2154).to_file("test_new_lines_excluded.geojson", driver='GeoJSON') def run(mask_hydro, crs): """Calculer le squelette hydrographique @@ -331,40 +420,50 @@ def run(mask_hydro, crs): gdf = check_geometry(mask_hydro) # Vérifier et corriger les géométries non valides ### Voronoi Diagram ### voronoi_lines = create_voronoi_lines(gdf, crs) + saved_voronoi_lines = voronoi_lines.copy() # merge all lines that can be merged without doubt (when only 2 lines are connected) # remove lines we don't want, then repeat until it doesn't change anymore gdf_lines = line_merge(voronoi_lines) + + # gdf_lines = unloop_geometry(voronoi_lines) + + gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_pre_pruning.geojson", driver='GeoJSON') + old_number_of_lines = len(gdf_lines) while True: gdf_lines = remove_extra_lines(gdf_lines) gdf_lines = line_merge(gdf_lines) - if len(gdf_lines) != old_number_of_lines: old_number_of_lines = len(gdf_lines) else: break - gdf_lines = connect_extremities(gdf_lines) - gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs + gdf_lines_partially_on_loops = get_lines_partially_on_loops(gdf_lines, saved_voronoi_lines, crs) + + gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) + # assembled = gpd.GeoDataFrame(pd.concat([gdf_connected_lines, gdf_lines], ignore_index=True)) + # assembled.to_file("test_assembled.geojson", driver='GeoJSON') # premerge + + gdf_lines = connect_extremities(gdf_lines, crs) + # gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs # connecting extremities creates lines we don't want (because of where we connect the extremities), # we remove them one last time - gdf_lines = remove_extra_lines(gdf_lines) + # gdf_lines = remove_extra_lines(gdf_lines) # gdf_lines = line_merge(gdf_lines) - gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_7.geojson", driver='GeoJSON') + gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_final.geojson", driver='GeoJSON') if __name__ == "__main__": - # Example usage - # Calcule du squelette - mask_hydro = gpd.read_file(MASK_GEOJSON_1) + mask_hydro = gpd.read_file(MASK_GEOJSON_5) + crs = mask_hydro.crs # Load a crs from input # Simplifier geometrie # buffer_geom = mask_hydro.apply(simplify_geometry) ## Appliquer un buffer - simplify_geom = mask_hydro.simplify(tolerance=2, preserve_topology=True) # simplifier les géométries avec Douglas-Peucker + simplify_geom = mask_hydro.simplify(tolerance=2) # simplifier les géométries avec Douglas-Peucker df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) df.to_file("test_0.geojson", driver='GeoJSON') # Squelette - # run(df, crs) - run(mask_hydro, crs) + run(df, crs) + # run(mask_hydro, crs) From 59bd0999fc4b8593da5417416fb55f466a9ca723 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 31 May 2024 08:54:01 +0200 Subject: [PATCH 04/68] save before creation of the Branch class --- skeleton.py | 519 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 400 insertions(+), 119 deletions(-) diff --git a/skeleton.py b/skeleton.py index 69e9fdf6..f4eb7b89 100644 --- a/skeleton.py +++ b/skeleton.py @@ -5,9 +5,10 @@ from geopandas.geodataframe import GeoDataFrame import pandas as pd import numpy as np +from math import sqrt -from shapely import LineString, Point, geometry -from shapely.geometry import CAP_STYLE, Polygon, MultiPoint +from shapely import LineString, Point, geometry, distance +from shapely.geometry import CAP_STYLE, Polygon, MultiPoint, Point from shapely.validation import make_valid from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate import logging @@ -24,6 +25,77 @@ BRIDGE_MAX_WIDTH = 50 VORONOI_MAX_LENGTH = 2 +# class Blob: +# def __init__(self, line:LineString): +# data = {'geometry':[], 'extremity_0':[], 'extremity_1':[]} +# self.gdf_lines = GeoDataFrame(data = data) +# self.extremities_set = set() +# self.add_line(line) + +# def add_line(self, line:LineString): +# extremity_0 = line.boundary.geoms[0] +# extremity_1 = line.boundary.geoms[1] +# new_data = {'geometry':[line], 'extremity_0':[extremity_0], 'extremity_1':[extremity_1]} +# gdf_line = GeoDataFrame(data = new_data) +# self.gdf_lines = pd.concat([self.gdf_lines, gdf_line]) +# self.extremities_set.add(extremity_0) +# self.extremities_set.add(extremity_1) + +# def is_line_connected(self, line:LineString) -> bool: +# extremity_0 = line.boundary.geoms[0] +# extremity_1 = line.boundary.geoms[1] +# if extremity_0 in self.extremities_set or extremity_1 in self.extremities_set: +# return True +# return False + +# def is_blob_connected(self, blob:'Blob') -> bool: +# if self.extremities_set.intersection(blob.extremities_set): +# return True +# return False + +# def add_blob(self, blob:'Blob'): +# self.gdf_lines = pd.concat([self.gdf_lines, blob.gdf_lines]) +# self.extremities_set = self.extremities_set.union(blob.extremities_set) + +# def save(self, file_name:str): +# self.gdf_lines.to_file(file_name + ".geojson", driver='GeoJSON') + +# def create_blobs(gdf_lines:GeoDataFrame)->List[Blob]: +# """ +# Args: +# -gdf_lines (GeoDataframe) : a list of linestring that are not connected by intermediate points +# """ +# blobs_list = [] +# for _, row in gdf_lines.iterrows(): +# line = row['geometry'] +# # line_connected = False +# # # search if the lines is connected to a blob +# # for blob in blobs_list: +# # if blob.is_line_connected(line): +# # blob.add_line(line) +# # line_connected = True +# # break +# # # if the line is not connected ot a blob, it creates its own blob +# # if not line_connected: +# blobs_list.append(Blob(line)) + +# # joint all connected blobs together +# disjoint_blobs_list = [] +# while blobs_list: +# blob = blobs_list.pop(len(blobs_list) - 1) +# eaten = False +# for gluton_blob in blobs_list: # gluton becuase it will "eat" the other blob +# if gluton_blob.is_blob_connected(blob): +# gluton_blob.add_blob(blob) +# eaten = True +# break +# if not eaten: +# disjoint_blobs_list.append(blob) + +# return disjoint_blobs_list + +class Branch: + def __init__(self, ) def fix_invalid_geometry(geometry): """ Fixer les géométries invalides @@ -98,34 +170,6 @@ def can_line_be_removed(line, vertices_dict): return False return True -def un_loop(loop): - polygon_loop = Polygon(loop.coords) - geometry = gpd.GeoSeries(polygon_loop, crs=2154) - gpd_loop = gpd.GeoDataFrame(geometry=geometry, crs=2154) - gpd_loop.to_file("loop.geojson", driver='GeoJSON') - # new_lines = triangulate(loop, 0, True) - # polygon_loop = MultiPoint(loop.coords) - # new_lines = voronoi_diagram(polygon_loop, envelope=polygon_loop, tolerance=0, edges=True) - - voronoi = create_voronoi_lines(gpd_loop, 2154) - - voronoi.to_file("voronoi.geojson", driver='GeoJSON') - - # coords_combine = loop.coords - # for index in voronoi.index : - # coords_combine = coords_combine + voronoi.iloc[index]['geometry'].coords - - voronoi.loc[len(voronoi.index)] = loop - - # combine_point = MultiPoint(coords_combine) - - # combine_point = MultiPoint(list(loop.coords) + [x.coords for x in voronoi['geometry']]) - triangulate_combine_points = triangulate(voronoi.unary_union, 0, True) - gpd.GeoDataFrame(geometry=triangulate_combine_points, crs=2154).to_file("triangulate_combine.geojson", driver='GeoJSON') - # geometry = gpd.GeoSeries(new_lines, crs=2154).explode(index_parts=False) - # geometry = gpd.GeoSeries(new_lines.geoms, crs=2154) - # gpd.GeoDataFrame(geometry=geometry, crs=2154).to_file("new_lines.geojson", driver='GeoJSON') - pass def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: """ @@ -155,8 +199,6 @@ def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: return vertices_dict def remove_extra_lines(gdf_lines:GeoDataFrame)->GeoDataFrame: - - gpd.GeoDataFrame(gdf_lines, crs=2154).to_file("test_ring.geojson", driver='GeoJSON') vertices_dict = get_vertices_dict(gdf_lines) lines_to_remove = [] @@ -289,12 +331,21 @@ def connect_extremities(gdf_lines:GeoDataFrame, crs)->GeoDataFrame: return: The same geodataframe with connection between extremities """ + # garanty that all lines have at least 3 segments, because the connexion is not # done on the last point, but the previous one. Therefore we need enough points - for index in gdf_lines.index: - line = gdf_lines.iloc[index]['geometry'] + new_lines = [] + for index, row in gdf_lines.iterrows(): + line = row['geometry'] if len(line.coords) < 4: # 3 segments, so 4 points - gdf_lines.loc[index] = line.segmentize(line.length/3) + line = line.segmentize(line.length/3) + new_lines.append(line) + gdf_lines = gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) + + # for index, row in gdf_lines.iterrows(): + # line = row['geometry'] + # if len(line.coords) < 4: # 3 segments, so 4 points + # row = line.segmentize(line.length/3) vertices_dict = get_vertices_dict(gdf_lines) # we remove all vertices that are not extremities (an extremity is connected to only one line) @@ -305,91 +356,291 @@ def connect_extremities(gdf_lines:GeoDataFrame, crs)->GeoDataFrame: for vertex in keys_to_remove: vertices_dict.pop(vertex, None) + ### + # WARNING !!! VISUALIZATION OF EXTREMITIES, TO REMOVE + geom = [vertex for vertex in vertices_dict.keys()] + test = gpd.GeoDataFrame(geometry=geom).set_crs(crs, allow_override=True) + test.to_file("test_extremities.geojson", driver='GeoJSON') + ### - best_candidate = {} - for vertex, [line] in vertices_dict.items(): - prior_point = get_prior_point(vertex, line) + ### + # WARNING !!! VISUALIZATION OF PRIOR POINTS, TO REMOVE + geom = [Point(*get_prior_point(vertex, line[0])) for vertex, line in vertices_dict.items()] + test = gpd.GeoDataFrame(geometry=geom).set_crs(crs, allow_override=True) + test.to_file("test_prior_points.geojson", driver='GeoJSON') + ### - # get the perpendicular line passing throught vertex, equation ax + by + c = 0 - a, b = vertex.coords[0][0] - prior_point[0], vertex.coords[0][1] - prior_point[1] - c = - a * vertex.coords[0][0] - b * vertex.coords[0][1] + new_lines = [] + # create a dataframe from the vertices (to make calculation faster) + data = {'x':[vertex.x for vertex in vertices_dict], + 'y':[vertex.y for vertex in vertices_dict], + 'vertex': vertices_dict.keys(), + 'line': [lines[0] for lines in vertices_dict.values()]} + df_vertex = pd.DataFrame(data=data) + + best_prior_points_to_connect = [] + for vertex_1, [line_1] in vertices_dict.items(): + prior_point_1 = get_prior_point(vertex_1, line_1) + + best_dot_value = 0 + best_prior_point = None + best_candidate = None + + # get all close enough vertices + df_vertex['square_distance'] = (df_vertex['x'] - vertex_1.x)**2 + (df_vertex['y'] - vertex_1.y)**2 + df_close_enough = df_vertex[df_vertex['square_distance'] < BRIDGE_MAX_WIDTH**2] + for index, extremity_row in df_close_enough.iterrows(): + + # we can connect only vertex with a previous one in the list + # if we allow the connection also with next vertices, the connection would be made twice + vertex_2 = extremity_row['vertex'] + if vertex_2 == vertex_1: + break - # make sure the prior_point is below the line (if above, we multiply the equation by -1 to change it) - if not a * prior_point[0] + b * prior_point[1] + c <= 0: - a, b, c = -a, -b, -c + prior_point_2 = get_prior_point(vertex_2, extremity_row['line']) + x_1 = prior_point_1[0] - vertex_1.x + y_1 = prior_point_1[1] - vertex_1.y + x_2 = prior_point_2[0] - vertex_2.x + y_2 = prior_point_2[1] - vertex_2.y - # search best vertex candidate to join this vertex i.e: - # the closest, not too far, vertex above the perpendicular line (to be "in front" of the river) - for other_vertex, [other_line] in vertices_dict.items(): - if other_vertex == vertex: - continue - # have to be above the perpendicular line - if a * other_vertex.coords[0][0] + b * other_vertex.coords[0][1] + c <= 0: + length_1 = sqrt(x_1**2 + y_1**2) + length_2 = sqrt(x_2**2 + y_2**2) + if not length_1 or not length_2: # a vector is null continue - # should not be too far: - squared_distance = (vertex.coords[0][0] - other_vertex.coords[0][0])**2 + (vertex.coords[0][1] - other_vertex.coords[0][1])**2 - if squared_distance > BRIDGE_MAX_WIDTH**2: - continue - other_prior_point = get_prior_point(other_vertex, other_line) - best_candidate[prior_point] = other_prior_point - new_lines = [] - already_done_prior_points = set() - # if 2 prior_points have each other as best candidate, we create a line between them - for prior_point, other_prior_point in best_candidate.items(): - if best_candidate[other_prior_point] == prior_point and other_prior_point not in already_done_prior_points: - already_done_prior_points.add(prior_point) - already_done_prior_points.add(other_prior_point) - # gdf_lines.loc[len(gdf_lines)] = [LineString([prior_point, other_prior_point])] - new_lines.append(LineString([prior_point, other_prior_point])) - - for index in gdf_lines.index: - line = gdf_lines.iloc[index]['geometry'] - start = 0 - end = len(line.coords) - if line.coords[2] in already_done_prior_points: - start = 2 - # new_lines.append(LineString(line.coords[0:2])) - if line.coords[-2] in already_done_prior_points: - end = -2 - # new_lines.append(LineString(line.coords[-2:])) - new_lines.append(LineString(line.coords[start:end])) + normalized_dot_product = x_1 * x_2 + y_1 * y_2 / (length_1 * length_2) + # if they are anticolinear enough, we keep it + if normalized_dot_product < best_dot_value : + best_dot_value = normalized_dot_product + best_prior_point = prior_point_2 + best_candidate = index + + if best_candidate: + best_prior_points_to_connect.append([prior_point_1, best_prior_point]) + + # search where both prior points are the best for each other + # for index, prior_points_pair in enumerate(best_prior_points_to_connect[:-1]): + # for opposite_prior_points_pair in best_prior_points_to_connect[index + 1:]: + for index, prior_points_pair in enumerate(best_prior_points_to_connect): + for opposite_prior_points_pair in best_prior_points_to_connect: + if prior_points_pair[0] == opposite_prior_points_pair[1] \ + and prior_points_pair[1] == opposite_prior_points_pair[0]: + new_lines.append(LineString(opposite_prior_points_pair)) + + return gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) + + # remove a part of a line when it had been extended, to make it smoother + # for index, row in gdf_lines.iterrows(): + # line = row['geometry'] + # start = 0 + # end = len(line.coords) + # if line.coords[2] in prior_points_connected: + # start = 2 + # # new_lines.append(LineString(line.coords[0:2])) + # if line.coords[-2] in prior_points_connected: + # end = -2 + # # new_lines.append(LineString(line.coords[-2:])) + # new_lines.append(LineString(line.coords[start:end])) + + # df_close_enough = df_vertex[(df_vertex['x'] - vertex_1.x)**2 + (df_vertex['y'] - vertex_1.y)**2) < BRIDGE_MAX_WIDTH] + + + + # for vertex_2, [line_2] in vertices_dict.items(): + # # if we have already done the calculation for vertex_1, vertex_2, no need to do it on vertex_2, vertex_1 + # if vertex_2 in vertice_done: + # continue + + # # check if the distance between both vertex is small enough for a bridge + # if vertex_1.distance(vertex_2) > BRIDGE_MAX_WIDTH: + # continue + + # # check how anticolinear 1 and 2 are + # prior_point_2 = get_prior_point(vertex_2, line_2) + # x_1 = prior_point_1[0] - vertex_1.x + # y_1 = prior_point_1[1] - vertex_1.y + # x_2 = prior_point_2[0] - vertex_2.x + # y_2 = prior_point_2[1] - vertex_2.y + + # length_1 = sqrt(x_1**2 + y_1**2) + # length_2 = sqrt(x_2**2 + y_2**2) + # if not length_1 or not length_2: # a vector is null + # continue + + # normalized_dot_product = x_1 * x_2 + y_1 * y_2 / (length_1 * length_2) + + # # if they are anticolinear enough, we keep it to draw a line later + # if normalized_dot_product < best_dot_value : + # best_dot_value = normalized_dot_product + # best_candidate = prior_point_2 + + # # if there is a candidate to draw a line (i.e : river under a bridge), we draw it + # if best_candidate: + # new_lines.append(LineString([prior_point_1, best_candidate])) + + # best_candidate = {} + # for vertex, [line] in vertices_dict.items(): + # prior_point = get_prior_point(vertex, line) + + # # get the perpendicular line passing throught vertex, equation ax + by + c = 0 + # a, b = vertex.coords[0][0] - prior_point[0], vertex.coords[0][1] - prior_point[1] + # c = - a * vertex.coords[0][0] - b * vertex.coords[0][1] + + # # make sure the prior_point is below the line (if above, we multiply the equation by -1 to change it) + # if not a * prior_point[0] + b * prior_point[1] + c <= 0: + # a, b, c = -a, -b, -c + + # # search best vertex candidate to join this vertex i.e: + # # the closest, not too far, vertex above the perpendicular line (to be "in front" of the river) + # for other_vertex, [other_line] in vertices_dict.items(): + # if other_vertex == vertex: + # continue + # # have to be above the perpendicular line + # if a * other_vertex.coords[0][0] + b * other_vertex.coords[0][1] + c <= 0: + # continue + # # should not be too far: + # squared_distance = (vertex.coords[0][0] - other_vertex.coords[0][0])**2 + (vertex.coords[0][1] - other_vertex.coords[0][1])**2 + # if squared_distance > BRIDGE_MAX_WIDTH**2: + # continue + # other_prior_point = get_prior_point(other_vertex, other_line) + # best_candidate[prior_point] = other_prior_point + + # new_lines = [] + # already_done_prior_points = set() + # # if 2 prior_points have each other as best candidate, we create a line between them + # for prior_point, other_prior_point in best_candidate.items(): + # if best_candidate[other_prior_point] == prior_point and other_prior_point not in already_done_prior_points: + # already_done_prior_points.add(prior_point) + # already_done_prior_points.add(other_prior_point) + # # gdf_lines.loc[len(gdf_lines)] = [LineString([prior_point, other_prior_point])] + # new_lines.append(LineString([prior_point, other_prior_point])) + + + # remove a part of the line, to make it smoother + # for index in gdf_lines.index: + # line = gdf_lines.iloc[index]['geometry'] + # start = 0 + # end = len(line.coords) + # if line.coords[2] in already_done_prior_points: + # start = 2 + # # new_lines.append(LineString(line.coords[0:2])) + # if line.coords[-2] in already_done_prior_points: + # end = -2 + # # new_lines.append(LineString(line.coords[-2:])) + # new_lines.append(LineString(line.coords[start:end])) # gdf_cutlines = gpd.GeoDataFrame(geometry=cut_lines) - return gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) + # return gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) # gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs # return -def get_lines_partially_on_loops(gdf_lines:GeoDataFrame, gdf_divided_lines:GeoDataFrame, crs:int)->GeoDataFrame: - # get all loops + +def add_a_line_on_each_loop(gdf_lines:GeoDataFrame, crs:int)->GeoDataFrame: + # separates loops and non loops + gdf_loops = gdf_lines[gdf_lines['geometry'].is_ring] + gdf_no_loops = gdf_lines[~gdf_lines['geometry'].is_ring] + + added_opposite_lines = [] + loop_cut_lines = [] + index_row_to_remove = [] + for index_loop, row_loop in gdf_loops.iterrows(): + points_on_loop = MultiPoint(list(row_loop['geometry'].coords)) + # get the line of gdf_lines that join the loop: + lines_joining_loop = gdf_no_loops[~gdf_no_loops['geometry'].disjoint(points_on_loop)].reset_index() + if len(lines_joining_loop) != 1: # we assume that, if there are 2+ lines connected to the loop, + continue # there is an "in and an "out", we don't need to try and connect the loop under a bridge + # If there are none, we can't do anything + + # we assume that the bridge is more or less in the same direction than the line joining the loop, + # and at the opposite side of the loop + line_to_continue = lines_joining_loop.loc[0]['geometry'] + + # search the extremity connected to the loop + if row_loop['geometry'].disjoint(line_to_continue.boundary.geoms[0]): + connected_extremity = line_to_continue.boundary.geoms[1] + other_extremity = line_to_continue.boundary.geoms[0] + elif row_loop['geometry'].disjoint(line_to_continue.boundary.geoms[1]): + connected_extremity = line_to_continue.boundary.geoms[0] + other_extremity = line_to_continue.boundary.geoms[1] + else: # should not happen, an extremity should be on the loop + continue + + # search the most distant point on the loop to the connected extremity, we assume + # it's where the river should continue + max_distance = 0 + opposite_connected_extremity = None + for coordinate in row_loop['geometry'].coords: + point = Point(coordinate) + current_distance = distance(point, connected_extremity) + if current_distance > max_distance: + max_distance = current_distance + opposite_connected_extremity = point + + # create a line, lenght 1m, that starts at the opposite point and goes the opposite way + length_connected_line = LineString([connected_extremity, other_extremity]).length + x = opposite_connected_extremity.x - (other_extremity.x - connected_extremity.x) / length_connected_line + y = opposite_connected_extremity.y - (other_extremity.y - connected_extremity.y) / length_connected_line + opposite_extremity_point = Point(x, y) + opposite_line = LineString([opposite_connected_extremity, opposite_extremity_point]) + added_opposite_lines.append(opposite_line) + + # dvide the loop in 2 lines, with extremities = [connected_extremity, opposite_connected_extremity] + index_row_to_remove.append(index_loop) + for index_point_loop, (x_point_loop, y_point_loop) in enumerate(list(row_loop['geometry'].coords)): + if x_point_loop == connected_extremity.x and y_point_loop == connected_extremity.y: + index_connected = index_point_loop + if x_point_loop == opposite_connected_extremity.x and y_point_loop == opposite_connected_extremity.y: + index_opposite_connected = index_point_loop + + if index_connected > index_opposite_connected: + index_connected, index_opposite_connected = index_opposite_connected, index_connected + + # create both lines for the loop cut + first_line = LineString(row_loop['geometry'].coords[index_connected:index_opposite_connected + 1]) + second_line = LineString(row_loop['geometry'].coords[index_opposite_connected:] + row_loop['geometry'].coords[:index_connected + 1]) + loop_cut_lines.append(first_line) + loop_cut_lines.append(second_line) + + gdf_lines = gdf_lines.drop(index = index_row_to_remove) # initial lines without the cut loops + gdf_new_loop_cut_lines = gpd.GeoDataFrame(geometry=loop_cut_lines).set_crs(crs, allow_override=True) # pairs of lines for the cut loops + gdf_new_opposite_lines = gpd.GeoDataFrame(geometry=added_opposite_lines).set_crs(crs, allow_override=True) # line added to the loop + + return gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_new_opposite_lines, gdf_new_loop_cut_lines], ignore_index=True)) + +def replace_lines_on_loops(gdf_lines:GeoDataFrame, gdf_divided_lines:GeoDataFrame, crs:int)->GeoDataFrame: + # separates loops and non loops gdf_loops = gdf_lines[gdf_lines['geometry'].is_ring].reset_index() + gdf_no_loops = gdf_lines[~gdf_lines['geometry'].is_ring].reset_index() # gets all points on loops points_on_loops = [] for index in gdf_loops.index: points_on_loops += list(gdf_loops.iloc[index]['geometry'].coords) - mp = MultiPoint(points_on_loops) + points_on_loops = MultiPoint(points_on_loops) # get lines with at least 1 point on a loop - commun_lines = gdf_divided_lines[~gdf_divided_lines['geometry'].disjoint(mp)].reset_index() - - # get lines with one point, and one point only, on loops - partially_on_loop_lines = [] - for index in commun_lines.index: - coords = commun_lines.iloc[index]['geometry'].coords - completely_covered = False - for index_loop in gdf_loops.index: - for begin_loop, end_loop in zip(gdf_loops.iloc[index_loop]['geometry'].coords[:-1], gdf_loops.iloc[index_loop]['geometry'].coords[1:]): - if (begin_loop in coords) and (end_loop in coords): - completely_covered = True - break - if completely_covered: - break - if not completely_covered: - partially_on_loop_lines.append(commun_lines.iloc[index]['geometry']) + commun_lines = gdf_divided_lines[~gdf_divided_lines['geometry'].disjoint(points_on_loops)].reset_index() + + # replace loops with those lines + return gpd.GeoDataFrame(pd.concat([gdf_no_loops, commun_lines], ignore_index=True)) - return gpd.GeoDataFrame(geometry=partially_on_loop_lines, crs=crs) + # # get lines with one point, and one point only, on loops + # partially_on_loop_lines = [] + # for index in commun_lines.index: + # coords = commun_lines.iloc[index]['geometry'].coords + # completely_covered = False + # for index_loop in gdf_loops.index: + # for begin_loop, end_loop in zip(gdf_loops.iloc[index_loop]['geometry'].coords[:-1], gdf_loops.iloc[index_loop]['geometry'].coords[1:]): + # if (begin_loop in coords) and (end_loop in coords): + # completely_covered = True + # break + # if completely_covered: + # break + # if not completely_covered: + # partially_on_loop_lines.append(commun_lines.iloc[index]['geometry']) + + # return gpd.GeoDataFrame(geometry=partially_on_loop_lines, crs=crs) # gdf_connected_lines.to_file("test_connected_lines.geojson", driver='GeoJSON') # assembled = gpd.GeoDataFrame(pd.concat([gdf_connected_lines, gdf_lines], ignore_index=True)) @@ -410,6 +661,24 @@ def get_lines_partially_on_loops(gdf_lines:GeoDataFrame, gdf_divided_lines:GeoDa # gpd.GeoDataFrame(new_lines_excluded, crs=2154).to_file("test_new_lines_excluded.geojson", driver='GeoJSON') +def get_branches(gdf_mask_hydro:GeoDataFrame, crs:int)->List[GeoDataFrame]: + """ create branches : a branch is a bunch of lines of river that are all connected""" + gdf_branches_list = [] + for index_branch, mask_branch_row in gdf_mask_hydro.iterrows(): + mask_branch = mask_branch_row["geometry"] + simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker + gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) + + gdf = check_geometry(gdf_branch_mask) # Vérifier et corriger les géométries non valides + + ### Voronoi Diagram ### + voronoi_lines = create_voronoi_lines(gdf, crs) + gdf_lines = line_merge(voronoi_lines) + # gdf_lines.to_file(f"branch_{index_branch}.geojson", driver='GeoJSON') + gdf_branches_list.append(gdf_lines) + pass + + def run(mask_hydro, crs): """Calculer le squelette hydrographique @@ -417,31 +686,43 @@ def run(mask_hydro, crs): - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - crs (str): code EPSG """ + + get_branches(mask_hydro, crs) + gdf = check_geometry(mask_hydro) # Vérifier et corriger les géométries non valides + ### Voronoi Diagram ### voronoi_lines = create_voronoi_lines(gdf, crs) - saved_voronoi_lines = voronoi_lines.copy() + gdf_lines = line_merge(voronoi_lines) + blobs_list = create_blobs(gdf_lines) + + pass + + # saved_voronoi_lines = voronoi_lines.copy() # merge all lines that can be merged without doubt (when only 2 lines are connected) # remove lines we don't want, then repeat until it doesn't change anymore - gdf_lines = line_merge(voronoi_lines) - # gdf_lines = unloop_geometry(voronoi_lines) - gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_pre_pruning.geojson", driver='GeoJSON') + # gdf_lines = line_merge(voronoi_lines) + + # gdf_lines.to_file("test_gdf_lines_1.geojson", driver='GeoJSON') + # old_number_of_lines = len(gdf_lines) + # while True: + # gdf_lines = remove_extra_lines(gdf_lines) + # gdf_lines = line_merge(gdf_lines) + # if len(gdf_lines) != old_number_of_lines: + # old_number_of_lines = len(gdf_lines) + # else: + # break - old_number_of_lines = len(gdf_lines) - while True: - gdf_lines = remove_extra_lines(gdf_lines) - gdf_lines = line_merge(gdf_lines) - if len(gdf_lines) != old_number_of_lines: - old_number_of_lines = len(gdf_lines) - else: - break + # gdf_lines = add_a_line_on_each_loop(gdf_lines, crs) - gdf_lines_partially_on_loops = get_lines_partially_on_loops(gdf_lines, saved_voronoi_lines, crs) - gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) + # gdf_lines = replace_lines_on_loops(gdf_lines, saved_voronoi_lines, crs) + # gdf_lines.to_file("test_add_a_line_on_each_loop.geojson", driver='GeoJSON') + # gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) + # assembled = gpd.GeoDataFrame(pd.concat([gdf_connected_lines, gdf_lines], ignore_index=True)) # assembled.to_file("test_assembled.geojson", driver='GeoJSON') # premerge @@ -452,18 +733,18 @@ def run(mask_hydro, crs): # we remove them one last time # gdf_lines = remove_extra_lines(gdf_lines) # gdf_lines = line_merge(gdf_lines) - + gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_final.geojson", driver='GeoJSON') + if __name__ == "__main__": mask_hydro = gpd.read_file(MASK_GEOJSON_5) - crs = mask_hydro.crs # Load a crs from input # Simplifier geometrie # buffer_geom = mask_hydro.apply(simplify_geometry) ## Appliquer un buffer - simplify_geom = mask_hydro.simplify(tolerance=2) # simplifier les géométries avec Douglas-Peucker - df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) - df.to_file("test_0.geojson", driver='GeoJSON') + # simplify_geom = mask_hydro.simplify(tolerance=2) # simplifier les géométries avec Douglas-Peucker + # df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) + # df.to_file("test_0.geojson", driver='GeoJSON') # Squelette - run(df, crs) + run(mask_hydro, crs) # run(mask_hydro, crs) From 4265cf03d6573ee146f819c68106d4561e3b4597 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 21 Jun 2024 08:06:51 +0200 Subject: [PATCH 05/68] Intermediary commit --- branch.py | 309 ++++++++++++++++++++++++++++++++++++++++++++++++++++ skeleton.py | 157 ++++++++++++++++++++++++-- 2 files changed, 454 insertions(+), 12 deletions(-) create mode 100644 branch.py diff --git a/branch.py b/branch.py new file mode 100644 index 00000000..9b932693 --- /dev/null +++ b/branch.py @@ -0,0 +1,309 @@ +from itertools import product +from typing import Dict, List, Tuple +from dataclasses import dataclass, field + +import geopandas as gpd +from geopandas.geodataframe import GeoDataFrame +import pandas as pd +import numpy as np +from math import sqrt + +from shapely import LineString, Point, geometry, distance +from shapely.geometry import CAP_STYLE, Polygon, MultiPoint, Point, MultiLineString +from shapely.validation import make_valid +from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate + +VORONOI_MAX_LENGTH = 2 +WATER_MIN_SIZE = 20 +MAX_BRIDGES_CANDIDATES = 3 +BRIDGE_MAX_WIDTH = 50 + +@dataclass +class Candidate: + branch_1:"Branch" + branch_2:"Branch" + extremity_1:tuple + extremity_2:tuple + squared_distance:float + middle_point:tuple = field(init=False) + line:LineString = field(init = False) + + def __post_init__(self): + self.middle_point = (self.extremity_1[0] + self.extremity_2[0])/2, (self.extremity_1[1] + self.extremity_2[1])/2 + self.line = LineString([ + (self.extremity_1[0], self.extremity_1[1]), + (self.extremity_2[0], self.extremity_2[1]) + ]) + + +class Branch: + """a branch is a geodataframe of lines of river that are all connected""" + + def __init__(self, branch_id: str, mask_branch:GeoDataFrame, crs:int): + """ + Args: + - crs (str): EPSG code + """ + self.branch_id = branch_id + self.crs = crs + simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker + gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) + + gdf = check_geometry(gdf_branch_mask) # Vérifier et corriger les géométries non valides + + voronoi_lines = create_voronoi_lines(gdf, self.crs) + self.gdf_lines = line_merge(voronoi_lines, self.crs) + self.set_modifications_from_changing_lines() + + def simplify(self): + """remove useless lines""" + old_number_of_lines = len(self.gdf_lines) + while old_number_of_lines > 1: # we will exit the loop when there is only 1 line or it cannot be simplify anymore + self.gdf_lines = remove_extra_lines(self.gdf_lines) + self.gdf_lines = line_merge(self.gdf_lines, self.crs) + if len(self.gdf_lines) != old_number_of_lines: + old_number_of_lines = len(self.gdf_lines) + else: + break + self.set_modifications_from_changing_lines() + + def set_modifications_from_changing_lines(self): + """when a modification to self.gdf_lines has been done, those modifications + must be taken into account for self.multiline""" + # create a multiline, set of all the lines in gdf_lines + self.multiline = MultiLineString([row['geometry'] for _, row in self.gdf_lines.iterrows()]) + + # create df_all_coords, with all the coordinates of all points of all lines in gdf_lines + """set all coordinates of all points""" + all_points = set() # we use a set instead of a list to remove doubles + for _, row in self.gdf_lines.iterrows(): + line = row['geometry'] + for coord in line.coords: + all_points.add(coord) + + self.all_points = list(all_points) + self.df_all_coords = pd.DataFrame(data={'x': [point[0] for point in all_points], + 'y': [point[1] for point in all_points],}) + + def distance_to_a_branch(self, other_branch:'Branch') -> float: + """return the distance to another branch""" + return self.multiline.distance(other_branch.multiline) + + def get_bridge_candidates(self, other_branch:'Branch'): + """""" + np_all_x = self.df_all_coords['x'].to_numpy() + np_all_y = self.df_all_coords['y'].to_numpy() + other_all_x = other_branch.df_all_coords['x'].to_numpy() + other_all_y = other_branch.df_all_coords['y'].to_numpy() + + x_diff = np_all_x - other_all_x[:,np.newaxis] # create a numpy matrix with all x from self minus all x from the other + y_diff = np_all_y - other_all_y[:,np.newaxis] # create a numpy matrix with all y from self minus all y from the other + + distance_squared = x_diff**2 + y_diff**2 + # index = np.unravel_index(np.argmax(distance_squared), distance_squared.shape) + # sort all the distances and get their indexes + indexes = np.unravel_index(np.argsort(distance_squared, axis=None), distance_squared.shape) + + # search for the best bridges candidates + candidates = [] + for index, (other_index, self_index) in enumerate(zip(indexes[0], indexes[1])): + # stop if we have enough candidates + if index >= MAX_BRIDGES_CANDIDATES: + break + # stop if the following candidates + if distance_squared[other_index][self_index] > BRIDGE_MAX_WIDTH * BRIDGE_MAX_WIDTH: + break + + candidates.append(Candidate(self, other_branch, self.all_points[self_index], other_branch.all_points[other_index], distance_squared[other_index][self_index])) + + # line = LineString([(self.df_all_coords['x'][index[1]], self.df_all_coords['y'][index[1]]), + # (other_branch.df_all_coords['x'][index[0]], other_branch.df_all_coords['y'][index[0]]), + # ]) + + return candidates + + + + def __repr__(self): + return str(self.branch_id) + +def check_geometry(initial_gdf): + """garanty polygons' validity + """ + # create simple geometry + gdf_simple = initial_gdf.explode(ignore_index=True) # ignore_index makes the resulting index multi-indexed + gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) + # remove invalid polygons + gdf_valid = gdf_without_duplicates.copy() + gdf_valid.geometry = gdf_valid.geometry.apply( + lambda geom: fix_invalid_geometry(geom) + ) + return gdf_valid + +def fix_invalid_geometry(geometry): + """ Fixer les géométries invalides + """ + if not geometry.is_valid: + return make_valid(geometry) + else: + return geometry + + +def create_voronoi_lines(mask_hydro, crs): + """ Créer les lignes de Voronoi + + Args: + -mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) + - crs (str): code EPSG + + Returns : + out lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique + """ + # we don't work on an empty geometry + if not mask_hydro['geometry'].dtype == 'geometry': + return + + # divide geometry into segments no longer than max_segment_length + segmentize_geom = (mask_hydro['geometry'].unary_union).segmentize(max_segment_length=VORONOI_MAX_LENGTH) + # Create the voronoi diagram and only keep polygon + regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) + + ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### + geometry = gpd.GeoSeries(regions.geoms, crs=crs).explode(index_parts=False) + df = gpd.GeoDataFrame(geometry=geometry, crs=crs) + lines_filter = df.sjoin(mask_hydro, predicate='within') # Opération spatiale "Within" + # Enregistrer le diagram de Voronoi (Squelette) + lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index + lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' + + return gpd.GeoDataFrame(lines_filter, crs=crs) + +def remove_extra_lines(gdf_lines:GeoDataFrame)->GeoDataFrame: + vertices_dict = get_vertices_dict(gdf_lines) + + lines_to_remove = [] + for vertex, line_list in vertices_dict.items(): + if len(line_list) == 1 : + continue + + # if len(line_list) !is not 1, then it's probably 3; Can't be 2 because of the previous merge, very unlikely to be 4+ + lines_to_keep = [] + lines_that_can_be_removed = [] + for line in line_list: + if can_line_be_removed(line, vertices_dict): + lines_that_can_be_removed.append(line) + else: + lines_to_keep.append(line) + + # if no line can be removed, we continue + if not lines_that_can_be_removed: + continue + + + # there is only 1 line that can be removed and 2+ to be kept, so we remove it + if len(lines_that_can_be_removed) == 1: + lines_to_remove.append(lines_that_can_be_removed[0]) + continue + + # strange case where 3+ lines are together, but isolated from the rest of the world + # we decide to keep the longuest line + if len(lines_to_keep) == 0: + max_distance = max([line.length for line in lines_that_can_be_removed]) + for index_line, line in enumerate(lines_that_can_be_removed): + if line.length == max_distance: + lines_that_can_be_removed.pop(index_line) + lines_to_keep.append(line) + + # at least 2 lines can be removed, we will keep the one forming the straightest line with + # a line to keep, we assume it's the most likely to "continue" the river + min_dot_product = 1 + line_that_should_not_be_removed = None + for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): + if vertex == line_to_keep.boundary.geoms[0]: + vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + else: + vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + + if vertex == line_that_can_be_removed.boundary.geoms[0]: + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + else: + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + + length_to_keep = np.linalg.norm(vector_to_keep) + length_to_remove = np.linalg.norm(vector_to_remove) + if not length_to_keep or not length_to_remove: # length = 0, probably won't happen + continue + + vector_to_keep = vector_to_keep / length_to_keep + vector_to_remove = vector_to_remove / length_to_remove + dot_product = np.dot(vector_to_keep, vector_to_remove) + # the smallest dot product (which should be negative) is the straightest couple of lines + if dot_product < min_dot_product: + min_dot_product = dot_product + line_that_should_not_be_removed = line_that_can_be_removed + + lines_that_can_be_removed.remove(line_that_should_not_be_removed) + lines_to_remove += lines_that_can_be_removed + + # return the lines without the lines to remove + return gdf_lines[~gdf_lines['geometry'].isin(lines_to_remove)] + +def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: + """ + get a dictionary of vertices listing all the lines having a specific vertex as one of its extremities + Args: + - gdf_lines:geodataframe containing a list of lines + return: + - a dict with the vertices are the keys and the values are lists of lines + """ + # prepare a vertice dict, containing all the lines connected on a same vertex + vertices_dict = {} + for index in gdf_lines.index : + line = gdf_lines.iloc[index]['geometry'] + if line.is_ring: # it's a loop : no extremity + continue + + point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] + try : + vertices_dict[point_a].append(line) + except KeyError: + vertices_dict[point_a] = [line] + try : + vertices_dict[point_b].append(line) + except KeyError: + vertices_dict[point_b] = [line] + return vertices_dict + +def can_line_be_removed(line, vertices_dict): + if line.length > WATER_MIN_SIZE: + return False + + point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] + if len(vertices_dict[point_a]) > 1 and len(vertices_dict[point_b]) > 1: + return False + return True + +def line_merge(voronoi_lines, crs): + """Fusionner tous les LINESTRING en un seul objet MultiLineString + + Args: + - voronoi_lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique + + Returns: + lines (GeoDataframe): Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique + + """ + # Fusionner tous les LINESTRING en un seul objet MultiLineString + merged_line = voronoi_lines.geometry.unary_union + # Appliquer un algo de fusion des lignes (ex. "ST_LineMerge") sur l'objet MultiLineString + merged_line = linemerge(merged_line) + + # in case the branch is reduced to a single line + if type(merged_line) == LineString: + return gpd.GeoDataFrame(geometry=[merged_line], crs=crs) + + # turn multipart geometries into multiple single geometries + geometry = gpd.GeoSeries(merged_line.geoms, crs=crs).explode(index_parts=False) + lines = gpd.GeoDataFrame(geometry=geometry, crs=crs) + + return lines \ No newline at end of file diff --git a/skeleton.py b/skeleton.py index f4eb7b89..649960a0 100644 --- a/skeleton.py +++ b/skeleton.py @@ -6,13 +6,16 @@ import pandas as pd import numpy as np from math import sqrt - +import psycopg2 +from psycopg2.extensions import cursor from shapely import LineString, Point, geometry, distance -from shapely.geometry import CAP_STYLE, Polygon, MultiPoint, Point +from shapely.geometry import CAP_STYLE, Polygon, MultiPoint, Point, MultiLineString from shapely.validation import make_valid from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate import logging +from branch import Branch, Candidate, BRIDGE_MAX_WIDTH + MASK_GEOJSON_1 = "/home/MDaab/data/squelette_test/MaskHydro_merge 1.geojson" MASK_GEOJSON_2 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm2.geojson" MASK_GEOJSON_3 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm3.geojson" @@ -22,8 +25,15 @@ MIDDLE_FILE = "squelette_filtrer.geojson" SAVE_FILE = "squelette_hydrographique.geojson" WATER_MIN_SIZE = 20 -BRIDGE_MAX_WIDTH = 50 VORONOI_MAX_LENGTH = 2 +MAX_BRIDGES = 2 + +DB_NAME = "bduni_france_consultation" +DB_HOST = "bduni_consult.ign.fr" +DB_USER = "invite" +DB_PASSWORD = "28de#" +DB_PORT = "5432" + # class Blob: # def __init__(self, line:LineString): @@ -94,8 +104,14 @@ # return disjoint_blobs_list -class Branch: - def __init__(self, ) + +def db_connector(): + """Return a connector to the postgis database""" + return psycopg2.connect(database=DB_NAME, + host=DB_HOST, + user=DB_USER, + password=DB_PASSWORD, + port=DB_PORT) def fix_invalid_geometry(geometry): """ Fixer les géométries invalides @@ -187,7 +203,6 @@ def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: continue point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] - # point_a, point_b = line.coords[0], line.coords[-1] try : vertices_dict[point_a].append(line) except KeyError: @@ -679,25 +694,138 @@ def get_branches(gdf_mask_hydro:GeoDataFrame, crs:int)->List[GeoDataFrame]: pass -def run(mask_hydro, crs): +def run(gdf_mask_hydro, crs): """Calculer le squelette hydrographique Args: - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - crs (str): code EPSG """ + branches_list = [] + + for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): + mask_branch = branch_mask_row["geometry"] + + new_branch = Branch(index_branch, mask_branch, crs) + new_branch.simplify() + branches_list.append(new_branch) + + branches_pair = [] + for index, branch_a in enumerate(branches_list[:-1]): + for branch_b in branches_list[index + 1:]: + distance = branch_a.distance_to_a_branch(branch_b) + if distance < BRIDGE_MAX_WIDTH: + branches_pair.append((branch_a, branch_b)) + + validated_candidates = [] + extremities_connected = set() + for branch_a, branch_b in branches_pair: + branch_a: Branch + candidates = branch_a.get_bridge_candidates(branch_b) + + # test each candidate to see if we can draw a line for them + nb_bridges_crossed = 0 + for candidate in candidates : + # we connect each extremity only once + if candidate.extremity_1 in extremities_connected or candidate.extremity_2 in extremities_connected : + continue + + line = f"LINESTRING({candidate.extremity_1[0]} {candidate.extremity_1[1]}, {candidate.extremity_2[0]} {candidate.extremity_2[1]})" + query_linear = f"SELECT cleabs FROM public.Construction_lineaire WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" + query_area = f"SELECT cleabs FROM public.Construction_surfacique WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" + + results = True + """ + with db_connector() as db_conn: + with db_conn.cursor() as db_cursor: + db_cursor.execute(query_linear) + results = db_cursor.fetchall() + # if no result with linear bridge, maybe with area bridge... + if not results: + db_cursor.execute(query_area) + results = db_cursor.fetchall() + """ + # if the line does not cross any bridge, we don't validate that candidate + if not results: + continue + + # candidate validated + extremities_connected.add(candidate.extremity_1) + extremities_connected.add(candidate.extremity_2) + validated_candidates.append(candidate) + nb_bridges_crossed += 1 + if nb_bridges_crossed >= MAX_BRIDGES: # max bridges reached between those 2 branches + break + + bridge_lines = [validated_candidate.line for validated_candidate in validated_candidates] + gdf_bridge_lines = gpd.GeoDataFrame(geometry=bridge_lines).set_crs(crs, allow_override=True) + gdf_bridge_lines.to_file("test_bridges.geojson", driver='GeoJSON') + + # for branch in branches_list: + # branch:Branch + # gdf_branch_lines = gpd.GeoDataFrame(branch.gdf_lines).set_crs(crs, allow_override=True) + # gdf_branch_lines.to_file(f"{branch.branch_id}.geojson", driver='GeoJSON') + + # branch_lines_list = gpd.GeoDataFrame() + # for branch in branches_list: + # branch:Branch + # data = {'branch_id': branch.branch_id, 'geometry': branch.gdf_lines} + # # data = {'branch_id': [branch.branch_id for _ in range(len(branch.gdf_lines))], 'geometry': branch.gdf_lines} + # gdf_branch_lines = gpd.GeoDataFrame(data).set_crs(crs, allow_override=True) + # branch_lines_list = gpd.GeoDataFrame(pd.concat([branch_lines_list, gdf_branch_lines], ignore_index=True)) + # # gdf_branch_lines.to_file(f"{branch.branch_id}.geojson", driver='GeoJSON') + + # branch_lines_list.to_file("test_branch_lines.geojson", driver='GeoJSON') + + branch_lines_list = [branch.gdf_lines for branch in branches_list] + gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) + gdf_branch_lines.to_file("test_branch_lines.geojson", driver='GeoJSON') + + # gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) + + + # validated_candidates + # gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_final.geojson", driver='GeoJSON') + + pass + # gdf_inbetween_lines = GeoDataFrame(geometry=inbetween_lines, crs=crs) + # gdf_inbetween_lines.to_file("test_bridge.geojson", driver='GeoJSON') + + + # query_list = "" + # for index, candidate in enumerate(candidates): + # candidate:Candidate + # geometry = f"LINESTRING({candidate.extremity_1[0]} {candidate.extremity_1[1]}, {candidate.extremity_2[0]} {candidate.extremity_2[1]})" + # query = f"SELECT cleabs FROM public.Construction_lineaire WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{geometry}'));" + # query_list += query + # if index >= 5: + # break + + # db_cursor.execute(query_list) + # results = db_cursor.fetchall() + +def run_branch(mask_hydro, crs): get_branches(mask_hydro, crs) + pass + +def run_old(mask_hydro, crs): + """Calculer le squelette hydrographique + + Args: + - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) + - crs (str): code EPSG + """ + + # get_branches(mask_hydro, crs) gdf = check_geometry(mask_hydro) # Vérifier et corriger les géométries non valides ### Voronoi Diagram ### voronoi_lines = create_voronoi_lines(gdf, crs) gdf_lines = line_merge(voronoi_lines) - blobs_list = create_blobs(gdf_lines) - - pass + return # saved_voronoi_lines = voronoi_lines.copy() # merge all lines that can be merged without doubt (when only 2 lines are connected) @@ -742,9 +870,14 @@ def run(mask_hydro, crs): crs = mask_hydro.crs # Load a crs from input # Simplifier geometrie # buffer_geom = mask_hydro.apply(simplify_geometry) ## Appliquer un buffer + # simplify_geom = mask_hydro.simplify(tolerance=2) # simplifier les géométries avec Douglas-Peucker # df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) - # df.to_file("test_0.geojson", driver='GeoJSON') + # run_old(df, crs) + + # run_branch(mask_hydro, crs) + # Squelette + # run(df, crs) run(mask_hydro, crs) - # run(mask_hydro, crs) + From 1feaf72a1af60d97507ad57c979bd4441103779e Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 21 Jun 2024 11:38:33 +0200 Subject: [PATCH 06/68] acceptable result --- branch.py | 4 +- skeleton.py | 209 ++++++++++++++++++++++++---------------------------- 2 files changed, 100 insertions(+), 113 deletions(-) diff --git a/branch.py b/branch.py index 9b932693..4d00ba7b 100644 --- a/branch.py +++ b/branch.py @@ -47,9 +47,9 @@ def __init__(self, branch_id: str, mask_branch:GeoDataFrame, crs:int): self.branch_id = branch_id self.crs = crs simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker - gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) + self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) - gdf = check_geometry(gdf_branch_mask) # Vérifier et corriger les géométries non valides + gdf = check_geometry(self.gdf_branch_mask) # Vérifier et corriger les géométries non valides voronoi_lines = create_voronoi_lines(gdf, self.crs) self.gdf_lines = line_merge(voronoi_lines, self.crs) diff --git a/skeleton.py b/skeleton.py index 649960a0..3804f1a6 100644 --- a/skeleton.py +++ b/skeleton.py @@ -26,7 +26,7 @@ SAVE_FILE = "squelette_hydrographique.geojson" WATER_MIN_SIZE = 20 VORONOI_MAX_LENGTH = 2 -MAX_BRIDGES = 2 +MAX_BRIDGES = 1 DB_NAME = "bduni_france_consultation" DB_HOST = "bduni_consult.ign.fr" @@ -35,75 +35,28 @@ DB_PORT = "5432" -# class Blob: -# def __init__(self, line:LineString): -# data = {'geometry':[], 'extremity_0':[], 'extremity_1':[]} -# self.gdf_lines = GeoDataFrame(data = data) -# self.extremities_set = set() -# self.add_line(line) - -# def add_line(self, line:LineString): -# extremity_0 = line.boundary.geoms[0] -# extremity_1 = line.boundary.geoms[1] -# new_data = {'geometry':[line], 'extremity_0':[extremity_0], 'extremity_1':[extremity_1]} -# gdf_line = GeoDataFrame(data = new_data) -# self.gdf_lines = pd.concat([self.gdf_lines, gdf_line]) -# self.extremities_set.add(extremity_0) -# self.extremities_set.add(extremity_1) - -# def is_line_connected(self, line:LineString) -> bool: -# extremity_0 = line.boundary.geoms[0] -# extremity_1 = line.boundary.geoms[1] -# if extremity_0 in self.extremities_set or extremity_1 in self.extremities_set: -# return True -# return False - -# def is_blob_connected(self, blob:'Blob') -> bool: -# if self.extremities_set.intersection(blob.extremities_set): -# return True -# return False - -# def add_blob(self, blob:'Blob'): -# self.gdf_lines = pd.concat([self.gdf_lines, blob.gdf_lines]) -# self.extremities_set = self.extremities_set.union(blob.extremities_set) - -# def save(self, file_name:str): -# self.gdf_lines.to_file(file_name + ".geojson", driver='GeoJSON') - -# def create_blobs(gdf_lines:GeoDataFrame)->List[Blob]: -# """ -# Args: -# -gdf_lines (GeoDataframe) : a list of linestring that are not connected by intermediate points -# """ -# blobs_list = [] -# for _, row in gdf_lines.iterrows(): -# line = row['geometry'] -# # line_connected = False -# # # search if the lines is connected to a blob -# # for blob in blobs_list: -# # if blob.is_line_connected(line): -# # blob.add_line(line) -# # line_connected = True -# # break -# # # if the line is not connected ot a blob, it creates its own blob -# # if not line_connected: -# blobs_list.append(Blob(line)) - -# # joint all connected blobs together -# disjoint_blobs_list = [] -# while blobs_list: -# blob = blobs_list.pop(len(blobs_list) - 1) -# eaten = False -# for gluton_blob in blobs_list: # gluton becuase it will "eat" the other blob -# if gluton_blob.is_blob_connected(blob): -# gluton_blob.add_blob(blob) -# eaten = True -# break -# if not eaten: -# disjoint_blobs_list.append(blob) - -# return disjoint_blobs_list +class GroupMaker: + def __init__(self, element_list): + self.element_set_list = [] + for element in element_list: + self.element_set_list.append({element}) + def find_index(self, element) -> int: + for index, element_set in enumerate(self.element_set_list): + if element in element_set: + return index + + def are_together(self, element_a, element_b)->bool: + """return true if 2 elements are already together""" + index = self.find_index(element_a) + return element_b in self.element_set_list[index] + + def put_together(self, element_a, element_b): + """put the set of a and b together""" + index_a = self.find_index(element_a) + set_a = self.element_set_list.pop(index_a) + index_b = self.find_index(element_b) + self.element_set_list[index_b] = self.element_set_list[index_b].union(set_a) def db_connector(): """Return a connector to the postgis database""" @@ -113,6 +66,27 @@ def db_connector(): password=DB_PASSWORD, port=DB_PORT) +def query_db_for_bridge(candidate:Candidate)->bool: + """ + Query the database to check if a candidate for a bridge indeed intersects a bridge + """ + line = f"LINESTRING({candidate.extremity_1[0]} {candidate.extremity_1[1]}, {candidate.extremity_2[0]} {candidate.extremity_2[1]})" + query_linear = f"SELECT cleabs FROM public.Construction_lineaire WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" + query_area = f"SELECT cleabs FROM public.Construction_surfacique WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" + + with db_connector() as db_conn: + with db_conn.cursor() as db_cursor: + db_cursor.execute(query_linear) + results = db_cursor.fetchall() + # if no result with linear bridge, maybe with area bridge... + if not results: + db_cursor.execute(query_area) + results = db_cursor.fetchall() + return True if results else False + + + + def fix_invalid_geometry(geometry): """ Fixer les géométries invalides """ @@ -694,33 +668,25 @@ def get_branches(gdf_mask_hydro:GeoDataFrame, crs:int)->List[GeoDataFrame]: pass -def run(gdf_mask_hydro, crs): - """Calculer le squelette hydrographique - - Args: - - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - - crs (str): code EPSG +def select_candidates(branches_pair_list:List[Tuple])->List[Candidate]: """ - branches_list = [] - - for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): - mask_branch = branch_mask_row["geometry"] - - new_branch = Branch(index_branch, mask_branch, crs) - new_branch.simplify() - branches_list.append(new_branch) - - branches_pair = [] - for index, branch_a in enumerate(branches_list[:-1]): - for branch_b in branches_list[index + 1:]: - distance = branch_a.distance_to_a_branch(branch_b) - if distance < BRIDGE_MAX_WIDTH: - branches_pair.append((branch_a, branch_b)) - + selects candidates between pairs of branches + """ + # get all the branches (each only once, hence the set) + branch_set = set() + for branch_a, branch_b, _ in branches_pair_list: + branch_set.add(branch_a) + branch_set.add(branch_b) + + branch_group = GroupMaker(list(branch_set)) validated_candidates = [] extremities_connected = set() - for branch_a, branch_b in branches_pair: + for branch_a, branch_b, _ in branches_pair_list: branch_a: Branch + + # we don't create link between 2 branches already connected + if branch_group.are_together(branch_a, branch_b): + continue candidates = branch_a.get_bridge_candidates(branch_b) # test each candidate to see if we can draw a line for them @@ -730,36 +696,53 @@ def run(gdf_mask_hydro, crs): if candidate.extremity_1 in extremities_connected or candidate.extremity_2 in extremities_connected : continue - line = f"LINESTRING({candidate.extremity_1[0]} {candidate.extremity_1[1]}, {candidate.extremity_2[0]} {candidate.extremity_2[1]})" - query_linear = f"SELECT cleabs FROM public.Construction_lineaire WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" - query_area = f"SELECT cleabs FROM public.Construction_surfacique WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" - - results = True - """ - with db_connector() as db_conn: - with db_conn.cursor() as db_cursor: - db_cursor.execute(query_linear) - results = db_cursor.fetchall() - # if no result with linear bridge, maybe with area bridge... - if not results: - db_cursor.execute(query_area) - results = db_cursor.fetchall() - """ - # if the line does not cross any bridge, we don't validate that candidate - if not results: - continue + # check with database + # is_bridge = query_db_for_bridge(candidate) + # # if the line does not cross any bridge, we don't validate that candidate + # if not is_bridge: + # continue # candidate validated extremities_connected.add(candidate.extremity_1) extremities_connected.add(candidate.extremity_2) validated_candidates.append(candidate) + branch_group.put_together(branch_a, branch_b) nb_bridges_crossed += 1 if nb_bridges_crossed >= MAX_BRIDGES: # max bridges reached between those 2 branches break + return validated_candidates + + +def run(gdf_mask_hydro, crs): + """Calculer le squelette hydrographique + + Args: + - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) + - crs (str): code EPSG + """ + # create branches + branches_list = [] + for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): + mask_branch = branch_mask_row["geometry"] + new_branch = Branch(index_branch, mask_branch, crs) + new_branch.simplify() + branches_list.append(new_branch) + + branches_pair_list = [] + for index, branch_a in enumerate(branches_list[:-1]): + for branch_b in branches_list[index + 1:]: + distance = branch_a.distance_to_a_branch(branch_b) + if distance < BRIDGE_MAX_WIDTH: + branches_pair_list.append((branch_a, branch_b, distance)) + + # sort the branches pairs by distance between them + branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) + + validated_candidates = select_candidates(branches_pair_list) bridge_lines = [validated_candidate.line for validated_candidate in validated_candidates] gdf_bridge_lines = gpd.GeoDataFrame(geometry=bridge_lines).set_crs(crs, allow_override=True) - gdf_bridge_lines.to_file("test_bridges.geojson", driver='GeoJSON') + gdf_bridge_lines.to_file("test_bridges_group_maker.geojson", driver='GeoJSON') # for branch in branches_list: # branch:Branch @@ -781,6 +764,10 @@ def run(gdf_mask_hydro, crs): gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) gdf_branch_lines.to_file("test_branch_lines.geojson", driver='GeoJSON') + mask_list = [branch.gdf_branch_mask for branch in branches_list] + gdf_mask = gpd.GeoDataFrame(pd.concat(mask_list, ignore_index=True)) + gdf_mask.to_file("test_mask.geojson", driver='GeoJSON') + # gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) From e7a11c0009731e114e462280759f01c505db8a26 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 25 Jun 2024 11:51:11 +0200 Subject: [PATCH 07/68] gap closed from mask skeleton lines to points on mask used to closed gaps --- branch.py | 352 ++++++++++++++++++++++++++++++---------------------- skeleton.py | 27 ++-- 2 files changed, 220 insertions(+), 159 deletions(-) diff --git a/branch.py b/branch.py index 4d00ba7b..7ccb7758 100644 --- a/branch.py +++ b/branch.py @@ -15,8 +15,8 @@ VORONOI_MAX_LENGTH = 2 WATER_MIN_SIZE = 20 -MAX_BRIDGES_CANDIDATES = 3 -BRIDGE_MAX_WIDTH = 50 +MAX_GAP_CANDIDATES = 3 +GAP_MAX_WIDTH = 50 @dataclass class Candidate: @@ -46,50 +46,94 @@ def __init__(self, branch_id: str, mask_branch:GeoDataFrame, crs:int): """ self.branch_id = branch_id self.crs = crs - simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker - self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) + # simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker + # self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) + + self.gdf_branch_mask = self.check_and_create_gdf_branch_mask(mask_branch) + self.gap_points = [] # will contain points on the exterior that are connected to close gaps + self.set_df_all_coords() + + def check_and_create_gdf_branch_mask(self, mask_branch): + """garanty polygons' validity""" + raw_gdf_branch_mask = gpd.GeoDataFrame(geometry=[mask_branch], crs=self.crs) + + # create simple geometry + gdf_simple = raw_gdf_branch_mask.explode(ignore_index=True) # ignore_index makes the resulting index multi-indexed + gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) + # remove invalid polygons + gdf_valid = gdf_without_duplicates.copy() + gdf_valid.geometry = gdf_valid.geometry.apply( + lambda geom: fix_invalid_geometry(geom) + ) + return gdf_valid + + def create_skeleton(self): + voronoi_lines = self.create_voronoi_lines() + + # draw a new line for each point added to closes gaps to the nearest points on voronoi_lines + np_points = get_df_points_from_gdf(voronoi_lines).to_numpy().transpose() + for gap_point in self.gap_points: + distance_squared = (np_points[0] - gap_point.x)**2 + (np_points[1] - gap_point.y)**2 + min_index = np.unravel_index(np.argmin(distance_squared, axis=None), distance_squared.shape)[0] + voronoi_lines.loc[len(voronoi_lines)] = LineString([gap_point, Point(np_points[0][min_index], np_points[1][min_index])]) - gdf = check_geometry(self.gdf_branch_mask) # Vérifier et corriger les géométries non valides - - voronoi_lines = create_voronoi_lines(gdf, self.crs) self.gdf_lines = line_merge(voronoi_lines, self.crs) - self.set_modifications_from_changing_lines() + # self.set_modifications_from_changing_lines() def simplify(self): """remove useless lines""" old_number_of_lines = len(self.gdf_lines) while old_number_of_lines > 1: # we will exit the loop when there is only 1 line or it cannot be simplify anymore - self.gdf_lines = remove_extra_lines(self.gdf_lines) + self.remove_extra_lines() self.gdf_lines = line_merge(self.gdf_lines, self.crs) if len(self.gdf_lines) != old_number_of_lines: old_number_of_lines = len(self.gdf_lines) else: break - self.set_modifications_from_changing_lines() - - def set_modifications_from_changing_lines(self): - """when a modification to self.gdf_lines has been done, those modifications - must be taken into account for self.multiline""" - # create a multiline, set of all the lines in gdf_lines - self.multiline = MultiLineString([row['geometry'] for _, row in self.gdf_lines.iterrows()]) + # self.set_modifications_from_changing_lines() - # create df_all_coords, with all the coordinates of all points of all lines in gdf_lines - """set all coordinates of all points""" + def set_df_all_coords(self): all_points = set() # we use a set instead of a list to remove doubles - for _, row in self.gdf_lines.iterrows(): - line = row['geometry'] - for coord in line.coords: + for _, row in self.gdf_branch_mask.iterrows(): + polygon:Polygon = row['geometry'] + for coord in polygon.exterior.coords: all_points.add(coord) self.all_points = list(all_points) self.df_all_coords = pd.DataFrame(data={'x': [point[0] for point in all_points], 'y': [point[1] for point in all_points],}) + # def set_modifications_from_changing_lines(self): + # """when a modification to self.gdf_lines has been done, those modifications + # must be taken into account for self.multiline""" + # # create a multiline, set of all the lines in gdf_lines + # self.multiline = MultiLineString([row['geometry'] for _, row in self.gdf_lines.iterrows()]) + + # # create df_all_coords, with all the coordinates of all points of all lines in gdf_lines + # """set all coordinates of all points""" + # all_points = set() # we use a set instead of a list to remove doubles + # # for _, row in self.gdf_lines.iterrows(): + # # line = row['geometry'] + # # for coord in line.coords: + # # all_points.add(coord) + # for _, row in self.gdf_branch_mask.iterrows(): + # polygon:Polygon = row['geometry'] + # for coord in polygon.exterior.coords: + # all_points.add(coord) + + # self.all_points = list(all_points) + # self.df_all_coords = pd.DataFrame(data={'x': [point[0] for point in all_points], + # 'y': [point[1] for point in all_points],}) + + # def distance_to_a_branch(self, other_branch:'Branch') -> float: + # """return the distance to another branch""" + # return self.multiline.distance(other_branch.multiline) + def distance_to_a_branch(self, other_branch:'Branch') -> float: """return the distance to another branch""" - return self.multiline.distance(other_branch.multiline) + return self.gdf_branch_mask.distance(other_branch.gdf_branch_mask)[0] # distance is given as a series with only 1 value - def get_bridge_candidates(self, other_branch:'Branch'): + def get_gap_candidates(self, other_branch:'Branch'): """""" np_all_x = self.df_all_coords['x'].to_numpy() np_all_y = self.df_all_coords['y'].to_numpy() @@ -104,41 +148,156 @@ def get_bridge_candidates(self, other_branch:'Branch'): # sort all the distances and get their indexes indexes = np.unravel_index(np.argsort(distance_squared, axis=None), distance_squared.shape) - # search for the best bridges candidates + # search for the best candidates to close gaps candidates = [] for index, (other_index, self_index) in enumerate(zip(indexes[0], indexes[1])): # stop if we have enough candidates - if index >= MAX_BRIDGES_CANDIDATES: + if index >= MAX_GAP_CANDIDATES: break # stop if the following candidates - if distance_squared[other_index][self_index] > BRIDGE_MAX_WIDTH * BRIDGE_MAX_WIDTH: + if distance_squared[other_index][self_index] > GAP_MAX_WIDTH * GAP_MAX_WIDTH: break candidates.append(Candidate(self, other_branch, self.all_points[self_index], other_branch.all_points[other_index], distance_squared[other_index][self_index])) + return candidates - # line = LineString([(self.df_all_coords['x'][index[1]], self.df_all_coords['y'][index[1]]), - # (other_branch.df_all_coords['x'][index[0]], other_branch.df_all_coords['y'][index[0]]), - # ]) + def remove_extra_lines(self): + vertices_dict = get_vertices_dict(self.gdf_lines) - return candidates + lines_to_remove = [] + for vertex, line_list in vertices_dict.items(): + if len(line_list) == 1 : + continue + # # the vertex used to close a gap have to be kept + # for point in self.gap_points: + # point:Point + # if point.equals_exact(vertex, 0.001): + # lines_to_keep.append(line[0]) # the should be only one line + # continue + + # if vertex in self.gap_points: + # lines_to_keep.append(line[0]) # the should be only one line + # continue + + # if len(line_list) !is not 1, then it's probably 3; Can't be 2 because of the previous merge, very unlikely to be 4+ + lines_to_keep = [] + lines_that_can_be_removed = [] + for line in line_list: + if self.can_line_be_removed(line, vertices_dict): + lines_that_can_be_removed.append(line) + else: + lines_to_keep.append(line) + + # if no line can be removed, we continue + if not lines_that_can_be_removed: + continue + + # there is only 1 line that can be removed and 2+ to be kept, so we remove it + if len(lines_that_can_be_removed) == 1: + lines_to_remove.append(lines_that_can_be_removed[0]) + continue + + # strange case where 3+ lines are together, but isolated from the rest of the world + # we decide to keep the longuest line + if len(lines_to_keep) == 0: + max_distance = max([line.length for line in lines_that_can_be_removed]) + for index_line, line in enumerate(lines_that_can_be_removed): + if line.length == max_distance: + lines_that_can_be_removed.pop(index_line) + lines_to_keep.append(line) + + # at least 2 lines can be removed, we will keep the one forming the straightest line with + # a line to keep, we assume it's the most likely to "continue" the river + min_dot_product = 1 + line_that_should_not_be_removed = None + for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): + if vertex == line_to_keep.boundary.geoms[0]: + vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + else: + vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + + if vertex == line_that_can_be_removed.boundary.geoms[0]: + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + else: + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + + length_to_keep = np.linalg.norm(vector_to_keep) + length_to_remove = np.linalg.norm(vector_to_remove) + if not length_to_keep or not length_to_remove: # length = 0, probably won't happen + continue + + vector_to_keep = vector_to_keep / length_to_keep + vector_to_remove = vector_to_remove / length_to_remove + dot_product = np.dot(vector_to_keep, vector_to_remove) + # the smallest dot product (which should be negative) is the straightest couple of lines + if dot_product < min_dot_product: + min_dot_product = dot_product + line_that_should_not_be_removed = line_that_can_be_removed + + lines_that_can_be_removed.remove(line_that_should_not_be_removed) + lines_to_remove += lines_that_can_be_removed + + # set the lines without the lines to remove + self.gdf_lines = self.gdf_lines[~self.gdf_lines['geometry'].isin(lines_to_remove)] + + def can_line_be_removed(self, line, vertices_dict): + if line.length > WATER_MIN_SIZE: + return False + + point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] + # if an extremity is used to close a gap, wde keep it + if point_a in self.gap_points or point_b in self.gap_points: + return False + + # if both extremities of the line is connected to another line, we can't remove that line + if len(vertices_dict[point_a]) > 1 and len(vertices_dict[point_b]) > 1: + return False + return True def __repr__(self): return str(self.branch_id) -def check_geometry(initial_gdf): - """garanty polygons' validity - """ - # create simple geometry - gdf_simple = initial_gdf.explode(ignore_index=True) # ignore_index makes the resulting index multi-indexed - gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) - # remove invalid polygons - gdf_valid = gdf_without_duplicates.copy() - gdf_valid.geometry = gdf_valid.geometry.apply( - lambda geom: fix_invalid_geometry(geom) - ) - return gdf_valid + def create_voronoi_lines(self): + """ Créer les lignes de Voronoi + """ + + # create_voronoi_lines(self.gdf_branch_mask, self.crs) + + # divide geometry into segments no longer than max_segment_length + segmentize_geom = (self.gdf_branch_mask['geometry'].unary_union).segmentize(max_segment_length=VORONOI_MAX_LENGTH) + # Create the voronoi diagram and only keep polygon + regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) + + ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### + geometry = gpd.GeoSeries(regions.geoms, crs=self.crs).explode(index_parts=False) + df = gpd.GeoDataFrame(geometry=geometry, crs=self.crs) + lines_filter = df.sjoin(self.gdf_branch_mask, predicate='within') # Opération spatiale "Within" + # Enregistrer le diagram de Voronoi (Squelette) + lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index + lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' + + return gpd.GeoDataFrame(lines_filter, crs=self.crs) + +def get_df_points_from_gdf(gdf:GeoDataFrame)->pd.DataFrame: + all_points = set() # we use a set instead of a list to remove doubles + for _, row in gdf.iterrows(): + unknown_geometry = row['geometry'] + if type(unknown_geometry) == Polygon: + line:MultiLineString = unknown_geometry.exterior + elif type(unknown_geometry) == MultiLineString: + line:MultiLineString = unknown_geometry + elif type(unknown_geometry) == LineString: + line:LineString = unknown_geometry + else : + raise NotImplementedError("Type unknown") + for coord in line.coords: + all_points.add(coord) + + all_points = list(all_points) + return pd.DataFrame(data={'x': [point[0] for point in all_points], + 'y': [point[1] for point in all_points],}) def fix_invalid_geometry(geometry): """ Fixer les géométries invalides @@ -147,106 +306,6 @@ def fix_invalid_geometry(geometry): return make_valid(geometry) else: return geometry - - -def create_voronoi_lines(mask_hydro, crs): - """ Créer les lignes de Voronoi - - Args: - -mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - - crs (str): code EPSG - - Returns : - out lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique - """ - # we don't work on an empty geometry - if not mask_hydro['geometry'].dtype == 'geometry': - return - - # divide geometry into segments no longer than max_segment_length - segmentize_geom = (mask_hydro['geometry'].unary_union).segmentize(max_segment_length=VORONOI_MAX_LENGTH) - # Create the voronoi diagram and only keep polygon - regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) - - ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### - geometry = gpd.GeoSeries(regions.geoms, crs=crs).explode(index_parts=False) - df = gpd.GeoDataFrame(geometry=geometry, crs=crs) - lines_filter = df.sjoin(mask_hydro, predicate='within') # Opération spatiale "Within" - # Enregistrer le diagram de Voronoi (Squelette) - lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index - lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' - - return gpd.GeoDataFrame(lines_filter, crs=crs) - -def remove_extra_lines(gdf_lines:GeoDataFrame)->GeoDataFrame: - vertices_dict = get_vertices_dict(gdf_lines) - - lines_to_remove = [] - for vertex, line_list in vertices_dict.items(): - if len(line_list) == 1 : - continue - - # if len(line_list) !is not 1, then it's probably 3; Can't be 2 because of the previous merge, very unlikely to be 4+ - lines_to_keep = [] - lines_that_can_be_removed = [] - for line in line_list: - if can_line_be_removed(line, vertices_dict): - lines_that_can_be_removed.append(line) - else: - lines_to_keep.append(line) - - # if no line can be removed, we continue - if not lines_that_can_be_removed: - continue - - - # there is only 1 line that can be removed and 2+ to be kept, so we remove it - if len(lines_that_can_be_removed) == 1: - lines_to_remove.append(lines_that_can_be_removed[0]) - continue - - # strange case where 3+ lines are together, but isolated from the rest of the world - # we decide to keep the longuest line - if len(lines_to_keep) == 0: - max_distance = max([line.length for line in lines_that_can_be_removed]) - for index_line, line in enumerate(lines_that_can_be_removed): - if line.length == max_distance: - lines_that_can_be_removed.pop(index_line) - lines_to_keep.append(line) - - # at least 2 lines can be removed, we will keep the one forming the straightest line with - # a line to keep, we assume it's the most likely to "continue" the river - min_dot_product = 1 - line_that_should_not_be_removed = None - for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): - if vertex == line_to_keep.boundary.geoms[0]: - vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) - else: - vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) - - if vertex == line_that_can_be_removed.boundary.geoms[0]: - vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) - else: - vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) - - length_to_keep = np.linalg.norm(vector_to_keep) - length_to_remove = np.linalg.norm(vector_to_remove) - if not length_to_keep or not length_to_remove: # length = 0, probably won't happen - continue - - vector_to_keep = vector_to_keep / length_to_keep - vector_to_remove = vector_to_remove / length_to_remove - dot_product = np.dot(vector_to_keep, vector_to_remove) - # the smallest dot product (which should be negative) is the straightest couple of lines - if dot_product < min_dot_product: - min_dot_product = dot_product - line_that_should_not_be_removed = line_that_can_be_removed - - lines_that_can_be_removed.remove(line_that_should_not_be_removed) - lines_to_remove += lines_that_can_be_removed - - # return the lines without the lines to remove - return gdf_lines[~gdf_lines['geometry'].isin(lines_to_remove)] def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: """ @@ -274,15 +333,6 @@ def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: vertices_dict[point_b] = [line] return vertices_dict -def can_line_be_removed(line, vertices_dict): - if line.length > WATER_MIN_SIZE: - return False - - point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] - if len(vertices_dict[point_a]) > 1 and len(vertices_dict[point_b]) > 1: - return False - return True - def line_merge(voronoi_lines, crs): """Fusionner tous les LINESTRING en un seul objet MultiLineString diff --git a/skeleton.py b/skeleton.py index 3804f1a6..308290ee 100644 --- a/skeleton.py +++ b/skeleton.py @@ -14,7 +14,7 @@ from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate import logging -from branch import Branch, Candidate, BRIDGE_MAX_WIDTH +from branch import Branch, Candidate, GAP_MAX_WIDTH MASK_GEOJSON_1 = "/home/MDaab/data/squelette_test/MaskHydro_merge 1.geojson" MASK_GEOJSON_2 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm2.geojson" @@ -377,7 +377,7 @@ def connect_extremities(gdf_lines:GeoDataFrame, crs)->GeoDataFrame: # get all close enough vertices df_vertex['square_distance'] = (df_vertex['x'] - vertex_1.x)**2 + (df_vertex['y'] - vertex_1.y)**2 - df_close_enough = df_vertex[df_vertex['square_distance'] < BRIDGE_MAX_WIDTH**2] + df_close_enough = df_vertex[df_vertex['square_distance'] < GAP_MAX_WIDTH**2] for index, extremity_row in df_close_enough.iterrows(): # we can connect only vertex with a previous one in the list @@ -687,7 +687,7 @@ def select_candidates(branches_pair_list:List[Tuple])->List[Candidate]: # we don't create link between 2 branches already connected if branch_group.are_together(branch_a, branch_b): continue - candidates = branch_a.get_bridge_candidates(branch_b) + candidates = branch_a.get_gap_candidates(branch_b) # test each candidate to see if we can draw a line for them nb_bridges_crossed = 0 @@ -725,14 +725,15 @@ def run(gdf_mask_hydro, crs): for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): mask_branch = branch_mask_row["geometry"] new_branch = Branch(index_branch, mask_branch, crs) - new_branch.simplify() + # new_branch.simplify() branches_list.append(new_branch) + # create branches_pair_list, that stores all pairs of branches close enough to have a bridge branches_pair_list = [] for index, branch_a in enumerate(branches_list[:-1]): for branch_b in branches_list[index + 1:]: distance = branch_a.distance_to_a_branch(branch_b) - if distance < BRIDGE_MAX_WIDTH: + if distance < GAP_MAX_WIDTH: branches_pair_list.append((branch_a, branch_b, distance)) # sort the branches pairs by distance between them @@ -740,9 +741,19 @@ def run(gdf_mask_hydro, crs): validated_candidates = select_candidates(branches_pair_list) + # create the bridge lines from the selected candidates bridge_lines = [validated_candidate.line for validated_candidate in validated_candidates] gdf_bridge_lines = gpd.GeoDataFrame(geometry=bridge_lines).set_crs(crs, allow_override=True) - gdf_bridge_lines.to_file("test_bridges_group_maker.geojson", driver='GeoJSON') + gdf_bridge_lines.to_file("test_bridges_group_maker_mask.geojson", driver='GeoJSON') + + for candidate in validated_candidates: + candidate.branch_1.gap_points.append(Point(candidate.extremity_1)) + candidate.branch_2.gap_points.append(Point(candidate.extremity_2)) + + for branch in branches_list: + branch:Branch + branch.create_skeleton() + branch.simplify() # for branch in branches_list: # branch:Branch @@ -762,11 +773,11 @@ def run(gdf_mask_hydro, crs): branch_lines_list = [branch.gdf_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - gdf_branch_lines.to_file("test_branch_lines.geojson", driver='GeoJSON') + gdf_branch_lines.to_file("test_branch_lines_mask.geojson", driver='GeoJSON') mask_list = [branch.gdf_branch_mask for branch in branches_list] gdf_mask = gpd.GeoDataFrame(pd.concat(mask_list, ignore_index=True)) - gdf_mask.to_file("test_mask.geojson", driver='GeoJSON') + gdf_mask.to_file("test_mask_mask.geojson", driver='GeoJSON') # gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) From 822403b877a0a7bc23cb745cf36461c53760c2ac Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 25 Jun 2024 11:52:30 +0200 Subject: [PATCH 08/68] closing gaps by proximity to mask create and keep lines to points on mask used to close gaps --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 3d43c141..490969a9 100644 --- a/environment.yml +++ b/environment.yml @@ -16,6 +16,7 @@ dependencies: - pdal>=2.6 - python-pdal>=3.2.1 - shapely>=2.0.3 + - psycopg2-binary # --------- hydra configs --------- # - hydra-core - hydra-colorlog From 25fe0c156ece918b4319a75f5d576394b56b62a2 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 28 Jun 2024 10:53:36 +0200 Subject: [PATCH 09/68] whole skeleton process works, no test yet --- skeleton.py | 881 -------------------------------- branch.py => skeleton/branch.py | 256 +++++----- skeleton/group_maker.py | 22 + skeleton/main_skeleton.py | 163 ++++++ test/__init__.py | 0 5 files changed, 303 insertions(+), 1019 deletions(-) delete mode 100644 skeleton.py rename branch.py => skeleton/branch.py (57%) create mode 100644 skeleton/group_maker.py create mode 100644 skeleton/main_skeleton.py create mode 100644 test/__init__.py diff --git a/skeleton.py b/skeleton.py deleted file mode 100644 index 308290ee..00000000 --- a/skeleton.py +++ /dev/null @@ -1,881 +0,0 @@ -from itertools import product -from typing import Dict, List, Tuple - -import geopandas as gpd -from geopandas.geodataframe import GeoDataFrame -import pandas as pd -import numpy as np -from math import sqrt -import psycopg2 -from psycopg2.extensions import cursor -from shapely import LineString, Point, geometry, distance -from shapely.geometry import CAP_STYLE, Polygon, MultiPoint, Point, MultiLineString -from shapely.validation import make_valid -from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate -import logging - -from branch import Branch, Candidate, GAP_MAX_WIDTH - -MASK_GEOJSON_1 = "/home/MDaab/data/squelette_test/MaskHydro_merge 1.geojson" -MASK_GEOJSON_2 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm2.geojson" -MASK_GEOJSON_3 = "/home/MDaab/data/squelette_test/ecoulement_brut_pm3.geojson" -MASK_GEOJSON_4 = "/home/MDaab/data/squelette_test/ecoulement_brut_lg1.geojson" -MASK_GEOJSON_5 = "/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson" - -MIDDLE_FILE = "squelette_filtrer.geojson" -SAVE_FILE = "squelette_hydrographique.geojson" -WATER_MIN_SIZE = 20 -VORONOI_MAX_LENGTH = 2 -MAX_BRIDGES = 1 - -DB_NAME = "bduni_france_consultation" -DB_HOST = "bduni_consult.ign.fr" -DB_USER = "invite" -DB_PASSWORD = "28de#" -DB_PORT = "5432" - - -class GroupMaker: - def __init__(self, element_list): - self.element_set_list = [] - for element in element_list: - self.element_set_list.append({element}) - - def find_index(self, element) -> int: - for index, element_set in enumerate(self.element_set_list): - if element in element_set: - return index - - def are_together(self, element_a, element_b)->bool: - """return true if 2 elements are already together""" - index = self.find_index(element_a) - return element_b in self.element_set_list[index] - - def put_together(self, element_a, element_b): - """put the set of a and b together""" - index_a = self.find_index(element_a) - set_a = self.element_set_list.pop(index_a) - index_b = self.find_index(element_b) - self.element_set_list[index_b] = self.element_set_list[index_b].union(set_a) - -def db_connector(): - """Return a connector to the postgis database""" - return psycopg2.connect(database=DB_NAME, - host=DB_HOST, - user=DB_USER, - password=DB_PASSWORD, - port=DB_PORT) - -def query_db_for_bridge(candidate:Candidate)->bool: - """ - Query the database to check if a candidate for a bridge indeed intersects a bridge - """ - line = f"LINESTRING({candidate.extremity_1[0]} {candidate.extremity_1[1]}, {candidate.extremity_2[0]} {candidate.extremity_2[1]})" - query_linear = f"SELECT cleabs FROM public.Construction_lineaire WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" - query_area = f"SELECT cleabs FROM public.Construction_surfacique WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" - - with db_connector() as db_conn: - with db_conn.cursor() as db_cursor: - db_cursor.execute(query_linear) - results = db_cursor.fetchall() - # if no result with linear bridge, maybe with area bridge... - if not results: - db_cursor.execute(query_area) - results = db_cursor.fetchall() - return True if results else False - - - - -def fix_invalid_geometry(geometry): - """ Fixer les géométries invalides - """ - if not geometry.is_valid: - return make_valid(geometry) - else: - return geometry - - -def check_geometry(initial_gdf): - """garanty polygons' validity - """ - # create simple geometry - gdf_simple = initial_gdf.explode(ignore_index=True) # ignore_index makes the resulting index multi-indexed - gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) - # remove invalid polygons - gdf_valid = gdf_without_duplicates.copy() - gdf_valid.geometry = gdf_valid.geometry.apply( - lambda geom: fix_invalid_geometry(geom) - ) - return gdf_valid - - -def filter_branch(df, crs): - """Filter branches from voronoi_lines to extract hydrographic section - - Args: - -df (GeoDataframe) : Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique - - crs (str): code EPSG - - Returns : - out (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique avec leurs occurences - """ - ### Étape 1 : Obtenez les limites de chaque ligne - boundaries = df.boundary - # df.reset_index() - df["id"] = df.index - - ### Étape 2 : Explosez les limites pour avoir une ligne par limite - exploded_boundaries = boundaries.explode(index_parts=False, ignore_index=False) - # Créez un DataFrame à partir des limites explosées avec une colonne 'geometry' - boundary_df = gpd.GeoDataFrame(geometry=exploded_boundaries) - - ### Étape 3 : Utilisez groupby pour compter le nombre d'occurrences de chaque points - boundary_df["id"] = boundary_df.index - boundary_df["wkt"] = boundary_df["geometry"].to_wkt() - occurences_df = boundary_df.groupby(["geometry"], dropna=True, as_index=False).agg({'id': [lambda x: x.iloc[0], 'count']}) - occurences_df.columns = ['geometry', 'id', 'occurences'] - - ### Étape 4 : Obtenir les occurences de chaque lignes de Voronoi - # Merger les points par occurences avec leur geometry - points_merge = pd.merge(boundary_df, occurences_df, on='geometry').drop(columns=['wkt', 'id_y']).drop_duplicates() - points_merge.columns = ['geometry', 'id', 'occurences'] - points_merge = gpd.GeoDataFrame(points_merge, crs=crs) - # Intersecter les points avec les lignes de Voronoi - points_intersect_lines = df.sjoin(points_merge, predicate='intersects') - # Grouper les lignes par index "id_left" et calculer les occurences "minimum" et "maximum" - out = points_intersect_lines.groupby('id_left', dropna=True).agg({'occurences': ['min'], 'geometry': [lambda x: x.iloc[0]]}) - out.columns = ['occurence_min', 'geometry'] - out = gpd.GeoDataFrame(out, crs=crs) - - return out - - -def can_line_be_removed(line, vertices_dict): - if line.length > WATER_MIN_SIZE: - return False - - point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] - if len(vertices_dict[point_a]) > 1 and len(vertices_dict[point_b]) > 1: - return False - return True - - -def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: - """ - get a dictionary of vertices listing all the lines having a specific vertex as one of its extremities - Args: - - gdf_lines:geodataframe containing a list of lines - return: - - a dict with the vertices are the keys and the values are lists of lines - """ - # prepare a vertice dict, containing all the lines connected on a same vertex - vertices_dict = {} - for index in gdf_lines.index : - line = gdf_lines.iloc[index]['geometry'] - if line.is_ring: # it's a loop : no extremity - continue - - point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] - try : - vertices_dict[point_a].append(line) - except KeyError: - vertices_dict[point_a] = [line] - try : - vertices_dict[point_b].append(line) - except KeyError: - vertices_dict[point_b] = [line] - return vertices_dict - -def remove_extra_lines(gdf_lines:GeoDataFrame)->GeoDataFrame: - vertices_dict = get_vertices_dict(gdf_lines) - - lines_to_remove = [] - for vertex, line_list in vertices_dict.items(): - if len(line_list) == 1 : - continue - - # if len(line_list) !is not 1, then it's probably 3; Can't be 2 because of the previous merge, very unlikely to be 4+ - lines_to_keep = [] - lines_that_can_be_removed = [] - for line in line_list: - if can_line_be_removed(line, vertices_dict): - lines_that_can_be_removed.append(line) - else: - lines_to_keep.append(line) - - # if no line can be removed, we continue - if not lines_that_can_be_removed: - continue - - # strange case where 3+ lines are together, but isolated from the rest of the world - if len(lines_to_keep) == 0: - lines_to_remove += lines_that_can_be_removed - continue - - # there is only 1 line that can be removed and 2+ to be kept, so we remove it - if len(lines_that_can_be_removed) == 1: - lines_to_remove.append(lines_that_can_be_removed[0]) - continue - - # at least 2 lines can be removed, we will keep the one forming the straightest line with - # a line to keep, we assume it's the most likely to "continue" the river - min_dot_product = 1 - line_that_should_not_be_removed = None - for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): - if vertex == line_to_keep.boundary.geoms[0]: - vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) - else: - vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) - - if vertex == line_that_can_be_removed.boundary.geoms[0]: - vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) - else: - vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) - - length_to_keep = np.linalg.norm(vector_to_keep) - length_to_remove = np.linalg.norm(vector_to_remove) - if not length_to_keep or not length_to_remove: # length = 0, probably won't happen - continue - - vector_to_keep = vector_to_keep / length_to_keep - vector_to_remove = vector_to_remove / length_to_remove - dot_product = np.dot(vector_to_keep, vector_to_remove) - # the smallest dot product (which should be negative) is the straightest couple of lines - if dot_product < min_dot_product: - min_dot_product = dot_product - line_that_should_not_be_removed = line_that_can_be_removed - - lines_that_can_be_removed.remove(line_that_should_not_be_removed) - lines_to_remove += lines_that_can_be_removed - - # return the lines without the lines to remove - return gdf_lines[~gdf_lines['geometry'].isin(lines_to_remove)] - -def line_merge(voronoi_lines): - """Fusionner tous les LINESTRING en un seul objet MultiLineString - - Args: - - voronoi_lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique - - Returns: - lines (GeoDataframe): Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique - - """ - # Fusionner tous les LINESTRING en un seul objet MultiLineString - merged_line = voronoi_lines.geometry.unary_union - # Appliquer un algo de fusion des lignes (ex. "ST_LineMerge") sur l'objet MultiLineString - merged_line = linemerge(merged_line) - - # turn multipart geometries into multiple single geometries - geometry = gpd.GeoSeries(merged_line.geoms, crs=crs).explode(index_parts=False) - lines = gpd.GeoDataFrame(geometry=geometry, crs=crs) - - return lines - - -def create_voronoi_lines(mask_hydro, crs): - """ Créer les lignes de Voronoi - - Args: - -mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - - crs (str): code EPSG - - Returns : - out lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique - """ - # we don't work on an empty geometry - if not mask_hydro['geometry'].dtype == 'geometry': - return - - # divide geometry into segments no longer than max_segment_length - segmentize_geom = (mask_hydro['geometry'].unary_union).segmentize(max_segment_length=VORONOI_MAX_LENGTH) - # Create the voronoi diagram and only keep polygon - regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) - - ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### - geometry = gpd.GeoSeries(regions.geoms, crs=crs).explode(index_parts=False) - df = gpd.GeoDataFrame(geometry=geometry, crs=crs) - lines_filter = df.sjoin(mask_hydro, predicate='within') # Opération spatiale "Within" - # Enregistrer le diagram de Voronoi (Squelette) - lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index - lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' - - return gpd.GeoDataFrame(lines_filter, crs=crs) - -def get_prior_point(vertex, line) -> Tuple[float, float]: - """ return the point right before an extremity vertex in a line - """ - if vertex == line.boundary.geoms[0]: - prior_point = line.coords[2] - else: - prior_point = line.coords[-2] - return prior_point - -def connect_extremities(gdf_lines:GeoDataFrame, crs)->GeoDataFrame: - """ - connect extremities of rivers, as we assume they are disconnected because of bridges - Args: - - gdf_lines: geodataframe containing a list of lines - return: - The same geodataframe with connection between extremities - """ - - # garanty that all lines have at least 3 segments, because the connexion is not - # done on the last point, but the previous one. Therefore we need enough points - new_lines = [] - for index, row in gdf_lines.iterrows(): - line = row['geometry'] - if len(line.coords) < 4: # 3 segments, so 4 points - line = line.segmentize(line.length/3) - new_lines.append(line) - gdf_lines = gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) - - # for index, row in gdf_lines.iterrows(): - # line = row['geometry'] - # if len(line.coords) < 4: # 3 segments, so 4 points - # row = line.segmentize(line.length/3) - - vertices_dict = get_vertices_dict(gdf_lines) - # we remove all vertices that are not extremities (an extremity is connected to only one line) - keys_to_remove = [] - for vertex, line_list in vertices_dict.items(): - if len(line_list) != 1: - keys_to_remove.append(vertex) - for vertex in keys_to_remove: - vertices_dict.pop(vertex, None) - - ### - # WARNING !!! VISUALIZATION OF EXTREMITIES, TO REMOVE - geom = [vertex for vertex in vertices_dict.keys()] - test = gpd.GeoDataFrame(geometry=geom).set_crs(crs, allow_override=True) - test.to_file("test_extremities.geojson", driver='GeoJSON') - ### - - ### - # WARNING !!! VISUALIZATION OF PRIOR POINTS, TO REMOVE - geom = [Point(*get_prior_point(vertex, line[0])) for vertex, line in vertices_dict.items()] - test = gpd.GeoDataFrame(geometry=geom).set_crs(crs, allow_override=True) - test.to_file("test_prior_points.geojson", driver='GeoJSON') - ### - - new_lines = [] - # create a dataframe from the vertices (to make calculation faster) - data = {'x':[vertex.x for vertex in vertices_dict], - 'y':[vertex.y for vertex in vertices_dict], - 'vertex': vertices_dict.keys(), - 'line': [lines[0] for lines in vertices_dict.values()]} - df_vertex = pd.DataFrame(data=data) - - best_prior_points_to_connect = [] - for vertex_1, [line_1] in vertices_dict.items(): - prior_point_1 = get_prior_point(vertex_1, line_1) - - best_dot_value = 0 - best_prior_point = None - best_candidate = None - - # get all close enough vertices - df_vertex['square_distance'] = (df_vertex['x'] - vertex_1.x)**2 + (df_vertex['y'] - vertex_1.y)**2 - df_close_enough = df_vertex[df_vertex['square_distance'] < GAP_MAX_WIDTH**2] - for index, extremity_row in df_close_enough.iterrows(): - - # we can connect only vertex with a previous one in the list - # if we allow the connection also with next vertices, the connection would be made twice - vertex_2 = extremity_row['vertex'] - if vertex_2 == vertex_1: - break - - prior_point_2 = get_prior_point(vertex_2, extremity_row['line']) - x_1 = prior_point_1[0] - vertex_1.x - y_1 = prior_point_1[1] - vertex_1.y - x_2 = prior_point_2[0] - vertex_2.x - y_2 = prior_point_2[1] - vertex_2.y - - length_1 = sqrt(x_1**2 + y_1**2) - length_2 = sqrt(x_2**2 + y_2**2) - if not length_1 or not length_2: # a vector is null - continue - - normalized_dot_product = x_1 * x_2 + y_1 * y_2 / (length_1 * length_2) - # if they are anticolinear enough, we keep it - if normalized_dot_product < best_dot_value : - best_dot_value = normalized_dot_product - best_prior_point = prior_point_2 - best_candidate = index - - if best_candidate: - best_prior_points_to_connect.append([prior_point_1, best_prior_point]) - - # search where both prior points are the best for each other - # for index, prior_points_pair in enumerate(best_prior_points_to_connect[:-1]): - # for opposite_prior_points_pair in best_prior_points_to_connect[index + 1:]: - for index, prior_points_pair in enumerate(best_prior_points_to_connect): - for opposite_prior_points_pair in best_prior_points_to_connect: - if prior_points_pair[0] == opposite_prior_points_pair[1] \ - and prior_points_pair[1] == opposite_prior_points_pair[0]: - new_lines.append(LineString(opposite_prior_points_pair)) - - return gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) - - # remove a part of a line when it had been extended, to make it smoother - # for index, row in gdf_lines.iterrows(): - # line = row['geometry'] - # start = 0 - # end = len(line.coords) - # if line.coords[2] in prior_points_connected: - # start = 2 - # # new_lines.append(LineString(line.coords[0:2])) - # if line.coords[-2] in prior_points_connected: - # end = -2 - # # new_lines.append(LineString(line.coords[-2:])) - # new_lines.append(LineString(line.coords[start:end])) - - # df_close_enough = df_vertex[(df_vertex['x'] - vertex_1.x)**2 + (df_vertex['y'] - vertex_1.y)**2) < BRIDGE_MAX_WIDTH] - - - - # for vertex_2, [line_2] in vertices_dict.items(): - # # if we have already done the calculation for vertex_1, vertex_2, no need to do it on vertex_2, vertex_1 - # if vertex_2 in vertice_done: - # continue - - # # check if the distance between both vertex is small enough for a bridge - # if vertex_1.distance(vertex_2) > BRIDGE_MAX_WIDTH: - # continue - - # # check how anticolinear 1 and 2 are - # prior_point_2 = get_prior_point(vertex_2, line_2) - # x_1 = prior_point_1[0] - vertex_1.x - # y_1 = prior_point_1[1] - vertex_1.y - # x_2 = prior_point_2[0] - vertex_2.x - # y_2 = prior_point_2[1] - vertex_2.y - - # length_1 = sqrt(x_1**2 + y_1**2) - # length_2 = sqrt(x_2**2 + y_2**2) - # if not length_1 or not length_2: # a vector is null - # continue - - # normalized_dot_product = x_1 * x_2 + y_1 * y_2 / (length_1 * length_2) - - # # if they are anticolinear enough, we keep it to draw a line later - # if normalized_dot_product < best_dot_value : - # best_dot_value = normalized_dot_product - # best_candidate = prior_point_2 - - # # if there is a candidate to draw a line (i.e : river under a bridge), we draw it - # if best_candidate: - # new_lines.append(LineString([prior_point_1, best_candidate])) - - # best_candidate = {} - # for vertex, [line] in vertices_dict.items(): - # prior_point = get_prior_point(vertex, line) - - # # get the perpendicular line passing throught vertex, equation ax + by + c = 0 - # a, b = vertex.coords[0][0] - prior_point[0], vertex.coords[0][1] - prior_point[1] - # c = - a * vertex.coords[0][0] - b * vertex.coords[0][1] - - # # make sure the prior_point is below the line (if above, we multiply the equation by -1 to change it) - # if not a * prior_point[0] + b * prior_point[1] + c <= 0: - # a, b, c = -a, -b, -c - - # # search best vertex candidate to join this vertex i.e: - # # the closest, not too far, vertex above the perpendicular line (to be "in front" of the river) - # for other_vertex, [other_line] in vertices_dict.items(): - # if other_vertex == vertex: - # continue - # # have to be above the perpendicular line - # if a * other_vertex.coords[0][0] + b * other_vertex.coords[0][1] + c <= 0: - # continue - # # should not be too far: - # squared_distance = (vertex.coords[0][0] - other_vertex.coords[0][0])**2 + (vertex.coords[0][1] - other_vertex.coords[0][1])**2 - # if squared_distance > BRIDGE_MAX_WIDTH**2: - # continue - # other_prior_point = get_prior_point(other_vertex, other_line) - # best_candidate[prior_point] = other_prior_point - - # new_lines = [] - # already_done_prior_points = set() - # # if 2 prior_points have each other as best candidate, we create a line between them - # for prior_point, other_prior_point in best_candidate.items(): - # if best_candidate[other_prior_point] == prior_point and other_prior_point not in already_done_prior_points: - # already_done_prior_points.add(prior_point) - # already_done_prior_points.add(other_prior_point) - # # gdf_lines.loc[len(gdf_lines)] = [LineString([prior_point, other_prior_point])] - # new_lines.append(LineString([prior_point, other_prior_point])) - - - # remove a part of the line, to make it smoother - # for index in gdf_lines.index: - # line = gdf_lines.iloc[index]['geometry'] - # start = 0 - # end = len(line.coords) - # if line.coords[2] in already_done_prior_points: - # start = 2 - # # new_lines.append(LineString(line.coords[0:2])) - # if line.coords[-2] in already_done_prior_points: - # end = -2 - # # new_lines.append(LineString(line.coords[-2:])) - # new_lines.append(LineString(line.coords[start:end])) - - # gdf_cutlines = gpd.GeoDataFrame(geometry=cut_lines) - - # return gpd.GeoDataFrame(geometry=new_lines).set_crs(crs, allow_override=True) - # gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs - # return - - -def add_a_line_on_each_loop(gdf_lines:GeoDataFrame, crs:int)->GeoDataFrame: - # separates loops and non loops - gdf_loops = gdf_lines[gdf_lines['geometry'].is_ring] - gdf_no_loops = gdf_lines[~gdf_lines['geometry'].is_ring] - - added_opposite_lines = [] - loop_cut_lines = [] - index_row_to_remove = [] - for index_loop, row_loop in gdf_loops.iterrows(): - points_on_loop = MultiPoint(list(row_loop['geometry'].coords)) - # get the line of gdf_lines that join the loop: - lines_joining_loop = gdf_no_loops[~gdf_no_loops['geometry'].disjoint(points_on_loop)].reset_index() - if len(lines_joining_loop) != 1: # we assume that, if there are 2+ lines connected to the loop, - continue # there is an "in and an "out", we don't need to try and connect the loop under a bridge - # If there are none, we can't do anything - - # we assume that the bridge is more or less in the same direction than the line joining the loop, - # and at the opposite side of the loop - line_to_continue = lines_joining_loop.loc[0]['geometry'] - - # search the extremity connected to the loop - if row_loop['geometry'].disjoint(line_to_continue.boundary.geoms[0]): - connected_extremity = line_to_continue.boundary.geoms[1] - other_extremity = line_to_continue.boundary.geoms[0] - elif row_loop['geometry'].disjoint(line_to_continue.boundary.geoms[1]): - connected_extremity = line_to_continue.boundary.geoms[0] - other_extremity = line_to_continue.boundary.geoms[1] - else: # should not happen, an extremity should be on the loop - continue - - # search the most distant point on the loop to the connected extremity, we assume - # it's where the river should continue - max_distance = 0 - opposite_connected_extremity = None - for coordinate in row_loop['geometry'].coords: - point = Point(coordinate) - current_distance = distance(point, connected_extremity) - if current_distance > max_distance: - max_distance = current_distance - opposite_connected_extremity = point - - # create a line, lenght 1m, that starts at the opposite point and goes the opposite way - length_connected_line = LineString([connected_extremity, other_extremity]).length - x = opposite_connected_extremity.x - (other_extremity.x - connected_extremity.x) / length_connected_line - y = opposite_connected_extremity.y - (other_extremity.y - connected_extremity.y) / length_connected_line - opposite_extremity_point = Point(x, y) - opposite_line = LineString([opposite_connected_extremity, opposite_extremity_point]) - added_opposite_lines.append(opposite_line) - - # dvide the loop in 2 lines, with extremities = [connected_extremity, opposite_connected_extremity] - index_row_to_remove.append(index_loop) - for index_point_loop, (x_point_loop, y_point_loop) in enumerate(list(row_loop['geometry'].coords)): - if x_point_loop == connected_extremity.x and y_point_loop == connected_extremity.y: - index_connected = index_point_loop - if x_point_loop == opposite_connected_extremity.x and y_point_loop == opposite_connected_extremity.y: - index_opposite_connected = index_point_loop - - if index_connected > index_opposite_connected: - index_connected, index_opposite_connected = index_opposite_connected, index_connected - - # create both lines for the loop cut - first_line = LineString(row_loop['geometry'].coords[index_connected:index_opposite_connected + 1]) - second_line = LineString(row_loop['geometry'].coords[index_opposite_connected:] + row_loop['geometry'].coords[:index_connected + 1]) - loop_cut_lines.append(first_line) - loop_cut_lines.append(second_line) - - gdf_lines = gdf_lines.drop(index = index_row_to_remove) # initial lines without the cut loops - gdf_new_loop_cut_lines = gpd.GeoDataFrame(geometry=loop_cut_lines).set_crs(crs, allow_override=True) # pairs of lines for the cut loops - gdf_new_opposite_lines = gpd.GeoDataFrame(geometry=added_opposite_lines).set_crs(crs, allow_override=True) # line added to the loop - - return gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_new_opposite_lines, gdf_new_loop_cut_lines], ignore_index=True)) - -def replace_lines_on_loops(gdf_lines:GeoDataFrame, gdf_divided_lines:GeoDataFrame, crs:int)->GeoDataFrame: - # separates loops and non loops - gdf_loops = gdf_lines[gdf_lines['geometry'].is_ring].reset_index() - gdf_no_loops = gdf_lines[~gdf_lines['geometry'].is_ring].reset_index() - - # gets all points on loops - points_on_loops = [] - for index in gdf_loops.index: - points_on_loops += list(gdf_loops.iloc[index]['geometry'].coords) - points_on_loops = MultiPoint(points_on_loops) - - # get lines with at least 1 point on a loop - commun_lines = gdf_divided_lines[~gdf_divided_lines['geometry'].disjoint(points_on_loops)].reset_index() - - # replace loops with those lines - return gpd.GeoDataFrame(pd.concat([gdf_no_loops, commun_lines], ignore_index=True)) - - # # get lines with one point, and one point only, on loops - # partially_on_loop_lines = [] - # for index in commun_lines.index: - # coords = commun_lines.iloc[index]['geometry'].coords - # completely_covered = False - # for index_loop in gdf_loops.index: - # for begin_loop, end_loop in zip(gdf_loops.iloc[index_loop]['geometry'].coords[:-1], gdf_loops.iloc[index_loop]['geometry'].coords[1:]): - # if (begin_loop in coords) and (end_loop in coords): - # completely_covered = True - # break - # if completely_covered: - # break - # if not completely_covered: - # partially_on_loop_lines.append(commun_lines.iloc[index]['geometry']) - - # return gpd.GeoDataFrame(geometry=partially_on_loop_lines, crs=crs) - # gdf_connected_lines.to_file("test_connected_lines.geojson", driver='GeoJSON') - - # assembled = gpd.GeoDataFrame(pd.concat([gdf_connected_lines, gdf_lines], ignore_index=True)) - # assembled.to_file("test_assembled.geojson", driver='GeoJSON') # premerge - - # test = commun_lines[any(commun_lines['geometry'].covered_by(gdf_lines))] - # new_lines_excluded = [] - - # test = gdf_lines.unary_union - - # for index in commun_lines.index: - # if not any(commun_lines.iloc[index]['geometry'].covered_by(gdf_lines)): - # new_lines_excluded.append(commun_lines.iloc[index]['geometry']) - - - # geometry = gpd.GeoSeries(regions.geoms, crs=crs).explode(index_parts=False) - # df = gpd.GeoDataFrame(geometry=geometry, crs=crs) - - # gpd.GeoDataFrame(new_lines_excluded, crs=2154).to_file("test_new_lines_excluded.geojson", driver='GeoJSON') - -def get_branches(gdf_mask_hydro:GeoDataFrame, crs:int)->List[GeoDataFrame]: - """ create branches : a branch is a bunch of lines of river that are all connected""" - gdf_branches_list = [] - for index_branch, mask_branch_row in gdf_mask_hydro.iterrows(): - mask_branch = mask_branch_row["geometry"] - simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker - gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) - - gdf = check_geometry(gdf_branch_mask) # Vérifier et corriger les géométries non valides - - ### Voronoi Diagram ### - voronoi_lines = create_voronoi_lines(gdf, crs) - gdf_lines = line_merge(voronoi_lines) - # gdf_lines.to_file(f"branch_{index_branch}.geojson", driver='GeoJSON') - gdf_branches_list.append(gdf_lines) - pass - - -def select_candidates(branches_pair_list:List[Tuple])->List[Candidate]: - """ - selects candidates between pairs of branches - """ - # get all the branches (each only once, hence the set) - branch_set = set() - for branch_a, branch_b, _ in branches_pair_list: - branch_set.add(branch_a) - branch_set.add(branch_b) - - branch_group = GroupMaker(list(branch_set)) - validated_candidates = [] - extremities_connected = set() - for branch_a, branch_b, _ in branches_pair_list: - branch_a: Branch - - # we don't create link between 2 branches already connected - if branch_group.are_together(branch_a, branch_b): - continue - candidates = branch_a.get_gap_candidates(branch_b) - - # test each candidate to see if we can draw a line for them - nb_bridges_crossed = 0 - for candidate in candidates : - # we connect each extremity only once - if candidate.extremity_1 in extremities_connected or candidate.extremity_2 in extremities_connected : - continue - - # check with database - # is_bridge = query_db_for_bridge(candidate) - # # if the line does not cross any bridge, we don't validate that candidate - # if not is_bridge: - # continue - - # candidate validated - extremities_connected.add(candidate.extremity_1) - extremities_connected.add(candidate.extremity_2) - validated_candidates.append(candidate) - branch_group.put_together(branch_a, branch_b) - nb_bridges_crossed += 1 - if nb_bridges_crossed >= MAX_BRIDGES: # max bridges reached between those 2 branches - break - return validated_candidates - - -def run(gdf_mask_hydro, crs): - """Calculer le squelette hydrographique - - Args: - - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - - crs (str): code EPSG - """ - # create branches - branches_list = [] - for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): - mask_branch = branch_mask_row["geometry"] - new_branch = Branch(index_branch, mask_branch, crs) - # new_branch.simplify() - branches_list.append(new_branch) - - # create branches_pair_list, that stores all pairs of branches close enough to have a bridge - branches_pair_list = [] - for index, branch_a in enumerate(branches_list[:-1]): - for branch_b in branches_list[index + 1:]: - distance = branch_a.distance_to_a_branch(branch_b) - if distance < GAP_MAX_WIDTH: - branches_pair_list.append((branch_a, branch_b, distance)) - - # sort the branches pairs by distance between them - branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) - - validated_candidates = select_candidates(branches_pair_list) - - # create the bridge lines from the selected candidates - bridge_lines = [validated_candidate.line for validated_candidate in validated_candidates] - gdf_bridge_lines = gpd.GeoDataFrame(geometry=bridge_lines).set_crs(crs, allow_override=True) - gdf_bridge_lines.to_file("test_bridges_group_maker_mask.geojson", driver='GeoJSON') - - for candidate in validated_candidates: - candidate.branch_1.gap_points.append(Point(candidate.extremity_1)) - candidate.branch_2.gap_points.append(Point(candidate.extremity_2)) - - for branch in branches_list: - branch:Branch - branch.create_skeleton() - branch.simplify() - - # for branch in branches_list: - # branch:Branch - # gdf_branch_lines = gpd.GeoDataFrame(branch.gdf_lines).set_crs(crs, allow_override=True) - # gdf_branch_lines.to_file(f"{branch.branch_id}.geojson", driver='GeoJSON') - - # branch_lines_list = gpd.GeoDataFrame() - # for branch in branches_list: - # branch:Branch - # data = {'branch_id': branch.branch_id, 'geometry': branch.gdf_lines} - # # data = {'branch_id': [branch.branch_id for _ in range(len(branch.gdf_lines))], 'geometry': branch.gdf_lines} - # gdf_branch_lines = gpd.GeoDataFrame(data).set_crs(crs, allow_override=True) - # branch_lines_list = gpd.GeoDataFrame(pd.concat([branch_lines_list, gdf_branch_lines], ignore_index=True)) - # # gdf_branch_lines.to_file(f"{branch.branch_id}.geojson", driver='GeoJSON') - - # branch_lines_list.to_file("test_branch_lines.geojson", driver='GeoJSON') - - branch_lines_list = [branch.gdf_lines for branch in branches_list] - gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - gdf_branch_lines.to_file("test_branch_lines_mask.geojson", driver='GeoJSON') - - mask_list = [branch.gdf_branch_mask for branch in branches_list] - gdf_mask = gpd.GeoDataFrame(pd.concat(mask_list, ignore_index=True)) - gdf_mask.to_file("test_mask_mask.geojson", driver='GeoJSON') - - # gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) - - - # validated_candidates - # gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_final.geojson", driver='GeoJSON') - - pass - # gdf_inbetween_lines = GeoDataFrame(geometry=inbetween_lines, crs=crs) - # gdf_inbetween_lines.to_file("test_bridge.geojson", driver='GeoJSON') - - - # query_list = "" - # for index, candidate in enumerate(candidates): - # candidate:Candidate - # geometry = f"LINESTRING({candidate.extremity_1[0]} {candidate.extremity_1[1]}, {candidate.extremity_2[0]} {candidate.extremity_2[1]})" - # query = f"SELECT cleabs FROM public.Construction_lineaire WHERE gcms_detruit = false AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{geometry}'));" - # query_list += query - # if index >= 5: - # break - - # db_cursor.execute(query_list) - # results = db_cursor.fetchall() - -def run_branch(mask_hydro, crs): - - get_branches(mask_hydro, crs) - pass - -def run_old(mask_hydro, crs): - """Calculer le squelette hydrographique - - Args: - - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - - crs (str): code EPSG - """ - - # get_branches(mask_hydro, crs) - - gdf = check_geometry(mask_hydro) # Vérifier et corriger les géométries non valides - - ### Voronoi Diagram ### - voronoi_lines = create_voronoi_lines(gdf, crs) - gdf_lines = line_merge(voronoi_lines) - - return - # saved_voronoi_lines = voronoi_lines.copy() - - # merge all lines that can be merged without doubt (when only 2 lines are connected) - # remove lines we don't want, then repeat until it doesn't change anymore - - - # gdf_lines = line_merge(voronoi_lines) - - # gdf_lines.to_file("test_gdf_lines_1.geojson", driver='GeoJSON') - # old_number_of_lines = len(gdf_lines) - # while True: - # gdf_lines = remove_extra_lines(gdf_lines) - # gdf_lines = line_merge(gdf_lines) - # if len(gdf_lines) != old_number_of_lines: - # old_number_of_lines = len(gdf_lines) - # else: - # break - - # gdf_lines = add_a_line_on_each_loop(gdf_lines, crs) - - - # gdf_lines = replace_lines_on_loops(gdf_lines, saved_voronoi_lines, crs) - # gdf_lines.to_file("test_add_a_line_on_each_loop.geojson", driver='GeoJSON') - # gdf_lines = gpd.GeoDataFrame(pd.concat([gdf_lines, gdf_lines_partially_on_loops], ignore_index=True)) - - # assembled = gpd.GeoDataFrame(pd.concat([gdf_connected_lines, gdf_lines], ignore_index=True)) - # assembled.to_file("test_assembled.geojson", driver='GeoJSON') # premerge - - gdf_lines = connect_extremities(gdf_lines, crs) - # gdf_lines = gdf_lines.set_crs(crs, allow_override=True) # the lines added to connect the extremities have no crs - - # connecting extremities creates lines we don't want (because of where we connect the extremities), - # we remove them one last time - # gdf_lines = remove_extra_lines(gdf_lines) - # gdf_lines = line_merge(gdf_lines) - - gpd.GeoDataFrame(gdf_lines, crs=crs).to_file("test_final.geojson", driver='GeoJSON') - - -if __name__ == "__main__": - mask_hydro = gpd.read_file(MASK_GEOJSON_5) - crs = mask_hydro.crs # Load a crs from input - # Simplifier geometrie - # buffer_geom = mask_hydro.apply(simplify_geometry) ## Appliquer un buffer - - # simplify_geom = mask_hydro.simplify(tolerance=2) # simplifier les géométries avec Douglas-Peucker - # df = gpd.GeoDataFrame(geometry=simplify_geom, crs=crs) - # run_old(df, crs) - - # run_branch(mask_hydro, crs) - - # Squelette - # run(df, crs) - run(mask_hydro, crs) - diff --git a/branch.py b/skeleton/branch.py similarity index 57% rename from branch.py rename to skeleton/branch.py index 7ccb7758..9527c447 100644 --- a/branch.py +++ b/skeleton/branch.py @@ -1,35 +1,29 @@ from itertools import product -from typing import Dict, List, Tuple +from typing import Dict, List from dataclasses import dataclass, field +from omegaconf import DictConfig import geopandas as gpd from geopandas.geodataframe import GeoDataFrame import pandas as pd import numpy as np -from math import sqrt -from shapely import LineString, Point, geometry, distance -from shapely.geometry import CAP_STYLE, Polygon, MultiPoint, Point, MultiLineString +from shapely import LineString, Point +from shapely.geometry import Polygon, MultiLineString from shapely.validation import make_valid -from shapely.ops import voronoi_diagram, linemerge, split, snap, triangulate +from shapely.ops import voronoi_diagram, linemerge -VORONOI_MAX_LENGTH = 2 -WATER_MIN_SIZE = 20 -MAX_GAP_CANDIDATES = 3 -GAP_MAX_WIDTH = 50 @dataclass class Candidate: - branch_1:"Branch" - branch_2:"Branch" - extremity_1:tuple - extremity_2:tuple - squared_distance:float - middle_point:tuple = field(init=False) - line:LineString = field(init = False) + branch_1: "Branch" + branch_2: "Branch" + extremity_1: tuple + extremity_2: tuple + squared_distance: float + line: LineString = field(init=False) def __post_init__(self): - self.middle_point = (self.extremity_1[0] + self.extremity_2[0])/2, (self.extremity_1[1] + self.extremity_2[1])/2 self.line = LineString([ (self.extremity_1[0], self.extremity_1[1]), (self.extremity_2[0], self.extremity_2[1]) @@ -39,149 +33,126 @@ def __post_init__(self): class Branch: """a branch is a geodataframe of lines of river that are all connected""" - def __init__(self, branch_id: str, mask_branch:GeoDataFrame, crs:int): - """ + def __init__(self, config: DictConfig, branch_id: str, branch_mask: GeoDataFrame, crs: int): + """ Args: - crs (str): EPSG code """ + self.config = config self.branch_id = branch_id self.crs = crs # simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker # self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) - self.gdf_branch_mask = self.check_and_create_gdf_branch_mask(mask_branch) + self.set_gdf_branch_mask(branch_mask) self.gap_points = [] # will contain points on the exterior that are connected to close gaps - self.set_df_all_coords() + self.df_all_coords = get_df_points_from_gdf(self.gdf_branch_mask) - def check_and_create_gdf_branch_mask(self, mask_branch): - """garanty polygons' validity""" - raw_gdf_branch_mask = gpd.GeoDataFrame(geometry=[mask_branch], crs=self.crs) + def set_gdf_branch_mask(self, branch_mask: GeoDataFrame): + """ + set branch_mask as self.gdf_branch_mask, after checking and correcting it's validity + Args: + - branch_mask (GeoDataFrame): a polygon as mask of the branch + """ + raw_gdf_branch_mask = gpd.GeoDataFrame(geometry=[branch_mask], crs=self.crs) - # create simple geometry - gdf_simple = raw_gdf_branch_mask.explode(ignore_index=True) # ignore_index makes the resulting index multi-indexed + # create simple geometry + gdf_simple = raw_gdf_branch_mask.explode(ignore_index=True) gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) # remove invalid polygons gdf_valid = gdf_without_duplicates.copy() gdf_valid.geometry = gdf_valid.geometry.apply( lambda geom: fix_invalid_geometry(geom) ) - return gdf_valid + self.gdf_branch_mask = gdf_valid def create_skeleton(self): + """ + create a skeleton for the branch + """ voronoi_lines = self.create_voronoi_lines() - # draw a new line for each point added to closes gaps to the nearest points on voronoi_lines + # draw a new line for each point added to closes gaps to the nearest points on voronoi_lines np_points = get_df_points_from_gdf(voronoi_lines).to_numpy().transpose() for gap_point in self.gap_points: distance_squared = (np_points[0] - gap_point.x)**2 + (np_points[1] - gap_point.y)**2 - min_index = np.unravel_index(np.argmin(distance_squared, axis=None), distance_squared.shape)[0] - voronoi_lines.loc[len(voronoi_lines)] = LineString([gap_point, Point(np_points[0][min_index], np_points[1][min_index])]) - - self.gdf_lines = line_merge(voronoi_lines, self.crs) - # self.set_modifications_from_changing_lines() + min_index = np.unravel_index(np.argmin(distance_squared, axis=None), distance_squared.shape)[0] + line_to_close_the_gap = LineString([gap_point, Point(np_points[0][min_index], np_points[1][min_index])]) + voronoi_lines.loc[len(voronoi_lines)] = line_to_close_the_gap + self.gdf_skeleton_lines = line_merge(voronoi_lines, self.crs) def simplify(self): - """remove useless lines""" - old_number_of_lines = len(self.gdf_lines) - while old_number_of_lines > 1: # we will exit the loop when there is only 1 line or it cannot be simplify anymore + """remove useless lines from skeleton_lines""" + old_number_of_lines = len(self.gdf_skeleton_lines) + while old_number_of_lines > 1: + # we will exit the loop when there is only 1 line left + # or it cannot be simplify anymore (the number of lines cease to vary) self.remove_extra_lines() - self.gdf_lines = line_merge(self.gdf_lines, self.crs) - if len(self.gdf_lines) != old_number_of_lines: - old_number_of_lines = len(self.gdf_lines) + self.gdf_skeleton_lines = line_merge(self.gdf_skeleton_lines, self.crs) + if len(self.gdf_skeleton_lines) != old_number_of_lines: + old_number_of_lines = len(self.gdf_skeleton_lines) else: break - # self.set_modifications_from_changing_lines() - - def set_df_all_coords(self): - all_points = set() # we use a set instead of a list to remove doubles - for _, row in self.gdf_branch_mask.iterrows(): - polygon:Polygon = row['geometry'] - for coord in polygon.exterior.coords: - all_points.add(coord) - - self.all_points = list(all_points) - self.df_all_coords = pd.DataFrame(data={'x': [point[0] for point in all_points], - 'y': [point[1] for point in all_points],}) - - # def set_modifications_from_changing_lines(self): - # """when a modification to self.gdf_lines has been done, those modifications - # must be taken into account for self.multiline""" - # # create a multiline, set of all the lines in gdf_lines - # self.multiline = MultiLineString([row['geometry'] for _, row in self.gdf_lines.iterrows()]) - - # # create df_all_coords, with all the coordinates of all points of all lines in gdf_lines - # """set all coordinates of all points""" - # all_points = set() # we use a set instead of a list to remove doubles - # # for _, row in self.gdf_lines.iterrows(): - # # line = row['geometry'] - # # for coord in line.coords: - # # all_points.add(coord) - # for _, row in self.gdf_branch_mask.iterrows(): - # polygon:Polygon = row['geometry'] - # for coord in polygon.exterior.coords: - # all_points.add(coord) - - # self.all_points = list(all_points) - # self.df_all_coords = pd.DataFrame(data={'x': [point[0] for point in all_points], - # 'y': [point[1] for point in all_points],}) - - # def distance_to_a_branch(self, other_branch:'Branch') -> float: - # """return the distance to another branch""" - # return self.multiline.distance(other_branch.multiline) - - def distance_to_a_branch(self, other_branch:'Branch') -> float: - """return the distance to another branch""" - return self.gdf_branch_mask.distance(other_branch.gdf_branch_mask)[0] # distance is given as a series with only 1 value - - def get_gap_candidates(self, other_branch:'Branch'): - """""" + + def distance_to_a_branch(self, other_branch: 'Branch') -> float: + """ + return the distance to another branch + Args: + - other_branch (Branch): the other branch we want the distance to + """ + return self.gdf_branch_mask.distance(other_branch.gdf_branch_mask)[0] + + def get_gap_candidates(self, other_branch: 'Branch') -> List[Candidate]: + """ + Return all candidates to close the gap with the other branch + Args: + - other_branch (Branch): the other branch we want the distance to + """ np_all_x = self.df_all_coords['x'].to_numpy() np_all_y = self.df_all_coords['y'].to_numpy() other_all_x = other_branch.df_all_coords['x'].to_numpy() other_all_y = other_branch.df_all_coords['y'].to_numpy() - x_diff = np_all_x - other_all_x[:,np.newaxis] # create a numpy matrix with all x from self minus all x from the other - y_diff = np_all_y - other_all_y[:,np.newaxis] # create a numpy matrix with all y from self minus all y from the other + # create 2 numpy matrix with all x, y from self, minus all x, y from the other + x_diff = np_all_x - other_all_x[:, np.newaxis] + y_diff = np_all_y - other_all_y[:, np.newaxis] distance_squared = x_diff**2 + y_diff**2 # index = np.unravel_index(np.argmax(distance_squared), distance_squared.shape) # sort all the distances and get their indexes - indexes = np.unravel_index(np.argsort(distance_squared, axis=None), distance_squared.shape) + indexes = np.unravel_index(np.argsort(distance_squared, axis=None), distance_squared.shape) - # search for the best candidates to close gaps + # search for the best candidates to close gaps candidates = [] for index, (other_index, self_index) in enumerate(zip(indexes[0], indexes[1])): # stop if we have enough candidates - if index >= MAX_GAP_CANDIDATES: + if index >= self.config.BRANCH.MAX_GAP_CANDIDATES: break - # stop if the following candidates - if distance_squared[other_index][self_index] > GAP_MAX_WIDTH * GAP_MAX_WIDTH: + # stop if the following candidates + if distance_squared[other_index][self_index] > self.config.MAX_GAP_WIDTH * self.config.MAX_GAP_WIDTH: break - candidates.append(Candidate(self, other_branch, self.all_points[self_index], other_branch.all_points[other_index], distance_squared[other_index][self_index])) + candidates.append( + Candidate(self, + other_branch, + (self.df_all_coords['x'][self_index], self.df_all_coords['y'][self_index]), + (other_branch.df_all_coords['x'][other_index], other_branch.df_all_coords['y'][other_index]), + distance_squared[other_index][self_index] + ) + ) return candidates def remove_extra_lines(self): - vertices_dict = get_vertices_dict(self.gdf_lines) + vertices_dict = get_vertices_dict(self.gdf_skeleton_lines) lines_to_remove = [] for vertex, line_list in vertices_dict.items(): - if len(line_list) == 1 : + if len(line_list) == 1: continue - # # the vertex used to close a gap have to be kept - # for point in self.gap_points: - # point:Point - # if point.equals_exact(vertex, 0.001): - # lines_to_keep.append(line[0]) # the should be only one line - # continue - - # if vertex in self.gap_points: - # lines_to_keep.append(line[0]) # the should be only one line - # continue - - # if len(line_list) !is not 1, then it's probably 3; Can't be 2 because of the previous merge, very unlikely to be 4+ + # if len(line_list) is not 1, then it's probably 3 because + # it can't be 2 because of the previous merge and it's very unlikely to be 4+ lines_to_keep = [] lines_that_can_be_removed = [] for line in line_list: @@ -214,14 +185,18 @@ def remove_extra_lines(self): line_that_should_not_be_removed = None for line_to_keep, line_that_can_be_removed in product(lines_to_keep, lines_that_can_be_removed): if vertex == line_to_keep.boundary.geoms[0]: - vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + vector_to_keep = np.array(line_to_keep.boundary.geoms[1].coords[0]) \ + - np.array(vertex.coords[0]) else: - vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + vector_to_keep = np.array(line_to_keep.boundary.geoms[0].coords[0]) \ + - np.array(vertex.coords[0]) if vertex == line_that_can_be_removed.boundary.geoms[0]: - vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) - np.array(vertex.coords[0]) + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[1].coords[0]) \ + - np.array(vertex.coords[0]) else: - vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) - np.array(vertex.coords[0]) + vector_to_remove = np.array(line_that_can_be_removed.boundary.geoms[0].coords[0]) \ + - np.array(vertex.coords[0]) length_to_keep = np.linalg.norm(vector_to_keep) length_to_remove = np.linalg.norm(vector_to_remove) @@ -240,10 +215,12 @@ def remove_extra_lines(self): lines_to_remove += lines_that_can_be_removed # set the lines without the lines to remove - self.gdf_lines = self.gdf_lines[~self.gdf_lines['geometry'].isin(lines_to_remove)] + self.gdf_skeleton_lines = self.gdf_skeleton_lines[~self.gdf_skeleton_lines['geometry'].isin(lines_to_remove)] def can_line_be_removed(self, line, vertices_dict): - if line.length > WATER_MIN_SIZE: + """Return true if a line can be removed + """ + if line.length > self.config.BRANCH.WATER_MIN_SIZE: return False point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] @@ -259,45 +236,46 @@ def can_line_be_removed(self, line, vertices_dict): def __repr__(self): return str(self.branch_id) - def create_voronoi_lines(self): - """ Créer les lignes de Voronoi + def create_voronoi_lines(self) -> GeoDataFrame: + """ Return Voronoi lines from the mask. + Only internal lines """ - - # create_voronoi_lines(self.gdf_branch_mask, self.crs) - # divide geometry into segments no longer than max_segment_length - segmentize_geom = (self.gdf_branch_mask['geometry'].unary_union).segmentize(max_segment_length=VORONOI_MAX_LENGTH) + united_geom = self.gdf_branch_mask['geometry'].unary_union + segmentize_geom = united_geom.segmentize(max_segment_length=self.config.BRANCH.VORONOI_MAX_LENGTH) # Create the voronoi diagram and only keep polygon regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) - ##### Filtrer les lignes de Voronoi en excluant celles à l'extérieur du masques ###### + # remove Voronoi lines exterior to the mask geometry = gpd.GeoSeries(regions.geoms, crs=self.crs).explode(index_parts=False) df = gpd.GeoDataFrame(geometry=geometry, crs=self.crs) lines_filter = df.sjoin(self.gdf_branch_mask, predicate='within') # Opération spatiale "Within" - # Enregistrer le diagram de Voronoi (Squelette) + # save Voronoi lines lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' return gpd.GeoDataFrame(lines_filter, crs=self.crs) -def get_df_points_from_gdf(gdf:GeoDataFrame)->pd.DataFrame: + +def get_df_points_from_gdf(gdf: GeoDataFrame) -> pd.DataFrame: all_points = set() # we use a set instead of a list to remove doubles for _, row in gdf.iterrows(): unknown_geometry = row['geometry'] - if type(unknown_geometry) == Polygon: - line:MultiLineString = unknown_geometry.exterior - elif type(unknown_geometry) == MultiLineString: - line:MultiLineString = unknown_geometry - elif type(unknown_geometry) == LineString: - line:LineString = unknown_geometry - else : + if isinstance(unknown_geometry, Polygon): + line: MultiLineString = unknown_geometry.exterior + elif isinstance(unknown_geometry, MultiLineString): + line: MultiLineString = unknown_geometry + elif isinstance(unknown_geometry, LineString): + line: LineString = unknown_geometry + else: raise NotImplementedError("Type unknown") for coord in line.coords: all_points.add(coord) all_points = list(all_points) - return pd.DataFrame(data={'x': [point[0] for point in all_points], - 'y': [point[1] for point in all_points],}) + return pd.DataFrame(data={'x': [point[0] for point in all_points], + 'y': [point[1] for point in all_points], }) + def fix_invalid_geometry(geometry): """ Fixer les géométries invalides @@ -307,7 +285,8 @@ def fix_invalid_geometry(geometry): else: return geometry -def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: + +def get_vertices_dict(gdf_lines: GeoDataFrame) -> Dict[Point, List[LineString]]: """ get a dictionary of vertices listing all the lines having a specific vertex as one of its extremities Args: @@ -317,22 +296,23 @@ def get_vertices_dict(gdf_lines:GeoDataFrame)->Dict[Point, List[LineString]]: """ # prepare a vertice dict, containing all the lines connected on a same vertex vertices_dict = {} - for index in gdf_lines.index : + for index in gdf_lines.index: line = gdf_lines.iloc[index]['geometry'] if line.is_ring: # it's a loop : no extremity - continue + continue point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] - try : + try: vertices_dict[point_a].append(line) except KeyError: vertices_dict[point_a] = [line] - try : + try: vertices_dict[point_b].append(line) except KeyError: vertices_dict[point_b] = [line] return vertices_dict + def line_merge(voronoi_lines, crs): """Fusionner tous les LINESTRING en un seul objet MultiLineString @@ -349,11 +329,11 @@ def line_merge(voronoi_lines, crs): merged_line = linemerge(merged_line) # in case the branch is reduced to a single line - if type(merged_line) == LineString: + if isinstance(merged_line, LineString): return gpd.GeoDataFrame(geometry=[merged_line], crs=crs) # turn multipart geometries into multiple single geometries - geometry = gpd.GeoSeries(merged_line.geoms, crs=crs).explode(index_parts=False) + geometry = gpd.GeoSeries(merged_line.geoms, crs=crs).explode(index_parts=False) lines = gpd.GeoDataFrame(geometry=geometry, crs=crs) - return lines \ No newline at end of file + return lines diff --git a/skeleton/group_maker.py b/skeleton/group_maker.py new file mode 100644 index 00000000..8d579108 --- /dev/null +++ b/skeleton/group_maker.py @@ -0,0 +1,22 @@ +class GroupMaker: + def __init__(self, element_list): + self.element_set_list = [] + for element in element_list: + self.element_set_list.append({element}) + + def find_index(self, element) -> int: + for index, element_set in enumerate(self.element_set_list): + if element in element_set: + return index + + def are_together(self, element_a, element_b) -> bool: + """return true if 2 elements are already together""" + index = self.find_index(element_a) + return element_b in self.element_set_list[index] + + def put_together(self, element_a, element_b): + """put the set of a and b together""" + index_a = self.find_index(element_a) + set_a = self.element_set_list.pop(index_a) + index_b = self.find_index(element_b) + self.element_set_list[index_b] = self.element_set_list[index_b].union(set_a) \ No newline at end of file diff --git a/skeleton/main_skeleton.py b/skeleton/main_skeleton.py new file mode 100644 index 00000000..476f33dc --- /dev/null +++ b/skeleton/main_skeleton.py @@ -0,0 +1,163 @@ +from typing import List, Tuple + +import hydra +from omegaconf import DictConfig + +import geopandas as gpd +import pandas as pd +import psycopg2 +# from shapely import Point +from shapely.geometry import Point + + +from branch import Branch, Candidate +from group_maker import GroupMaker + + +def db_connector(config: DictConfig): + """Return a connector to the postgis database""" + return psycopg2.connect( + database=config.DB_UNI.DB_NAME, + host=config.DB_UNI.DB_HOST, + user=config.DB_UNI.DB_USER, + password=config.DB_UNI.DB_PASSWORD, + port=config.DB_UNI.DB_PORT + ) + + +def query_db_for_bridge(config: DictConfig, candidate: Candidate) -> bool: + """ + Query the database to check if a candidate for a bridge indeed intersects a bridge + """ + middle_x = (candidate.extremity_1[0] + candidate.extremity_2[0]) / 2 + middle_y = (candidate.extremity_1[1] + candidate.extremity_2[1]) / 2 + new_ext_1_x = (candidate.extremity_1[0] - middle_x) * config.RATIO_GAP + middle_x + new_ext_1_y = (candidate.extremity_1[1] - middle_y) * config.RATIO_GAP + middle_y + new_ext_2_x = (candidate.extremity_2[0] - middle_x) * config.RATIO_GAP + middle_x + new_ext_2_y = (candidate.extremity_2[1] - middle_y) * config.RATIO_GAP + middle_y + line = f"LINESTRING({new_ext_1_x} {new_ext_1_y}, {new_ext_2_x} {new_ext_2_y})" + query_linear = ( + "SELECT cleabs FROM public.Construction_lineaire " + "WHERE gcms_detruit = false " + f"AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" + ) + query_area = ( + "SELECT cleabs FROM public.Construction_surfacique " + "WHERE gcms_detruit = false " + f"AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" + ) + + with db_connector(config) as db_conn: + with db_conn.cursor() as db_cursor: + db_cursor.execute(query_linear) + results = db_cursor.fetchall() + # if no result with linear bridge, maybe with area bridge... + if not results: + db_cursor.execute(query_area) + results = db_cursor.fetchall() + return True if results else False + + +def select_candidates(config: DictConfig, branches_pair_list: List[Tuple]) -> List[Candidate]: + """ + selects candidates between pairs of branches + """ + # get all the branches (each only once, hence the set) + branch_set = set() + for branch_a, branch_b, _ in branches_pair_list: + branch_set.add(branch_a) + branch_set.add(branch_b) + + branch_group = GroupMaker(list(branch_set)) + validated_candidates = [] + extremities_connected = set() + for branch_a, branch_b, _ in branches_pair_list: + branch_a: Branch + + # we don't create link between 2 branches already connected + if branch_group.are_together(branch_a, branch_b): + continue + candidates = branch_a.get_gap_candidates(branch_b) + + # test each candidate to see if we can draw a line for them + nb_bridges_crossed = 0 + for candidate in candidates: + candidate: Candidate + # we connect each extremity only once + if candidate.extremity_1 in extremities_connected or candidate.extremity_2 in extremities_connected: + continue + + # if the gap is wide enough, we check with DB_Uni to see if there is a bridge + if candidate.squared_distance > config.GAP_WIDTH_CHECK_DB * config.GAP_WIDTH_CHECK_DB: + is_bridge = query_db_for_bridge(config, candidate) + # if the line does not cross any bridge, we don't validate that candidate + if not is_bridge: + continue + + # candidate validated + extremities_connected.add(candidate.extremity_1) + extremities_connected.add(candidate.extremity_2) + validated_candidates.append(candidate) + branch_group.put_together(branch_a, branch_b) + nb_bridges_crossed += 1 + if nb_bridges_crossed >= config.MAX_BRIDGES: # max bridges reached between those 2 branches + break + return validated_candidates + +@hydra.main(config_path="../configs/", config_name="configs_skeleton.yaml") +def run(config: DictConfig): +# def run(gdf_mask_hydro, crs): + """Calculer le squelette hydrographique + + Args: + - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) + - crs (str): code EPSG + """ + gdf_mask_hydro = gpd.read_file(config.MASK_INPUT_PATH) + crs = gdf_mask_hydro.crs # Load a crs from input + + # create branches + branches_list = [] + for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): + mask_branch = branch_mask_row["geometry"] + new_branch = Branch(config, index_branch, mask_branch, crs) + branches_list.append(new_branch) + + # create branches_pair_list, that stores all pairs of branches close enough to have a bridge + branches_pair_list = [] + for index, branch_a in enumerate(branches_list[:-1]): + for branch_b in branches_list[index + 1:]: + distance = branch_a.distance_to_a_branch(branch_b) + if distance < config.MAX_GAP_WIDTH: + branches_pair_list.append((branch_a, branch_b, distance)) + + # sort the branches pairs by distance between them + branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) + + validated_candidates = select_candidates(config, branches_pair_list) + + # create the bridge lines from the selected candidates + bridge_lines = [validated_candidate.line for validated_candidate in validated_candidates] + gdf_bridge_lines = gpd.GeoDataFrame(geometry=bridge_lines).set_crs(crs, allow_override=True) + gdf_bridge_lines.to_file("test_bridges_group_maker_mask.geojson", driver='GeoJSON') + + # add the extremities used on each branch to close a gap to the list of gap_point of that branch, + # to create a new line toward that extremity and know not to remove it during the "simplify" + for candidate in validated_candidates: + candidate.branch_1.gap_points.append(Point(candidate.extremity_1)) + candidate.branch_2.gap_points.append(Point(candidate.extremity_2)) + + # get skeletons for all branches + for branch in branches_list: + branch: Branch + branch.create_skeleton() + branch.simplify() + + # saving skeleton lines from all branches + branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] + gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) + gdf_branch_lines.to_file(config.SKELETON_OUTPUT_PATH, driver='GeoJSON') + + +if __name__ == "__main__": + run() diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b From d64bb340f71445f6b545a647bbceed49cee7bd10 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 2 Jul 2024 15:24:49 +0200 Subject: [PATCH 10/68] Clean version to create skeleton, + tests --- configs/configs_skeleton.yaml | 32 ++++++++ skeleton/__init__.py | 0 skeleton/branch.py | 86 +++++++++++++-------- skeleton/group_maker.py | 6 +- skeleton/main_skeleton.py | 116 +++++++++++++++++++++------- test/skeleton/__init__.py | 0 test/skeleton/test_branch.py | 40 ++++++++++ test/skeleton/test_group_maker.py | 12 +++ test/skeleton/test_main_skeleton.py | 63 +++++++++++++++ test_files/40.geojson | 7 ++ test_files/43.geojson | 7 ++ test_files/44.geojson | 7 ++ test_files/90.geojson | 7 ++ 13 files changed, 321 insertions(+), 62 deletions(-) create mode 100644 configs/configs_skeleton.yaml create mode 100644 skeleton/__init__.py create mode 100644 test/skeleton/__init__.py create mode 100644 test/skeleton/test_branch.py create mode 100644 test/skeleton/test_group_maker.py create mode 100644 test/skeleton/test_main_skeleton.py create mode 100644 test_files/40.geojson create mode 100644 test_files/43.geojson create mode 100644 test_files/44.geojson create mode 100644 test_files/90.geojson diff --git a/configs/configs_skeleton.yaml b/configs/configs_skeleton.yaml new file mode 100644 index 00000000..f8fb2c87 --- /dev/null +++ b/configs/configs_skeleton.yaml @@ -0,0 +1,32 @@ +# @package _global_ + +# path to original working directory +# hydra hijacks working directory by changing it to the current log directory, +# so it's useful to have this path as a special variable +# learn more here: https://hydra.cc/docs/next/tutorials/basic/running_your_app/working_directory +work_dir: ${hydra:runtime.cwd} + +FILE_PATH: + MASK_INPUT_PATH: + SKELETON_LINES_OUTPUT_PATH: # only branches' skeleton (if empty, no file is created) + GAP_LINES_OUTPUT_PATH: # only new lines to close gaps (if empty, no file is created) + GLOBAL_LINES_OUTPUT_PATH: # lines from both the skeletons and the gap lines + +MAX_GAP_WIDTH: 200 # distance max in meter of any gap between 2 branches we will try to close with a line +MAX_BRIDGES: 1 # number max of bridges that can be created between 2 branches +GAP_WIDTH_CHECK_DB: 30 # for a gap at least this wide, we check with DB_UNI. Smaller than that, we assume a line automatically exists +RATIO_GAP: 0.25 # when checking if a candidate line to close a gap intersects a bridge in BD UNI, that's the ratio ]0.1] of the line to consider. + # we only check part of the line, because when the line is really long it can intersect with bridges not corresponding to that line, + # so we make the line smaller + +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: + DB_PASSWORD: + DB_PORT: 5432 + +BRANCH: + VORONOI_MAX_LENGTH: 2 # max size of a voronoi line + WATER_MIN_SIZE: 20 # min size of a skeleton line to be sure not to be removed + MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches \ No newline at end of file diff --git a/skeleton/__init__.py b/skeleton/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/skeleton/branch.py b/skeleton/branch.py index 9527c447..89627920 100644 --- a/skeleton/branch.py +++ b/skeleton/branch.py @@ -7,15 +7,18 @@ from geopandas.geodataframe import GeoDataFrame import pandas as pd import numpy as np - -from shapely import LineString, Point +from shapely import LineString, Point, Geometry from shapely.geometry import Polygon, MultiLineString from shapely.validation import make_valid from shapely.ops import voronoi_diagram, linemerge +from pyproj.crs.crs import CRS @dataclass class Candidate: + """ + a candidate contains the data to close a gap between 2 branches with a line + """ branch_1: "Branch" branch_2: "Branch" extremity_1: tuple @@ -31,12 +34,16 @@ def __post_init__(self): class Branch: - """a branch is a geodataframe of lines of river that are all connected""" + """a branch contains all the data relative to a single 'entity' of water, defined by a single polygon + that is the mask (the contour) of the branch """ - def __init__(self, config: DictConfig, branch_id: str, branch_mask: GeoDataFrame, crs: int): + def __init__(self, config: DictConfig, branch_id: str, branch_mask: GeoDataFrame, crs: CRS): """ Args: - - crs (str): EPSG code + - config (DictConfig): the config dict from hydra + - branch_id (str): an identifiant for the branch + - branch_mask (GeoDataFrame): a polygon as mask of the branch + - crs (CRS): EPSG code of the geodataframe """ self.config = config self.branch_id = branch_id @@ -45,14 +52,16 @@ def __init__(self, config: DictConfig, branch_id: str, branch_mask: GeoDataFrame # self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) self.set_gdf_branch_mask(branch_mask) + self.gdf_branch_mask.to_file(f"/home/MDaab/code/lidro/branches/{self.branch_id}.geojson", driver='GeoJSON') + self.gap_points = [] # will contain points on the exterior that are connected to close gaps self.df_all_coords = get_df_points_from_gdf(self.gdf_branch_mask) def set_gdf_branch_mask(self, branch_mask: GeoDataFrame): """ - set branch_mask as self.gdf_branch_mask, after checking and correcting it's validity + sets branch_mask as self.gdf_branch_mask, after checking and correcting its validity Args: - - branch_mask (GeoDataFrame): a polygon as mask of the branch + - branch_mask (GeoDataFrame): a polygon as mask of the branch """ raw_gdf_branch_mask = gpd.GeoDataFrame(geometry=[branch_mask], crs=self.crs) @@ -68,7 +77,7 @@ def set_gdf_branch_mask(self, branch_mask: GeoDataFrame): def create_skeleton(self): """ - create a skeleton for the branch + creates a skeleton for the branch """ voronoi_lines = self.create_voronoi_lines() @@ -82,7 +91,9 @@ def create_skeleton(self): self.gdf_skeleton_lines = line_merge(voronoi_lines, self.crs) def simplify(self): - """remove useless lines from skeleton_lines""" + """ + removes useless lines from skeleton_lines + """ old_number_of_lines = len(self.gdf_skeleton_lines) while old_number_of_lines > 1: # we will exit the loop when there is only 1 line left @@ -96,15 +107,15 @@ def simplify(self): def distance_to_a_branch(self, other_branch: 'Branch') -> float: """ - return the distance to another branch + returns the distance to another branch Args: - - other_branch (Branch): the other branch we want the distance to + - other_branch (Branch): the other branch we want the distance to """ return self.gdf_branch_mask.distance(other_branch.gdf_branch_mask)[0] - def get_gap_candidates(self, other_branch: 'Branch') -> List[Candidate]: + def get_candidates(self, other_branch: 'Branch') -> List[Candidate]: """ - Return all candidates to close the gap with the other branch + Returns all the possible candidates (up to MAX_GAP_CANDIDATES) to close the gap with the other branch Args: - other_branch (Branch): the other branch we want the distance to """ @@ -143,6 +154,10 @@ def get_gap_candidates(self, other_branch: 'Branch') -> List[Candidate]: return candidates def remove_extra_lines(self): + """ + removes the 'spikes" (the lone lines going outward) from a skeleton. + Keeps the extremities of long enough lines + """ vertices_dict = get_vertices_dict(self.gdf_skeleton_lines) lines_to_remove = [] @@ -217,8 +232,12 @@ def remove_extra_lines(self): # set the lines without the lines to remove self.gdf_skeleton_lines = self.gdf_skeleton_lines[~self.gdf_skeleton_lines['geometry'].isin(lines_to_remove)] - def can_line_be_removed(self, line, vertices_dict): - """Return true if a line can be removed + def can_line_be_removed(self, line: Geometry, vertices_dict: Dict[Point, List[LineString]]): + """ + Returns true if a line can be removed + Args: + - line (Geometry) : the line to test + - vertices_dict (Dict[Point, List[LineString]]) : a dictionary off all the lines with a point as an extremity """ if line.length > self.config.BRANCH.WATER_MIN_SIZE: return False @@ -237,8 +256,8 @@ def __repr__(self): return str(self.branch_id) def create_voronoi_lines(self) -> GeoDataFrame: - """ Return Voronoi lines from the mask. - Only internal lines + """ Returns the Voronoi lines from the mask. + (Only the lines that are completely inside the mask are returned) """ # divide geometry into segments no longer than max_segment_length united_geom = self.gdf_branch_mask['geometry'].unary_union @@ -258,6 +277,13 @@ def create_voronoi_lines(self) -> GeoDataFrame: def get_df_points_from_gdf(gdf: GeoDataFrame) -> pd.DataFrame: + """ + Return a 2-columns dataframe (col x, y), containing the coords of + all the points in a geodataframe + -Args : + - gdf (GeoDataFrame) : the geodataframe we want the points' coordinates from + """ + all_points = set() # we use a set instead of a list to remove doubles for _, row in gdf.iterrows(): unknown_geometry = row['geometry'] @@ -278,7 +304,10 @@ def get_df_points_from_gdf(gdf: GeoDataFrame) -> pd.DataFrame: def fix_invalid_geometry(geometry): - """ Fixer les géométries invalides + """ + return the geometry, fixed + Args: + - gdf_lines:geodataframe containing a list of lines """ if not geometry.is_valid: return make_valid(geometry) @@ -313,19 +342,14 @@ def get_vertices_dict(gdf_lines: GeoDataFrame) -> Dict[Point, List[LineString]]: return vertices_dict -def line_merge(voronoi_lines, crs): - """Fusionner tous les LINESTRING en un seul objet MultiLineString - +def line_merge(gdf_lines: GeoDataFrame, crs: CRS) -> GeoDataFrame: + """ + Merges together all the lines of gdf_lines (supposed to be Linestring) into MultiLinestrings Args: - - voronoi_lines (GeoDataframe) : Lignes de Voronoi comprises à l'intérieur du masque hydrographique - - Returns: - lines (GeoDataframe): Lignes de Voronoi fusionnées comprises à l'intérieur du masque hydrographique - + - gdf_lines (GeoDataframe) : contains a list of Linestring to merge """ - # Fusionner tous les LINESTRING en un seul objet MultiLineString - merged_line = voronoi_lines.geometry.unary_union - # Appliquer un algo de fusion des lignes (ex. "ST_LineMerge") sur l'objet MultiLineString + # merge the linestring together + merged_line = gdf_lines.geometry.unary_union merged_line = linemerge(merged_line) # in case the branch is reduced to a single line @@ -334,6 +358,4 @@ def line_merge(voronoi_lines, crs): # turn multipart geometries into multiple single geometries geometry = gpd.GeoSeries(merged_line.geoms, crs=crs).explode(index_parts=False) - lines = gpd.GeoDataFrame(geometry=geometry, crs=crs) - - return lines + return gpd.GeoDataFrame(geometry=geometry, crs=crs) diff --git a/skeleton/group_maker.py b/skeleton/group_maker.py index 8d579108..0221ee78 100644 --- a/skeleton/group_maker.py +++ b/skeleton/group_maker.py @@ -15,8 +15,10 @@ def are_together(self, element_a, element_b) -> bool: return element_b in self.element_set_list[index] def put_together(self, element_a, element_b): - """put the set of a and b together""" + """put the set of a and b together if they aren't""" + if self.are_together(element_a, element_b): + return index_a = self.find_index(element_a) set_a = self.element_set_list.pop(index_a) index_b = self.find_index(element_b) - self.element_set_list[index_b] = self.element_set_list[index_b].union(set_a) \ No newline at end of file + self.element_set_list[index_b] = self.element_set_list[index_b].union(set_a) diff --git a/skeleton/main_skeleton.py b/skeleton/main_skeleton.py index 476f33dc..bf73c943 100644 --- a/skeleton/main_skeleton.py +++ b/skeleton/main_skeleton.py @@ -2,20 +2,24 @@ import hydra from omegaconf import DictConfig +from pyproj.crs.crs import CRS import geopandas as gpd import pandas as pd +from geopandas.geodataframe import GeoDataFrame import psycopg2 -# from shapely import Point from shapely.geometry import Point - from branch import Branch, Candidate from group_maker import GroupMaker def db_connector(config: DictConfig): - """Return a connector to the postgis database""" + """ + Return a connector to the postgis database + args: + - config (DictConfig): the config dict from hydra + """ return psycopg2.connect( database=config.DB_UNI.DB_NAME, host=config.DB_UNI.DB_HOST, @@ -25,16 +29,24 @@ def db_connector(config: DictConfig): ) -def query_db_for_bridge(config: DictConfig, candidate: Candidate) -> bool: +def query_db_for_bridge_across_gap(config: DictConfig, candidate: Candidate) -> bool: """ - Query the database to check if a candidate for a bridge indeed intersects a bridge + Query the database to check if a candidate to close a gap between 2 branches intersects a bridge + args: + - config (DictConfig): the config dict from hydra + - candidate (Candidate): the candidate we want to check if it crosses a bridge """ + # creation of the segment that will be checked to see if it intersects a bridge + # that segment is the line between the 2 extremities, reduced by a ratio ( if too long, the line can intesects + # bridge from another area) middle_x = (candidate.extremity_1[0] + candidate.extremity_2[0]) / 2 middle_y = (candidate.extremity_1[1] + candidate.extremity_2[1]) / 2 new_ext_1_x = (candidate.extremity_1[0] - middle_x) * config.RATIO_GAP + middle_x new_ext_1_y = (candidate.extremity_1[1] - middle_y) * config.RATIO_GAP + middle_y new_ext_2_x = (candidate.extremity_2[0] - middle_x) * config.RATIO_GAP + middle_x new_ext_2_y = (candidate.extremity_2[1] - middle_y) * config.RATIO_GAP + middle_y + + # creation of queries line = f"LINESTRING({new_ext_1_x} {new_ext_1_y}, {new_ext_2_x} {new_ext_2_y})" query_linear = ( "SELECT cleabs FROM public.Construction_lineaire " @@ -47,6 +59,7 @@ def query_db_for_bridge(config: DictConfig, candidate: Candidate) -> bool: f"AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" ) + # execution of queries with db_connector(config) as db_conn: with db_conn.cursor() as db_cursor: db_cursor.execute(query_linear) @@ -55,20 +68,36 @@ def query_db_for_bridge(config: DictConfig, candidate: Candidate) -> bool: if not results: db_cursor.execute(query_area) results = db_cursor.fetchall() + return True if results else False -def select_candidates(config: DictConfig, branches_pair_list: List[Tuple]) -> List[Candidate]: +def select_candidates( + config: DictConfig, + branches_pair_list: List[Tuple[Candidate, Candidate, float]] + ) -> List[Candidate]: """ - selects candidates between pairs of branches + create candidates between pairs of branches + args: + - config (DictConfig): the config dict from hydra + - branches_pair_list (List[Tuple[Candidate, Candidate, float]]) : a list of paris, containing + 2 candidates and the minimal distance between them """ + + # sort the branches pairs by distance between each pair (so the first gap to be closed is the smallest) + branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) + # get all the branches (each only once, hence the set) branch_set = set() for branch_a, branch_b, _ in branches_pair_list: branch_set.add(branch_a) branch_set.add(branch_b) + # use GroupMaker to avoid cycles (if branch A connected to branch B, and + # branch B connected to branch C, then + # C cannot be connected to A branch_group = GroupMaker(list(branch_set)) + validated_candidates = [] extremities_connected = set() for branch_a, branch_b, _ in branches_pair_list: @@ -77,9 +106,10 @@ def select_candidates(config: DictConfig, branches_pair_list: List[Tuple]) -> Li # we don't create link between 2 branches already connected if branch_group.are_together(branch_a, branch_b): continue - candidates = branch_a.get_gap_candidates(branch_b) - # test each candidate to see if we can draw a line for them + candidates = branch_a.get_candidates(branch_b) # get all possible candidates between A abd B + + # test each candidate to see if we can draw a line between its branches nb_bridges_crossed = 0 for candidate in candidates: candidate: Candidate @@ -88,8 +118,9 @@ def select_candidates(config: DictConfig, branches_pair_list: List[Tuple]) -> Li continue # if the gap is wide enough, we check with DB_Uni to see if there is a bridge + # On the other hand, if it's small enough the candidate is automatically validated if candidate.squared_distance > config.GAP_WIDTH_CHECK_DB * config.GAP_WIDTH_CHECK_DB: - is_bridge = query_db_for_bridge(config, candidate) + is_bridge = query_db_for_bridge_across_gap(config, candidate) # if the line does not cross any bridge, we don't validate that candidate if not is_bridge: continue @@ -98,30 +129,39 @@ def select_candidates(config: DictConfig, branches_pair_list: List[Tuple]) -> Li extremities_connected.add(candidate.extremity_1) extremities_connected.add(candidate.extremity_2) validated_candidates.append(candidate) + # a candidate has been validated between A and B, so we put together A and B branch_group.put_together(branch_a, branch_b) nb_bridges_crossed += 1 if nb_bridges_crossed >= config.MAX_BRIDGES: # max bridges reached between those 2 branches break return validated_candidates -@hydra.main(config_path="../configs/", config_name="configs_skeleton.yaml") -def run(config: DictConfig): -# def run(gdf_mask_hydro, crs): - """Calculer le squelette hydrographique +def create_branches_list(config: DictConfig, gdf_hydro_global_mask: GeoDataFrame, crs: CRS) -> List[Branch]: + """ + create the list of branches from the global mask Args: - - mask_hydro (GeoJSON) : Mask Hydrographic (Polygone) - - crs (str): code EPSG + - config (DictConfig): the config dict from hydra + - gdf_hydro_global_mask (GeoDataFrame): a geodataframe containing a list of polygon, + each polygon being the mask of a river's branch + - crs (CRS); the crs of gdf_hydro_global_mask """ - gdf_mask_hydro = gpd.read_file(config.MASK_INPUT_PATH) - crs = gdf_mask_hydro.crs # Load a crs from input - # create branches branches_list = [] - for index_branch, branch_mask_row in gdf_mask_hydro.iterrows(): + for index_branch, branch_mask_row in gdf_hydro_global_mask.iterrows(): mask_branch = branch_mask_row["geometry"] new_branch = Branch(config, index_branch, mask_branch, crs) branches_list.append(new_branch) + return branches_list + + +def create_branches_pair(config: DictConfig, branches_list: List[Branch]) -> List[Tuple[Candidate, Candidate, float]]: + """ + create a list of pairs of branches, containing all the pairs of branches close enough from each other + for the water to flow anyway between them, and return them sorted by distance + Args: + - config (DictConfig): the config dict from hydra + """ # create branches_pair_list, that stores all pairs of branches close enough to have a bridge branches_pair_list = [] @@ -130,16 +170,30 @@ def run(config: DictConfig): distance = branch_a.distance_to_a_branch(branch_b) if distance < config.MAX_GAP_WIDTH: branches_pair_list.append((branch_a, branch_b, distance)) + return branches_pair_list - # sort the branches pairs by distance between them - branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) + +@hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_skeleton.yaml") +def run(config: DictConfig): + """ + Get whole hydrographic skeleton + args: + - config (DictConfig): the config dict from hydra + """ + + gdf_hydro_global_mask = gpd.read_file(config.FILE_PATH.MASK_INPUT_PATH) + crs = gdf_hydro_global_mask.crs # Load a crs from input + + branches_list = create_branches_list(config, gdf_hydro_global_mask, crs) + branches_pair_list = create_branches_pair(config, branches_list) validated_candidates = select_candidates(config, branches_pair_list) - # create the bridge lines from the selected candidates - bridge_lines = [validated_candidate.line for validated_candidate in validated_candidates] - gdf_bridge_lines = gpd.GeoDataFrame(geometry=bridge_lines).set_crs(crs, allow_override=True) - gdf_bridge_lines.to_file("test_bridges_group_maker_mask.geojson", driver='GeoJSON') + # create the gap lines from the selected candidates + gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] + if config.FILE_PATH.GAP_LINES_OUTPUT_PATH: + gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) + gdf_gap_lines.to_file(config.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, # to create a new line toward that extremity and know not to remove it during the "simplify" @@ -153,10 +207,16 @@ def run(config: DictConfig): branch.create_skeleton() branch.simplify() - # saving skeleton lines from all branches + # putting all skeleton lines together, and save them if their is a path branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - gdf_branch_lines.to_file(config.SKELETON_OUTPUT_PATH, driver='GeoJSON') + if config.FILE_PATH.GAP_LINES_OUTPUT_PATH: + gdf_branch_lines.to_file(config.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') + + # saving all lines + branch_lines_list.append(gdf_gap_lines) + gdf_global_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) + gdf_global_lines.to_file(config.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') if __name__ == "__main__": diff --git a/test/skeleton/__init__.py b/test/skeleton/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py new file mode 100644 index 00000000..7e0eb187 --- /dev/null +++ b/test/skeleton/test_branch.py @@ -0,0 +1,40 @@ +import sys + +from hydra import compose, initialize +import geopandas as gpd +from omegaconf import DictConfig + +from skeleton.branch import Branch, get_vertices_dict + +sys.path.append('skeleton') + +BRANCH_TEST_1_PATH = "test_files/90.geojson" + + +def read_branch(config: DictConfig, branch_path: str, branch_name: str) -> Branch: + gdf_branch_mask = gpd.read_file(branch_path) + crs = gdf_branch_mask.crs + mask_branch = gdf_branch_mask["geometry"][0] + return Branch(config, branch_name, mask_branch, crs) + + +def test_branch_1(): + """test creation/simplification of skeleton lines for a branch""" + with initialize(version_base="1.2", config_path="../../configs"): + config = compose( + config_name="configs_skeleton.yaml", + ) + branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") + + # create branch_1's skeleton + branch_1.create_skeleton() + branch_1.simplify() + + # number of extremity of branch_1's skeleton + vertices_dict = get_vertices_dict(branch_1.gdf_skeleton_lines) + extremities_cpt = 0 + for lines_list in vertices_dict.values(): + if len(lines_list) == 1: + extremities_cpt += 1 + + assert extremities_cpt == 3 # check that this branch's skeleton has exactly 3 extremities diff --git a/test/skeleton/test_group_maker.py b/test/skeleton/test_group_maker.py new file mode 100644 index 00000000..7c4983b4 --- /dev/null +++ b/test/skeleton/test_group_maker.py @@ -0,0 +1,12 @@ +from skeleton.group_maker import GroupMaker + + +def test_group_maker(): + A = 'A' + B = 'B' + C = 'C' + element_list = [A, B, C] + group_maker = GroupMaker(element_list) + group_maker.put_together(A, B) + assert group_maker.are_together(A, B) + assert not group_maker.are_together(A, C) diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py new file mode 100644 index 00000000..1c4c0d1f --- /dev/null +++ b/test/skeleton/test_main_skeleton.py @@ -0,0 +1,63 @@ +from hydra import compose, initialize + +import pandas as pd +import geopandas as gpd + +from skeleton.main_skeleton import create_branches_list, create_branches_pair +from skeleton.main_skeleton import select_candidates, query_db_for_bridge_across_gap +from skeleton.branch import Candidate + +from test.skeleton.test_branch import read_branch + +USER = "TO_DEFINE" +PASSWORD = "TO_DEFINE" + +CRS = 2154 + +MAIN_SKELETON_TEST_1_1_PATH = "test_files/40.geojson" +MAIN_SKELETON_TEST_1_2_PATH = "test_files/43.geojson" +MAIN_SKELETON_TEST_1_3_PATH = "test_files/44.geojson" + + +def test_main_skeleton_1(): + with initialize(version_base="1.2", config_path="../../configs"): + config = compose( + config_name="configs_skeleton.yaml", + ) + + branch_2_1 = read_branch(config, MAIN_SKELETON_TEST_1_1_PATH, "test_main_skeleton_1_1") + branch_2_2 = read_branch(config, MAIN_SKELETON_TEST_1_2_PATH, "test_main_skeleton_1_2") + branch_2_3 = read_branch(config, MAIN_SKELETON_TEST_1_3_PATH, "test_main_skeleton_1_3") + + three_branches = [branch_2_1.gdf_branch_mask, branch_2_2.gdf_branch_mask, branch_2_3.gdf_branch_mask] + gdf_three_branches = gpd.GeoDataFrame(pd.concat(three_branches, ignore_index=True)) + + branches_list = create_branches_list(config, gdf_three_branches, CRS) + branches_pair_list = create_branches_pair(config, branches_list) + validated_candidates = select_candidates(config, branches_pair_list) + + assert len(validated_candidates) == 2 # 3 branches close from each other, so there should be 2 candidates + candidate_1 = validated_candidates[0] + candidate_2 = validated_candidates[1] + + assert (candidate_1.squared_distance < 30) + assert (candidate_2.squared_distance < 75) + + +def test_main_skeleton_2(): + """Test : query_db_for_bridge_across_gap """ + with initialize(version_base="1.2", config_path="../../configs"): + config = compose( + config_name="configs_skeleton.yaml", + overrides=[ + f"DB_UNI.DB_USER={USER}", + f'DB_UNI.DB_PASSWORD="{PASSWORD}"', + ], + ) + dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) + dummy_candidate_2 = Candidate(None, None, (690880.5, 6737964.5), (690830.5, 6737951.586065615), 0) + + is_bridge_1 = query_db_for_bridge_across_gap(config, dummy_candidate_1) + is_bridge_2 = query_db_for_bridge_across_gap(config, dummy_candidate_2) + assert not is_bridge_1 + assert is_bridge_2 diff --git a/test_files/40.geojson b/test_files/40.geojson new file mode 100644 index 00000000..7cccd82c --- /dev/null +++ b/test_files/40.geojson @@ -0,0 +1,7 @@ +{ +"type": "FeatureCollection", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 699065.413934385520406, 6759448.5 ], [ 699063.5, 6759461.5 ], [ 699066.262332465150394, 6759463.18974278960377 ], [ 699081.333572532981634, 6759487.313280043192208 ], [ 699081.5, 6759490.586065614596009 ], [ 699083.472945803543553, 6759492.716397003270686 ], [ 699083.5, 6759501.5 ], [ 699085.471177920582704, 6759502.707364517264068 ], [ 699084.472945803543553, 6759521.283602996729314 ], [ 699078.5, 6759537.5 ], [ 699080.5, 6759537.5 ], [ 699082.716397003736347, 6759533.527054196223617 ], [ 699086.435427015880123, 6759533.564589496701956 ], [ 699087.5, 6759535.5 ], [ 699089.5, 6759535.5 ], [ 699089.5, 6759533.5 ], [ 699087.56458949635271, 6759532.435427015647292 ], [ 699087.926403762074187, 6759529.952435625717044 ], [ 699093.5, 6759531.5 ], [ 699092.506778866751119, 6759526.857554757036269 ], [ 699095.5, 6759508.586179815232754 ], [ 699092.516235235147178, 6759498.220095711760223 ], [ 699093.5, 6759494.5 ], [ 699091.752795581589453, 6759493.833355349488556 ], [ 699088.607240879791789, 6759486.55697579216212 ], [ 699087.5, 6759480.413934385403991 ], [ 699082.614180701202713, 6759474.574025148525834 ], [ 699081.5, 6759467.5 ], [ 699078.820317181060091, 6759465.926470963284373 ], [ 699075.607240879791789, 6759460.55697579216212 ], [ 699075.5, 6759454.5 ], [ 699069.5, 6759449.5 ], [ 699065.413934385520406, 6759448.5 ] ] ] } } +] +} diff --git a/test_files/43.geojson b/test_files/43.geojson new file mode 100644 index 00000000..1308a6ef --- /dev/null +++ b/test_files/43.geojson @@ -0,0 +1,7 @@ +{ +"type": "FeatureCollection", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 698272.483764764852822, 6759201.220095711760223 ], [ 698270.5, 6759205.5 ], [ 698280.686719955410808, 6759208.666427466087043 ], [ 698291.413820184767246, 6759215.5 ], [ 698297.283602996263653, 6759215.527054196223617 ], [ 698304.413820184767246, 6759218.5 ], [ 698319.283602996263653, 6759218.527054196223617 ], [ 698327.413820184767246, 6759222.5 ], [ 698332.574025148525834, 6759222.614180700853467 ], [ 698333.5, 6759224.5 ], [ 698338.574025148525834, 6759224.614180700853467 ], [ 698339.5, 6759226.5 ], [ 698348.413934385520406, 6759230.5 ], [ 698358.686719955410808, 6759231.666427466087043 ], [ 698374.247204418410547, 6759248.166644650511444 ], [ 698375.5, 6759255.586065614596009 ], [ 698381.159515680046752, 6759261.048410073854029 ], [ 698381.5, 6759263.5 ], [ 698384.926470962003805, 6759263.820317179895937 ], [ 698389.413934385520406, 6759269.5 ], [ 698394.926470962003805, 6759271.820317179895937 ], [ 698400.5, 6759277.5 ], [ 698407.413820184767246, 6759279.5 ], [ 698429.586179815232754, 6759279.5 ], [ 698446.586179815232754, 6759278.5 ], [ 698459.0, 6759275.5 ], [ 698498.142445242963731, 6759276.50677886698395 ], [ 698513.556975790997967, 6759280.607240878976882 ], [ 698527.413820184767246, 6759287.5 ], [ 698536.413820184767246, 6759289.5 ], [ 698551.073601511539891, 6759288.501806816086173 ], [ 698554.413820184767246, 6759290.5 ], [ 698562.142445242963731, 6759289.50677886698395 ], [ 698571.413820184767246, 6759292.5 ], [ 698581.283602996263653, 6759292.527054196223617 ], [ 698586.413934385520406, 6759295.5 ], [ 698592.707095105201006, 6759296.677118103019893 ], [ 698593.5, 6759298.5 ], [ 698599.833355349488556, 6759299.752795581705868 ], [ 698600.5, 6759304.5 ], [ 698609.262332465150394, 6759314.18974278960377 ], [ 698615.5, 6759326.5 ], [ 698618.951589926262386, 6759326.840484320186079 ], [ 698620.5, 6759330.586065614596009 ], [ 698632.413934385520406, 6759341.5 ], [ 698643.142445242963731, 6759341.50677886698395 ], [ 698657.413820184767246, 6759345.5 ], [ 698664.435427015880123, 6759345.564589496701956 ], [ 698665.5, 6759348.5 ], [ 698673.707095105201006, 6759350.677118103019893 ], [ 698675.413934385520406, 6759353.5 ], [ 698683.707095105201006, 6759354.677118103019893 ], [ 698685.5, 6759358.5 ], [ 698689.810257209115662, 6759359.737667533569038 ], [ 698691.5, 6759362.5 ], [ 698697.926470962003805, 6759365.820317179895937 ], [ 698701.247204418410547, 6759369.166644650511444 ], [ 698701.5, 6759371.5 ], [ 698706.060660171788186, 6759373.939339828677475 ], [ 698706.5, 6759376.586065614596009 ], [ 698717.086370622622781, 6759385.965689181350172 ], [ 698717.5, 6759388.5 ], [ 698722.247204418410547, 6759389.166644650511444 ], [ 698723.483764764852822, 6759392.220095711760223 ], [ 698721.262332465150394, 6759398.81025721039623 ], [ 698714.810257209115662, 6759405.262332466430962 ], [ 698711.5, 6759405.5 ], [ 698708.5, 6759409.5 ], [ 698710.5, 6759417.0 ], [ 698707.5, 6759435.586179815232754 ], [ 698714.392759120208211, 6759443.44302420783788 ], [ 698715.5, 6759450.586179815232754 ], [ 698718.472945803543553, 6759455.716397003270686 ], [ 698717.5, 6759462.5 ], [ 698732.686719955410808, 6759467.666427466087043 ], [ 698737.4393572693225, 6759473.577802591957152 ], [ 698737.247204418410547, 6759476.833355349488556 ], [ 698727.5, 6759488.5 ], [ 698726.5, 6759501.5 ], [ 698730.5, 6759500.586065614596009 ], [ 698728.655030296300538, 6759496.335879155434668 ], [ 698737.5, 6759484.586065614596009 ], [ 698738.840484319953248, 6759480.048410073854029 ], [ 698741.5, 6759479.5 ], [ 698741.5, 6759475.413820184767246 ], [ 698736.586065614479594, 6759464.5 ], [ 698733.586065614479594, 6759462.5 ], [ 698727.443024209002033, 6759462.392759121023118 ], [ 698723.820317181060091, 6759458.926470963284373 ], [ 698723.5, 6759455.5 ], [ 698721.528822079417296, 6759454.292635482735932 ], [ 698720.5, 6759443.413934385403991 ], [ 698717.5, 6759439.5 ], [ 698713.913629377377219, 6759438.034310818649828 ], [ 698711.607240879791789, 6759433.55697579216212 ], [ 698711.5606427306775, 6759426.577802591957152 ], [ 698715.5, 6759421.586179815232754 ], [ 698714.666427467018366, 6759411.313280043192208 ], [ 698720.073529037996195, 6759405.820317179895937 ], [ 698727.56603432178963, 6759401.566034321673214 ], [ 698731.313280044589192, 6759394.666427466087043 ], [ 698739.586179815232754, 6759395.5 ], [ 698743.5, 6759394.5 ], [ 698744.564572984119877, 6759392.564589496701956 ], [ 698749.5, 6759393.5 ], [ 698750.425974851474166, 6759391.614180700853467 ], [ 698756.5, 6759390.5 ], [ 698761.292904894798994, 6759385.677118103019893 ], [ 698768.586065614479594, 6759384.5 ], [ 698777.189742790884338, 6759377.737667533569038 ], [ 698779.586065614479594, 6759377.5 ], [ 698783.166644650511444, 6759371.752795581705868 ], [ 698789.5, 6759371.5 ], [ 698792.048410073737614, 6759366.840484320186079 ], [ 698795.586065614479594, 6759366.5 ], [ 698799.5, 6759362.586065614596009 ], [ 698799.752795581589453, 6759359.166644650511444 ], [ 698801.5, 6759358.5 ], [ 698801.5, 6759355.5 ], [ 698798.500451771891676, 6759354.036811839789152 ], [ 698800.752795581589453, 6759347.166644650511444 ], [ 698809.5, 6759338.586065614596009 ], [ 698810.048410073737614, 6759333.840484320186079 ], [ 698813.577802593121305, 6759331.56064273044467 ], [ 698820.5, 6759330.5 ], [ 698821.666427467018366, 6759325.313280043192208 ], [ 698824.5, 6759323.5 ], [ 698826.707364517031237, 6759318.52882207930088 ], [ 698834.707095105201006, 6759320.677118103019893 ], [ 698840.413934385520406, 6759325.5 ], [ 698852.833355349488556, 6759330.752795581705868 ], [ 698853.5, 6759332.5 ], [ 698863.686719955410808, 6759333.666427466087043 ], [ 698870.413934385520406, 6759339.5 ], [ 698876.686719955410808, 6759341.666427466087043 ], [ 698890.413934385520406, 6759350.5 ], [ 698893.707095105201006, 6759350.677118103019893 ], [ 698894.5, 6759352.5 ], [ 698902.686719955410808, 6759355.666427466087043 ], [ 698920.413934385520406, 6759366.5 ], [ 698926.556975790997967, 6759367.607240878976882 ], [ 698931.413934385520406, 6759371.5 ], [ 698937.686719955410808, 6759372.666427466087043 ], [ 698942.413934385520406, 6759376.5 ], [ 698970.556975790997967, 6759384.607240878976882 ], [ 698985.810257209115662, 6759391.737667533569038 ], [ 698995.159515680046752, 6759401.048410073854029 ], [ 698996.5, 6759406.5 ], [ 699000.159515680046752, 6759408.048410073854029 ], [ 699003.5, 6759414.5 ], [ 699007.247204418410547, 6759417.166644650511444 ], [ 699008.5, 6759421.586065614596009 ], [ 699014.086370622622781, 6759425.965689181350172 ], [ 699018.5, 6759432.586065614596009 ], [ 699021.413934385520406, 6759435.5 ], [ 699023.926470962003805, 6759435.820317179895937 ], [ 699026.5, 6759439.5 ], [ 699031.5, 6759440.5 ], [ 699031.5, 6759438.5 ], [ 699021.840484319953248, 6759431.951589926145971 ], [ 699020.544953120406717, 6759428.364470270462334 ], [ 699022.635529730119742, 6759425.544953119941056 ], [ 699028.5, 6759426.5 ], [ 699027.5, 6759422.5 ], [ 699018.507222909945995, 6759417.147025710903108 ], [ 699018.527054196456447, 6759407.716397003270686 ], [ 699022.048410073737614, 6759400.840484320186079 ], [ 699028.422197406878695, 6759399.56064273044467 ], [ 699032.413934385520406, 6759404.5 ], [ 699034.5, 6759404.5 ], [ 699034.5, 6759401.413934385403991 ], [ 699027.701729563763365, 6759396.751328073441982 ], [ 699027.5, 6759392.5 ], [ 699022.443024209002033, 6759392.392759121023118 ], [ 698983.586179815232754, 6759375.5 ], [ 698975.577802593121305, 6759374.43935726955533 ], [ 698954.443024209002033, 6759367.392759121023118 ], [ 698949.586065614479594, 6759363.5 ], [ 698942.425974851474166, 6759363.385819299146533 ], [ 698934.586065614479594, 6759356.5 ], [ 698930.425974851474166, 6759356.385819299146533 ], [ 698928.586065614479594, 6759353.5 ], [ 698921.425974851474166, 6759352.385819299146533 ], [ 698920.5, 6759350.5 ], [ 698915.292904894798994, 6759350.322881896980107 ], [ 698909.5, 6759344.5 ], [ 698883.292904894798994, 6759331.322881896980107 ], [ 698877.586065614479594, 6759326.5 ], [ 698872.313280044589192, 6759325.333572533912957 ], [ 698861.586179815232754, 6759318.5 ], [ 698856.443024209002033, 6759318.392759121023118 ], [ 698848.586179815232754, 6759313.5 ], [ 698836.586179815232754, 6759310.5 ], [ 698824.5, 6759310.5 ], [ 698822.159515680046752, 6759318.951589926145971 ], [ 698815.5, 6759320.5 ], [ 698814.060660171788186, 6759324.060660171322525 ], [ 698810.833355349488556, 6759327.247204418294132 ], [ 698808.5, 6759327.5 ], [ 698806.086370622622781, 6759333.034310818649828 ], [ 698801.5, 6759335.413934385403991 ], [ 698801.262332465150394, 6759337.81025721039623 ], [ 698793.5, 6759345.5 ], [ 698791.159515680046752, 6759350.951589926145971 ], [ 698788.5, 6759351.5 ], [ 698789.426442500087433, 6759355.463963138870895 ], [ 698786.5, 6759356.5 ], [ 698785.412305920384824, 6759361.5 ], [ 698788.488999214023352, 6759365.181332128122449 ], [ 698785.729315414093435, 6759369.310762764886022 ], [ 698781.725387349841185, 6759366.790798705071211 ], [ 698783.587694079615176, 6759362.5 ], [ 698781.5, 6759362.5 ], [ 698775.810257209115662, 6759370.262332466430962 ], [ 698771.413820184767246, 6759370.5 ], [ 698762.422197406878695, 6759375.43935726955533 ], [ 698758.413934385520406, 6759375.5 ], [ 698752.556975790997967, 6759381.392759121023118 ], [ 698741.142445242963731, 6759384.49322113301605 ], [ 698729.716397003736347, 6759383.472945803776383 ], [ 698724.820317181060091, 6759379.926470963284373 ], [ 698718.5, 6759369.5 ], [ 698715.586065614479594, 6759366.5 ], [ 698711.965689182863571, 6759366.086370624601841 ], [ 698707.5, 6759359.5 ], [ 698701.048410073737614, 6759356.159515679813921 ], [ 698698.586065614479594, 6759352.5 ], [ 698695.048410073737614, 6759352.159515679813921 ], [ 698694.5, 6759349.5 ], [ 698690.189742790884338, 6759348.262332466430962 ], [ 698686.5, 6759344.5 ], [ 698683.166644650511444, 6759344.247204418294132 ], [ 698677.586065614479594, 6759338.5 ], [ 698664.577802593121305, 6759335.43935726955533 ], [ 698656.586179815232754, 6759331.5 ], [ 698641.577802593121305, 6759330.43935726955533 ], [ 698632.073529037996195, 6759325.179682820104063 ], [ 698623.820317181060091, 6759316.926470963284373 ], [ 698623.5, 6759314.5 ], [ 698620.737667534849606, 6759312.81025721039623 ], [ 698613.5, 6759300.413934385403991 ], [ 698609.737667534849606, 6759297.81025721039623 ], [ 698609.5, 6759295.5 ], [ 698606.939339828211814, 6759295.060660171322525 ], [ 698599.586065614479594, 6759285.5 ], [ 698596.443024209002033, 6759285.392759121023118 ], [ 698591.586179815232754, 6759282.5 ], [ 698580.0, 6759283.5 ], [ 698565.857554757036269, 6759282.49322113301605 ], [ 698527.716397003736347, 6759277.472945803776383 ], [ 698522.425974851474166, 6759276.385819299146533 ], [ 698521.5, 6759274.5 ], [ 698516.425974851474166, 6759273.385819299146533 ], [ 698512.586179815232754, 6759270.5 ], [ 698495.586179815232754, 6759266.5 ], [ 698464.413820184767246, 6759265.5 ], [ 698458.413934385520406, 6759266.5 ], [ 698456.283602996263653, 6759268.472945803776383 ], [ 698446.142445242963731, 6759269.49322113301605 ], [ 698438.413820184767246, 6759268.5 ], [ 698411.577802593121305, 6759270.43935726955533 ], [ 698393.939339828211814, 6759261.060660171322525 ], [ 698388.752795581589453, 6759255.833355349488556 ], [ 698388.5, 6759253.413934385403991 ], [ 698384.840484319953248, 6759250.951589926145971 ], [ 698380.5, 6759243.5 ], [ 698377.737667534849606, 6759241.81025721039623 ], [ 698375.5, 6759236.5 ], [ 698373.614180701202713, 6759235.574025148525834 ], [ 698373.5, 6759231.5 ], [ 698369.939339828211814, 6759230.060660171322525 ], [ 698369.5, 6759227.5 ], [ 698363.292904894798994, 6759226.322881896980107 ], [ 698359.5, 6759222.5 ], [ 698352.716397003736347, 6759222.472945803776383 ], [ 698340.443024209002033, 6759219.392759121023118 ], [ 698335.5, 6759215.5 ], [ 698329.048410073737614, 6759213.159515679813921 ], [ 698328.5, 6759210.5 ], [ 698316.852974289562553, 6759210.492777089588344 ], [ 698315.5, 6759208.5 ], [ 698313.5, 6759208.5 ], [ 698313.159515680046752, 6759210.951589926145971 ], [ 698310.857554757036269, 6759211.49322113301605 ], [ 698306.292904894798994, 6759210.322881896980107 ], [ 698301.586179815232754, 6759206.5 ], [ 698298.292904894798994, 6759206.322881896980107 ], [ 698297.5, 6759204.5 ], [ 698291.536036859615706, 6759206.426442499272525 ], [ 698289.586065614479594, 6759203.5 ], [ 698281.716397003736347, 6759203.472945803776383 ], [ 698275.292904894798994, 6759201.322881896980107 ], [ 698274.5, 6759197.5 ], [ 698271.5, 6759197.5 ], [ 698272.483764764852822, 6759201.220095711760223 ] ] ] } } +] +} diff --git a/test_files/44.geojson b/test_files/44.geojson new file mode 100644 index 00000000..0618305b --- /dev/null +++ b/test_files/44.geojson @@ -0,0 +1,7 @@ +{ +"type": "FeatureCollection", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 699038.5, 6759434.5 ], [ 699036.5, 6759442.5 ], [ 699042.413820184767246, 6759446.5 ], [ 699045.833355349488556, 6759446.752795581705868 ], [ 699046.5, 6759449.5 ], [ 699051.951589926262386, 6759451.840484320186079 ], [ 699054.5, 6759455.5 ], [ 699056.5, 6759455.5 ], [ 699058.5, 6759443.5 ], [ 699053.292904894798994, 6759443.322881896980107 ], [ 699051.586065614479594, 6759439.5 ], [ 699040.586179815232754, 6759434.5 ], [ 699038.5, 6759434.5 ] ] ] } } +] +} diff --git a/test_files/90.geojson b/test_files/90.geojson new file mode 100644 index 00000000..e2ae0971 --- /dev/null +++ b/test_files/90.geojson @@ -0,0 +1,7 @@ +{ +"type": "FeatureCollection", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 706730.5, 6754661.5 ], [ 706730.5, 6754664.5 ], [ 706737.5, 6754663.5 ], [ 706740.5, 6754654.586179815232754 ], [ 706739.516235235147178, 6754649.779904288239777 ], [ 706743.5, 6754637.586179815232754 ], [ 706743.5606427306775, 6754632.577802591957152 ], [ 706746.752795581589453, 6754625.166644650511444 ], [ 706748.5, 6754624.5 ], [ 706747.501806815736927, 6754620.926398488692939 ], [ 706748.528822079417296, 6754605.707364517264068 ], [ 706750.5, 6754604.5 ], [ 706754.56458949635271, 6754579.564572984352708 ], [ 706757.5, 6754578.5 ], [ 706755.381012363242917, 6754573.279904288239777 ], [ 706759.607240879791789, 6754559.44302420783788 ], [ 706761.5, 6754557.586065614596009 ], [ 706760.506778866751119, 6754551.857554757036269 ], [ 706764.5, 6754538.586179815232754 ], [ 706764.5606427306775, 6754533.577802591957152 ], [ 706766.5, 6754530.586179815232754 ], [ 706766.527054196456447, 6754516.716397003270686 ], [ 706769.5, 6754510.586179815232754 ], [ 706772.5, 6754497.586179815232754 ], [ 706772.56458949635271, 6754488.564572984352708 ], [ 706775.5, 6754487.5 ], [ 706774.501806815736927, 6754479.926398488692939 ], [ 706775.5606427306775, 6754473.577802591957152 ], [ 706777.666427467018366, 6754467.313280043192208 ], [ 706780.5, 6754464.586065614596009 ], [ 706780.528822079417296, 6754458.707364517264068 ], [ 706782.5, 6754457.5 ], [ 706783.666427467018366, 6754451.313280043192208 ], [ 706787.5, 6754449.5 ], [ 706788.677118103485554, 6754441.292904894798994 ], [ 706793.5, 6754438.5 ], [ 706794.048410073737614, 6754432.840484320186079 ], [ 706800.5, 6754431.5 ], [ 706800.5, 6754429.5 ], [ 706791.5, 6754429.5 ], [ 706790.036811842699535, 6754431.49954822845757 ], [ 706787.913629377377219, 6754430.034310818649828 ], [ 706788.5, 6754421.413820184767246 ], [ 706786.501806815736927, 6754417.073601511307061 ], [ 706789.5, 6754405.586179815232754 ], [ 706789.5606427306775, 6754395.577802591957152 ], [ 706790.913629377377219, 6754391.965689181350172 ], [ 706794.5, 6754390.5 ], [ 706794.5, 6754388.5 ], [ 706792.528822079417296, 6754387.292635482735932 ], [ 706792.5, 6754380.0 ], [ 706795.5, 6754378.5 ], [ 706793.5, 6754375.0 ], [ 706793.506778866751119, 6754364.857554757036269 ], [ 706795.5, 6754357.586179815232754 ], [ 706794.501806815736927, 6754349.926398488692939 ], [ 706796.5, 6754339.586179815232754 ], [ 706797.528822079417296, 6754315.707364517264068 ], [ 706800.5, 6754313.586065614596009 ], [ 706800.5, 6754311.413934385403991 ], [ 706797.614180701202713, 6754309.574025148525834 ], [ 706796.5, 6754303.0 ], [ 706798.5, 6754295.586179815232754 ], [ 706797.528822079417296, 6754286.707364517264068 ], [ 706799.5, 6754285.5 ], [ 706799.5, 6754283.5 ], [ 706796.500451771891676, 6754282.036811839789152 ], [ 706796.527054196456447, 6754268.716397003270686 ], [ 706799.5, 6754265.5 ], [ 706796.5, 6754258.0 ], [ 706796.5, 6754233.0 ], [ 706798.5, 6754209.413934385403991 ], [ 706795.5606427306775, 6754205.422197408042848 ], [ 706795.5, 6754195.413934385403991 ], [ 706793.516235235147178, 6754193.220095711760223 ], [ 706794.5, 6754164.413820184767246 ], [ 706789.506778866751119, 6754141.142445242963731 ], [ 706789.506778866751119, 6754128.857554757036269 ], [ 706794.5, 6754116.586179815232754 ], [ 706797.614180701202713, 6754103.425974851474166 ], [ 706799.5, 6754102.5 ], [ 706799.939339828211814, 6754098.939339828677475 ], [ 706803.5, 6754097.5 ], [ 706801.5, 6754094.0 ], [ 706803.5, 6754075.586179815232754 ], [ 706803.5, 6754059.413820184767246 ], [ 706801.501806815736927, 6754045.073601511307061 ], [ 706802.5, 6754038.413934385403991 ], [ 706799.527054196456447, 6754032.283602996729314 ], [ 706798.5, 6754020.5 ], [ 706795.666427467018366, 6754018.686719956807792 ], [ 706792.5, 6754010.0 ], [ 706793.607240879791789, 6753999.44302420783788 ], [ 706798.5, 6753998.5 ], [ 706799.5, 6753992.5 ], [ 706795.292904894798994, 6753992.322881896980107 ], [ 706793.527054196456447, 6753989.283602996729314 ], [ 706793.506778866751119, 6753977.857554757036269 ], [ 706795.5, 6753969.5 ], [ 706793.5, 6753969.5 ], [ 706793.254821590846404, 6753971.821841089054942 ], [ 706791.606587939313613, 6753972.447489879094064 ], [ 706787.501806815736927, 6753966.926398488692939 ], [ 706791.577802593121305, 6753962.56064273044467 ], [ 706800.5, 6753963.5 ], [ 706802.073529037996195, 6753960.820317179895937 ], [ 706805.5, 6753959.5 ], [ 706805.5, 6753957.413820184767246 ], [ 706804.5, 6753951.5 ], [ 706802.5, 6753951.5 ], [ 706801.435427015880123, 6753954.435410503298044 ], [ 706798.507222909945995, 6753953.147025710903108 ], [ 706802.5, 6753946.586179815232754 ], [ 706800.501806815736927, 6753938.073601511307061 ], [ 706800.516235235147178, 6753931.779904288239777 ], [ 706802.5, 6753927.5 ], [ 706787.425974851474166, 6753922.385819299146533 ], [ 706784.586179815232754, 6753919.5 ], [ 706782.5, 6753919.5 ], [ 706781.5, 6753922.413820184767246 ], [ 706781.5, 6753926.5 ], [ 706785.220095711643808, 6753925.516235235147178 ], [ 706786.633416056167334, 6753927.426398488692939 ], [ 706786.472945803543553, 6753934.283602996729314 ], [ 706784.5, 6753936.413934385403991 ], [ 706783.472945803543553, 6753954.283602996729314 ], [ 706781.5, 6753958.413820184767246 ], [ 706782.389315363252535, 6753965.565511114895344 ], [ 706780.468754711793736, 6753966.402775265276432 ], [ 706777.5, 6753963.5 ], [ 706773.5, 6753962.5 ], [ 706774.471177920582704, 6753967.292635482735932 ], [ 706772.5, 6753968.5 ], [ 706771.5, 6753974.5 ], [ 706774.951589926262386, 6753974.840484320186079 ], [ 706775.322881896514446, 6753977.707095105201006 ], [ 706768.5, 6753978.5 ], [ 706770.5, 6753983.0 ], [ 706768.5, 6753985.413934385403991 ], [ 706768.247204418410547, 6753989.833355349488556 ], [ 706763.5, 6753990.5 ], [ 706761.5, 6753992.5 ], [ 706758.262332465150394, 6753999.81025721039623 ], [ 706751.5, 6754008.413934385403991 ], [ 706750.262332465150394, 6754012.81025721039623 ], [ 706745.5, 6754018.413934385403991 ], [ 706745.221054494613782, 6754022.871220936998725 ], [ 706742.5, 6754023.5 ], [ 706734.5, 6754032.5 ], [ 706734.5, 6754040.5 ], [ 706739.147025710437447, 6754039.507222910411656 ], [ 706740.5, 6754042.5 ], [ 706750.5, 6754042.5 ], [ 706753.677118103485554, 6754034.292904894798994 ], [ 706755.5, 6754033.5 ], [ 706755.671733715804294, 6754027.303071970120072 ], [ 706760.5, 6754027.5 ], [ 706762.614180701202713, 6754018.425974851474166 ], [ 706764.5, 6754017.5 ], [ 706766.752795581589453, 6754011.166644650511444 ], [ 706771.166644650511444, 6754006.752795581705868 ], [ 706778.852388428989798, 6754003.765725327655673 ], [ 706780.5, 6754010.5 ], [ 706784.4393572693225, 6754014.577802591957152 ], [ 706786.5, 6754031.586179815232754 ], [ 706790.471177920582704, 6754037.707364517264068 ], [ 706790.5, 6754050.586179815232754 ], [ 706792.493221133248881, 6754054.857554757036269 ], [ 706793.5, 6754074.0 ], [ 706791.472945803543553, 6754092.283602996729314 ], [ 706784.5, 6754108.413820184767246 ], [ 706780.5, 6754130.413820184767246 ], [ 706780.5, 6754141.586179815232754 ], [ 706783.472945803543553, 6754148.716397003270686 ], [ 706787.493221133248881, 6754173.857554757036269 ], [ 706789.498193184263073, 6754203.926398488692939 ], [ 706788.060660171788186, 6754208.060660171322525 ], [ 706783.5, 6754209.5 ], [ 706784.413934385520406, 6754212.5 ], [ 706788.385819298797287, 6754213.425974851474166 ], [ 706789.5, 6754223.0 ], [ 706788.385819298797287, 6754233.574025148525834 ], [ 706784.5, 6754235.5 ], [ 706784.5, 6754237.5 ], [ 706788.471177920582704, 6754238.707364517264068 ], [ 706789.483764764852822, 6754245.220095711760223 ], [ 706788.43541050364729, 6754253.435427015647292 ], [ 706786.5, 6754254.5 ], [ 706786.5, 6754256.5 ], [ 706788.247204418410547, 6754257.166644650511444 ], [ 706790.5, 6754262.0 ], [ 706788.5, 6754265.413934385403991 ], [ 706789.5, 6754286.0 ], [ 706787.5, 6754297.413820184767246 ], [ 706788.493221133248881, 6754305.142445242963731 ], [ 706786.5, 6754310.413820184767246 ], [ 706786.5, 6754312.5 ], [ 706788.5, 6754314.0 ], [ 706786.5, 6754348.5 ], [ 706788.496103194658644, 6754350.108051982708275 ], [ 706784.5, 6754371.413820184767246 ], [ 706784.471177920582704, 6754381.292635482735932 ], [ 706782.5, 6754382.5 ], [ 706783.633416056167334, 6754389.573601511307061 ], [ 706781.471177920582704, 6754404.292635482735932 ], [ 706779.5, 6754405.5 ], [ 706780.455046879593283, 6754409.364470270462334 ], [ 706777.5, 6754415.413820184767246 ], [ 706778.493221133248881, 6754426.142445242963731 ], [ 706776.5, 6754430.413820184767246 ], [ 706775.472945803543553, 6754445.283602996729314 ], [ 706772.5, 6754449.413820184767246 ], [ 706774.472945803543553, 6754456.283602996729314 ], [ 706772.951589926262386, 6754459.159515679813921 ], [ 706770.5, 6754459.5 ], [ 706768.471177920582704, 6754480.292635482735932 ], [ 706766.5, 6754481.5 ], [ 706765.5, 6754486.413820184767246 ], [ 706765.43541050364729, 6754496.435427015647292 ], [ 706761.5, 6754498.413934385403991 ], [ 706763.493221133248881, 6754504.142445242963731 ], [ 706761.5, 6754506.413934385403991 ], [ 706760.4393572693225, 6754518.422197408042848 ], [ 706758.5, 6754520.413934385403991 ], [ 706759.493221133248881, 6754525.142445242963731 ], [ 706756.5, 6754535.413820184767246 ], [ 706756.472945803543553, 6754545.283602996729314 ], [ 706750.5, 6754565.413820184767246 ], [ 706745.4393572693225, 6754595.422197408042848 ], [ 706742.5, 6754600.413820184767246 ], [ 706740.472945803543553, 6754620.283602996729314 ], [ 706736.5, 6754631.413820184767246 ], [ 706735.4393572693225, 6754642.422197408042848 ], [ 706732.5, 6754648.413820184767246 ], [ 706732.471177920582704, 6754660.292635482735932 ], [ 706730.5, 6754661.5 ] ] ] } } +] +} From e69c9a59220be14b804754f3200aec517da78d3f Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 2 Jul 2024 15:56:22 +0200 Subject: [PATCH 11/68] forget to remove a line --- skeleton/branch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/skeleton/branch.py b/skeleton/branch.py index 89627920..cabf086e 100644 --- a/skeleton/branch.py +++ b/skeleton/branch.py @@ -52,8 +52,6 @@ def __init__(self, config: DictConfig, branch_id: str, branch_mask: GeoDataFrame # self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) self.set_gdf_branch_mask(branch_mask) - self.gdf_branch_mask.to_file(f"/home/MDaab/code/lidro/branches/{self.branch_id}.geojson", driver='GeoJSON') - self.gap_points = [] # will contain points on the exterior that are connected to close gaps self.df_all_coords = get_df_points_from_gdf(self.gdf_branch_mask) From bfc4969dc22f1ea120a4431c3bad4e0f0268caf2 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 3 Jul 2024 09:28:54 +0200 Subject: [PATCH 12/68] Change to add DB credentials as GITHUB secrets --- .github/workflows/ci.yaml | 5 ++++- test/skeleton/test_main_skeleton.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 843eae07..3341130b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,5 +30,8 @@ jobs: generate-run-shell: true - name: Unit test shell: micromamba-shell {0} - run: python -m pytest -s --log-cli-level DEBUG + run: | + sed -i "s/USER = 'TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" test/skeleton/test_main_skeleton.py + sed -i "s/PASSWORD = 'TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" test/skeleton/test_main_skeleton.py + python -m pytest -s --log-cli-level DEBUG diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index 1c4c0d1f..b14fa29a 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -9,8 +9,8 @@ from test.skeleton.test_branch import read_branch -USER = "TO_DEFINE" -PASSWORD = "TO_DEFINE" +USER = 'TO_BE_DEFINED' +PASSWORD = 'TO_BE_DEFINED' CRS = 2154 From 316a8c71465c4dc2f734dec084427d98fcaa57cf Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 3 Jul 2024 10:13:11 +0200 Subject: [PATCH 13/68] add dot_env for local variables (like passwords) --- .env_example | 2 ++ .github/workflows/ci.yaml | 7 ++++--- .gitignore | 4 +++- environment.yml | 1 + test/skeleton/test_main_skeleton.py | 15 ++++++++------- 5 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 .env_example diff --git a/.env_example b/.env_example new file mode 100644 index 00000000..e6b57ebd --- /dev/null +++ b/.env_example @@ -0,0 +1,2 @@ +DB_UNI_USER='TO_BE_DEFINED' +DB_UNI_PASSWORD='TO_BE_DEFINED' \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3341130b..c6e4d91c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,8 @@ jobs: - name: Unit test shell: micromamba-shell {0} run: | - sed -i "s/USER = 'TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" test/skeleton/test_main_skeleton.py - sed -i "s/PASSWORD = 'TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" test/skeleton/test_main_skeleton.py - python -m pytest -s --log-cli-level DEBUG + cp .env_example .env + sed -i "s/DB_UNI_USER='TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" test/skeleton/test_main_skeleton.py + sed -i "s/DB_UNI_PASSWORD='TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" test/skeleton/test_main_skeleton.py + pytest diff --git a/.gitignore b/.gitignore index 08b71877..08e57ebc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ **/__pycache__ tmp -main.log \ No newline at end of file +main.log + +.env \ No newline at end of file diff --git a/environment.yml b/environment.yml index 490969a9..0899ee3d 100644 --- a/environment.yml +++ b/environment.yml @@ -17,6 +17,7 @@ dependencies: - python-pdal>=3.2.1 - shapely>=2.0.3 - psycopg2-binary + - python-dotenv # --------- hydra configs --------- # - hydra-core - hydra-colorlog diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index b14fa29a..ff915b2f 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -1,19 +1,20 @@ -from hydra import compose, initialize +import os +from hydra import compose, initialize import pandas as pd import geopandas as gpd +from dotenv import load_dotenv from skeleton.main_skeleton import create_branches_list, create_branches_pair from skeleton.main_skeleton import select_candidates, query_db_for_bridge_across_gap from skeleton.branch import Candidate - from test.skeleton.test_branch import read_branch -USER = 'TO_BE_DEFINED' -PASSWORD = 'TO_BE_DEFINED' +load_dotenv() +DB_UNI_USER = os.getenv('DB_UNI_USER') +DB_UNI_PASSWORD = os.getenv('DB_UNI_PASSWORD') CRS = 2154 - MAIN_SKELETON_TEST_1_1_PATH = "test_files/40.geojson" MAIN_SKELETON_TEST_1_2_PATH = "test_files/43.geojson" MAIN_SKELETON_TEST_1_3_PATH = "test_files/44.geojson" @@ -50,8 +51,8 @@ def test_main_skeleton_2(): config = compose( config_name="configs_skeleton.yaml", overrides=[ - f"DB_UNI.DB_USER={USER}", - f'DB_UNI.DB_PASSWORD="{PASSWORD}"', + f"DB_UNI.DB_USER={DB_UNI_USER}", + f'DB_UNI.DB_PASSWORD="{DB_UNI_PASSWORD}"', ], ) dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) From c7e24ca00da01eea7f2abc22c15a9bd69ce36baf Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 3 Jul 2024 10:24:03 +0200 Subject: [PATCH 14/68] unittesting for git action --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6e4d91c..2066af0a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,5 +34,4 @@ jobs: cp .env_example .env sed -i "s/DB_UNI_USER='TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" test/skeleton/test_main_skeleton.py sed -i "s/DB_UNI_PASSWORD='TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" test/skeleton/test_main_skeleton.py - pytest - + pytest -s -k test_main_skeleton_2 From f29eb49911ce8c5d4cdf1bc974e214ba70a80646 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 3 Jul 2024 10:36:36 +0200 Subject: [PATCH 15/68] change gitignore --- .github/workflows/ci.yaml | 6 +++--- .gitignore | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2066af0a..976b12e3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,6 +32,6 @@ jobs: shell: micromamba-shell {0} run: | cp .env_example .env - sed -i "s/DB_UNI_USER='TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" test/skeleton/test_main_skeleton.py - sed -i "s/DB_UNI_PASSWORD='TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" test/skeleton/test_main_skeleton.py - pytest -s -k test_main_skeleton_2 + sed -i "s/DB_UNI_USER='TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" .env + sed -i "s/DB_UNI_PASSWORD='TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" .env + python -m pytest -s --log-cli-level DEBUG diff --git a/.gitignore b/.gitignore index 08e57ebc..82ec4846 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ tmp main.log -.env \ No newline at end of file +.env + +outputs/ \ No newline at end of file From 26f16e514912305253b20b10db625a816527fb06 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 3 Jul 2024 10:58:42 +0200 Subject: [PATCH 16/68] fix typo for credentials in ci.yaml --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 976b12e3..21b7395a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,6 +32,6 @@ jobs: shell: micromamba-shell {0} run: | cp .env_example .env - sed -i "s/DB_UNI_USER='TO_BE_DEFINED'/USER = ${{ secrets.DB_UNI_USER }}/g" .env - sed -i "s/DB_UNI_PASSWORD='TO_BE_DEFINED'/PASSWORD = ${{ secrets.DB_UNI_PASSWORD }}/g" .env + sed -i "s/DB_UNI_USER='TO_BE_DEFINED'/DB_UNI_USER=${{ secrets.DB_UNI_USER }}/g" .env + sed -i "s/DB_UNI_PASSWORD='TO_BE_DEFINED'/DB_UNI_PASSWORD=${{ secrets.DB_UNI_PASSWORD }}/g" .env python -m pytest -s --log-cli-level DEBUG From 9a77787b27bd745df0585d006cb637caf0b2f39a Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 3 Jul 2024 15:42:05 +0200 Subject: [PATCH 17/68] Some corrections following PR comments --- configs/configs_lidro.yaml | 26 +++++++++ configs/configs_skeleton.yaml | 32 ----------- {skeleton => lidro/skeleton}/__init__.py | 0 {skeleton => lidro/skeleton}/branch.py | 10 ++-- {skeleton => lidro/skeleton}/group_maker.py | 13 ++++- {skeleton => lidro/skeleton}/main_skeleton.py | 57 +++++++++++-------- test/skeleton/test_branch.py | 6 +- test/skeleton/test_group_maker.py | 2 +- test/skeleton/test_main_skeleton.py | 40 ++++++------- 9 files changed, 97 insertions(+), 89 deletions(-) delete mode 100644 configs/configs_skeleton.yaml rename {skeleton => lidro/skeleton}/__init__.py (100%) rename {skeleton => lidro/skeleton}/branch.py (97%) rename {skeleton => lidro/skeleton}/group_maker.py (67%) rename {skeleton => lidro/skeleton}/main_skeleton.py (81%) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 4eee3a7b..a5b68cb5 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -34,6 +34,32 @@ filter: # Classes to be considered as "non-water" keep_classes: [0, 1, 2, 3, 4, 5, 6, 17, 64, 65, 66, 67] # All classes +SKELETON: + FILE_PATH: + MASK_INPUT_PATH: + SKELETON_LINES_OUTPUT_PATH: # only branches' skeleton (if empty, no file is created) + GAP_LINES_OUTPUT_PATH: # only new lines to close gaps (if empty, no file is created) + GLOBAL_LINES_OUTPUT_PATH: # lines from both the skeletons and the gap lines + + MAX_GAP_WIDTH: 200 # distance max in meter of any gap between 2 branches we will try to close with a line + MAX_BRIDGES: 1 # number max of bridges that can be created between 2 branches + GAP_WIDTH_CHECK_DB: 30 # for a gap at least this wide, we check with DB_UNI. Smaller than that, we assume a line automatically exists + RATIO_GAP: 0.25 # when checking if a candidate line to close a gap intersects a bridge in BD UNI, that's the ratio ]0.1] of the line to consider. + # we only check part of the line, because when the line is really long it can intersect with bridges not corresponding to that line, + # so we make the line smaller + + DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: + DB_PASSWORD: + DB_PORT: 5432 + + BRANCH: + VORONOI_MAX_LENGTH: 2 # max size of a voronoi line + WATER_MIN_SIZE: 20 # min size of a skeleton line to be sure not to be removed + MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches + hydra: output_subdir: null run: diff --git a/configs/configs_skeleton.yaml b/configs/configs_skeleton.yaml deleted file mode 100644 index f8fb2c87..00000000 --- a/configs/configs_skeleton.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# @package _global_ - -# path to original working directory -# hydra hijacks working directory by changing it to the current log directory, -# so it's useful to have this path as a special variable -# learn more here: https://hydra.cc/docs/next/tutorials/basic/running_your_app/working_directory -work_dir: ${hydra:runtime.cwd} - -FILE_PATH: - MASK_INPUT_PATH: - SKELETON_LINES_OUTPUT_PATH: # only branches' skeleton (if empty, no file is created) - GAP_LINES_OUTPUT_PATH: # only new lines to close gaps (if empty, no file is created) - GLOBAL_LINES_OUTPUT_PATH: # lines from both the skeletons and the gap lines - -MAX_GAP_WIDTH: 200 # distance max in meter of any gap between 2 branches we will try to close with a line -MAX_BRIDGES: 1 # number max of bridges that can be created between 2 branches -GAP_WIDTH_CHECK_DB: 30 # for a gap at least this wide, we check with DB_UNI. Smaller than that, we assume a line automatically exists -RATIO_GAP: 0.25 # when checking if a candidate line to close a gap intersects a bridge in BD UNI, that's the ratio ]0.1] of the line to consider. - # we only check part of the line, because when the line is really long it can intersect with bridges not corresponding to that line, - # so we make the line smaller - -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: - DB_PASSWORD: - DB_PORT: 5432 - -BRANCH: - VORONOI_MAX_LENGTH: 2 # max size of a voronoi line - WATER_MIN_SIZE: 20 # min size of a skeleton line to be sure not to be removed - MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches \ No newline at end of file diff --git a/skeleton/__init__.py b/lidro/skeleton/__init__.py similarity index 100% rename from skeleton/__init__.py rename to lidro/skeleton/__init__.py diff --git a/skeleton/branch.py b/lidro/skeleton/branch.py similarity index 97% rename from skeleton/branch.py rename to lidro/skeleton/branch.py index cabf086e..be3a4e36 100644 --- a/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -48,8 +48,6 @@ def __init__(self, config: DictConfig, branch_id: str, branch_mask: GeoDataFrame self.config = config self.branch_id = branch_id self.crs = crs - # simplify_geom = mask_branch.simplify(tolerance=2) # simplifying geometries with Douglas-Peucker - # self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[simplify_geom], crs=crs) self.set_gdf_branch_mask(branch_mask) self.gap_points = [] # will contain points on the exterior that are connected to close gaps @@ -135,10 +133,10 @@ def get_candidates(self, other_branch: 'Branch') -> List[Candidate]: candidates = [] for index, (other_index, self_index) in enumerate(zip(indexes[0], indexes[1])): # stop if we have enough candidates - if index >= self.config.BRANCH.MAX_GAP_CANDIDATES: + if index >= self.config.SKELETON.BRANCH.MAX_GAP_CANDIDATES: break # stop if the following candidates - if distance_squared[other_index][self_index] > self.config.MAX_GAP_WIDTH * self.config.MAX_GAP_WIDTH: + if distance_squared[other_index][self_index] > self.config.SKELETON.MAX_GAP_WIDTH * self.config.SKELETON.MAX_GAP_WIDTH: break candidates.append( @@ -237,7 +235,7 @@ def can_line_be_removed(self, line: Geometry, vertices_dict: Dict[Point, List[Li - line (Geometry) : the line to test - vertices_dict (Dict[Point, List[LineString]]) : a dictionary off all the lines with a point as an extremity """ - if line.length > self.config.BRANCH.WATER_MIN_SIZE: + if line.length > self.config.SKELETON.BRANCH.WATER_MIN_SIZE: return False point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] @@ -259,7 +257,7 @@ def create_voronoi_lines(self) -> GeoDataFrame: """ # divide geometry into segments no longer than max_segment_length united_geom = self.gdf_branch_mask['geometry'].unary_union - segmentize_geom = united_geom.segmentize(max_segment_length=self.config.BRANCH.VORONOI_MAX_LENGTH) + segmentize_geom = united_geom.segmentize(max_segment_length=self.config.SKELETON.BRANCH.VORONOI_MAX_LENGTH) # Create the voronoi diagram and only keep polygon regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) diff --git a/skeleton/group_maker.py b/lidro/skeleton/group_maker.py similarity index 67% rename from skeleton/group_maker.py rename to lidro/skeleton/group_maker.py index 0221ee78..80260a0b 100644 --- a/skeleton/group_maker.py +++ b/lidro/skeleton/group_maker.py @@ -1,8 +1,15 @@ class GroupMaker: + """ + From a list of elements, GroupMaker is used to + group sets of elements according to single element from + the sets. For example: + At the beginning, we have {A}, {B}, {C}, {D}, {E} + put_together(A, B) -> {A, B}, {C}, {D}, {E} + put_together(C, D) -> {A, B}, {C, D}, {E} + put_together(A, D) -> {A, B, C, D}, {E} + """ def __init__(self, element_list): - self.element_set_list = [] - for element in element_list: - self.element_set_list.append({element}) + self.element_set_list = [{element} for element in element_list] def find_index(self, element) -> int: for index, element_set in enumerate(self.element_set_list): diff --git a/skeleton/main_skeleton.py b/lidro/skeleton/main_skeleton.py similarity index 81% rename from skeleton/main_skeleton.py rename to lidro/skeleton/main_skeleton.py index bf73c943..92d024ac 100644 --- a/skeleton/main_skeleton.py +++ b/lidro/skeleton/main_skeleton.py @@ -6,6 +6,7 @@ import geopandas as gpd import pandas as pd +from pandas import DataFrame as df from geopandas.geodataframe import GeoDataFrame import psycopg2 from shapely.geometry import Point @@ -21,11 +22,11 @@ def db_connector(config: DictConfig): - config (DictConfig): the config dict from hydra """ return psycopg2.connect( - database=config.DB_UNI.DB_NAME, - host=config.DB_UNI.DB_HOST, - user=config.DB_UNI.DB_USER, - password=config.DB_UNI.DB_PASSWORD, - port=config.DB_UNI.DB_PORT + database=config.SKELETON.DB_UNI.DB_NAME, + host=config.SKELETON.DB_UNI.DB_HOST, + user=config.SKELETON.DB_UNI.DB_USER, + password=config.SKELETON.DB_UNI.DB_PASSWORD, + port=config.SKELETON.DB_UNI.DB_PORT ) @@ -41,21 +42,23 @@ def query_db_for_bridge_across_gap(config: DictConfig, candidate: Candidate) -> # bridge from another area) middle_x = (candidate.extremity_1[0] + candidate.extremity_2[0]) / 2 middle_y = (candidate.extremity_1[1] + candidate.extremity_2[1]) / 2 - new_ext_1_x = (candidate.extremity_1[0] - middle_x) * config.RATIO_GAP + middle_x - new_ext_1_y = (candidate.extremity_1[1] - middle_y) * config.RATIO_GAP + middle_y - new_ext_2_x = (candidate.extremity_2[0] - middle_x) * config.RATIO_GAP + middle_x - new_ext_2_y = (candidate.extremity_2[1] - middle_y) * config.RATIO_GAP + middle_y + new_ext_1_x = (candidate.extremity_1[0] - middle_x) * config.SKELETON.RATIO_GAP + middle_x + new_ext_1_y = (candidate.extremity_1[1] - middle_y) * config.SKELETON.RATIO_GAP + middle_y + new_ext_2_x = (candidate.extremity_2[0] - middle_x) * config.SKELETON.RATIO_GAP + middle_x + new_ext_2_y = (candidate.extremity_2[1] - middle_y) * config.SKELETON.RATIO_GAP + middle_y # creation of queries line = f"LINESTRING({new_ext_1_x} {new_ext_1_y}, {new_ext_2_x} {new_ext_2_y})" query_linear = ( "SELECT cleabs FROM public.Construction_lineaire " "WHERE gcms_detruit = false " + "AND nature = 'Pont' " f"AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" ) query_area = ( "SELECT cleabs FROM public.Construction_surfacique " "WHERE gcms_detruit = false " + "AND nature = 'Pont' " f"AND ST_Intersects(ST_Force2D(geometrie), ST_GeomFromText('{line}'));" ) @@ -69,6 +72,9 @@ def query_db_for_bridge_across_gap(config: DictConfig, candidate: Candidate) -> db_cursor.execute(query_area) results = db_cursor.fetchall() + if results: + pass + return True if results else False @@ -85,7 +91,7 @@ def select_candidates( """ # sort the branches pairs by distance between each pair (so the first gap to be closed is the smallest) - branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) + branches_pair_list = sorted(branches_pair_list, key=lambda branches_pair: branches_pair[2]) # get all the branches (each only once, hence the set) branch_set = set() @@ -119,7 +125,7 @@ def select_candidates( # if the gap is wide enough, we check with DB_Uni to see if there is a bridge # On the other hand, if it's small enough the candidate is automatically validated - if candidate.squared_distance > config.GAP_WIDTH_CHECK_DB * config.GAP_WIDTH_CHECK_DB: + if candidate.squared_distance > config.SKELETON.GAP_WIDTH_CHECK_DB * config.SKELETON.GAP_WIDTH_CHECK_DB: is_bridge = query_db_for_bridge_across_gap(config, candidate) # if the line does not cross any bridge, we don't validate that candidate if not is_bridge: @@ -132,7 +138,7 @@ def select_candidates( # a candidate has been validated between A and B, so we put together A and B branch_group.put_together(branch_a, branch_b) nb_bridges_crossed += 1 - if nb_bridges_crossed >= config.MAX_BRIDGES: # max bridges reached between those 2 branches + if nb_bridges_crossed >= config.SKELETON.MAX_BRIDGES: # max bridges reached between those 2 branches break return validated_candidates @@ -158,7 +164,7 @@ def create_branches_list(config: DictConfig, gdf_hydro_global_mask: GeoDataFrame def create_branches_pair(config: DictConfig, branches_list: List[Branch]) -> List[Tuple[Candidate, Candidate, float]]: """ create a list of pairs of branches, containing all the pairs of branches close enough from each other - for the water to flow anyway between them, and return them sorted by distance + for the water to flow anyway between them Args: - config (DictConfig): the config dict from hydra """ @@ -168,32 +174,35 @@ def create_branches_pair(config: DictConfig, branches_list: List[Branch]) -> Lis for index, branch_a in enumerate(branches_list[:-1]): for branch_b in branches_list[index + 1:]: distance = branch_a.distance_to_a_branch(branch_b) - if distance < config.MAX_GAP_WIDTH: + if distance < config.SKELETON.MAX_GAP_WIDTH: branches_pair_list.append((branch_a, branch_b, distance)) return branches_pair_list -@hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_skeleton.yaml") +@hydra.main(version_base="1.2", config_path="../../configs/", config_name="configs_lidro.yaml") def run(config: DictConfig): """ - Get whole hydrographic skeleton + Get whole hydrographic skeleton lines, based on + a geojson file containing multiple river contours. + The result file will contain "skeleton" lines running in the middle of + those contours to describe the flow of the rivers, and lines crossing the contours + to join with the skeleton lines, as bridge and other elements may "break" river contours args: - config (DictConfig): the config dict from hydra """ - gdf_hydro_global_mask = gpd.read_file(config.FILE_PATH.MASK_INPUT_PATH) + gdf_hydro_global_mask = gpd.read_file(config.SKELETON.FILE_PATH.MASK_INPUT_PATH) crs = gdf_hydro_global_mask.crs # Load a crs from input branches_list = create_branches_list(config, gdf_hydro_global_mask, crs) branches_pair_list = create_branches_pair(config, branches_list) - validated_candidates = select_candidates(config, branches_pair_list) # create the gap lines from the selected candidates gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] - if config.FILE_PATH.GAP_LINES_OUTPUT_PATH: + if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) - gdf_gap_lines.to_file(config.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') + gdf_gap_lines.to_file(config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, # to create a new line toward that extremity and know not to remove it during the "simplify" @@ -207,16 +216,16 @@ def run(config: DictConfig): branch.create_skeleton() branch.simplify() - # putting all skeleton lines together, and save them if their is a path + # putting all skeleton lines together, and save them if there is a path branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - if config.FILE_PATH.GAP_LINES_OUTPUT_PATH: - gdf_branch_lines.to_file(config.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') + if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: + gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') # saving all lines branch_lines_list.append(gdf_gap_lines) gdf_global_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - gdf_global_lines.to_file(config.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') + gdf_global_lines.to_file(config.SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') if __name__ == "__main__": diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index 7e0eb187..5e33a72f 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -4,9 +4,9 @@ import geopandas as gpd from omegaconf import DictConfig -from skeleton.branch import Branch, get_vertices_dict +from lidro.skeleton.branch import Branch, get_vertices_dict -sys.path.append('skeleton') +sys.path.append('lidro/skeleton') BRANCH_TEST_1_PATH = "test_files/90.geojson" @@ -22,7 +22,7 @@ def test_branch_1(): """test creation/simplification of skeleton lines for a branch""" with initialize(version_base="1.2", config_path="../../configs"): config = compose( - config_name="configs_skeleton.yaml", + config_name="configs_lidro.yaml", ) branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") diff --git a/test/skeleton/test_group_maker.py b/test/skeleton/test_group_maker.py index 7c4983b4..f30fff5b 100644 --- a/test/skeleton/test_group_maker.py +++ b/test/skeleton/test_group_maker.py @@ -1,4 +1,4 @@ -from skeleton.group_maker import GroupMaker +from lidro.skeleton.group_maker import GroupMaker def test_group_maker(): diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index ff915b2f..68da856a 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -5,9 +5,9 @@ import geopandas as gpd from dotenv import load_dotenv -from skeleton.main_skeleton import create_branches_list, create_branches_pair -from skeleton.main_skeleton import select_candidates, query_db_for_bridge_across_gap -from skeleton.branch import Candidate +from lidro.skeleton.main_skeleton import create_branches_list, create_branches_pair +from lidro.skeleton.main_skeleton import select_candidates, query_db_for_bridge_across_gap +from lidro.skeleton.branch import Candidate from test.skeleton.test_branch import read_branch load_dotenv() @@ -23,7 +23,7 @@ def test_main_skeleton_1(): with initialize(version_base="1.2", config_path="../../configs"): config = compose( - config_name="configs_skeleton.yaml", + config_name="configs_lidro.yaml", ) branch_2_1 = read_branch(config, MAIN_SKELETON_TEST_1_1_PATH, "test_main_skeleton_1_1") @@ -45,20 +45,20 @@ def test_main_skeleton_1(): assert (candidate_2.squared_distance < 75) -def test_main_skeleton_2(): - """Test : query_db_for_bridge_across_gap """ - with initialize(version_base="1.2", config_path="../../configs"): - config = compose( - config_name="configs_skeleton.yaml", - overrides=[ - f"DB_UNI.DB_USER={DB_UNI_USER}", - f'DB_UNI.DB_PASSWORD="{DB_UNI_PASSWORD}"', - ], - ) - dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) - dummy_candidate_2 = Candidate(None, None, (690880.5, 6737964.5), (690830.5, 6737951.586065615), 0) +# def test_main_skeleton_2(): +# """Test : query_db_for_bridge_across_gap """ +# with initialize(version_base="1.2", config_path="../../configs"): +# config = compose( +# config_name="configs_lidro.yaml", +# overrides=[ +# f"SKELETON.DB_UNI.DB_USER={DB_UNI_USER}", +# f'SKELETON.DB_UNI.DB_PASSWORD="{DB_UNI_PASSWORD}"', +# ], +# ) +# dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) +# dummy_candidate_2 = Candidate(None, None, (689272.5, 6760595.5), (689322.5, 6760553.5), 0) - is_bridge_1 = query_db_for_bridge_across_gap(config, dummy_candidate_1) - is_bridge_2 = query_db_for_bridge_across_gap(config, dummy_candidate_2) - assert not is_bridge_1 - assert is_bridge_2 +# is_bridge_1 = query_db_for_bridge_across_gap(config, dummy_candidate_1) +# is_bridge_2 = query_db_for_bridge_across_gap(config, dummy_candidate_2) +# assert not is_bridge_1 +# assert is_bridge_2 From d16339cbd6655bc490a1f288657db09c14e1968c Mon Sep 17 00:00:00 2001 From: mdupays Date: Fri, 26 Apr 2024 16:48:47 +0200 Subject: [PATCH 18/68] Refacto README, Dockerfile and test --- Dockerfile | 16 ++++++ Makefile | 40 +++++++++++++++ README.md | 50 ++++++++----------- ci/test.sh | 1 + configs/configs_lidro.yaml | 13 +---- .../rasters/create_mask_raster.py | 26 ++++++---- .../vectors/convert_to_vector.py | 4 +- lidro/main_create_mask.py | 5 +- main_create_mask.log | 44 ++++++++++++++++ test/rasters/test_create_mask_raster.py | 9 +--- test/vectors/test_convert_to_vector.py | 33 +++++++++--- 11 files changed, 172 insertions(+), 69 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 ci/test.sh create mode 100644 main_create_mask.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e69d4f0a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM mambaorg/micromamba:latest as mamba_pdal +COPY environment.yml /environment.yml +USER root +RUN micromamba env create -n lidro -f /environment.yml + +FROM debian:bullseye-slim + + +WORKDIR /lidro +RUN mkdir tmp +COPY lidro lidro +COPY test test +COPY configs configs + +# Copy test data that are stored directly in the lidro-data repository ("http://gitlab.forge-idi.ign.fr/Lidar/lidro-data.git") +COPY data/pointcloud data/pointcloud \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..001ed431 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +# Makefile to manage main tasks +# cf. https://blog.ianpreston.ca/conda/python/bash/2020/05/13/conda_envs.html#makefile + +# Oneshell means I can run multiple lines in a recipe in the same shell, so I don't have to +# chain commands together with semicolon +.ONESHELL: +SHELL = /bin/bash +install: + pip install -e . + +install-precommit: + pre-commit install + +testing: + ./ci/test.sh + +mamba-env-create: + mamba env create -n lidro -f environment.yml + +mamba-env-update: + mamba env update -n lidro -f environment.yml + +############################## +# Docker +############################## + +PROJECT_NAME=ignimagelidar/lidro +VERSION=`python -m lidro.version` + +docker-build: + docker build -t ${PROJECT_NAME}:${VERSION} -f Dockerfile . + +docker-test: + docker run --rm -it ${PROJECT_NAME}:${VERSION} python -m pytest -s + +docker-remove: + docker rmi -f `docker images | grep ${PROJECT_NAME} | tr -s ' ' | cut -d ' ' -f 3` + +docker-deploy: + docker push ${PROJECT_NAME}:${VERSION} \ No newline at end of file diff --git a/README.md b/README.md index 8446a3ab..431e7da9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -# LIDRO - +### LIDRO Lidro (Aplanissement des surfaces d'eaux) est un outil permettant de créer automatiquement des points virtuels le long des surfaces d'eaux afin de créer des modèles numériques cohérents avec les modèles hydrologiques. Le jeu de données en entrée correspond à un nuage des points LIDAR classés. ## Contexte @@ -16,36 +15,31 @@ Les données en entrées : - données vectorielles représentant le réseau hydrographique issu des différentes bases de données IGN (BDUnis, BDTopo, etc.) Trois grands axes du processus à mettre en place en distanguant l'échelle de traitmeent associé : -* 1- Création de masques hydrographiques à l'échelle de la dalle LIDAR -* 2- Création de masques hydrographiques pré-filtrés à l'échelle du chantier, soit : - * la suppression de ces masques dans les zones ZICAD/ZIPVA - * la suppression des aires < 150 m² - * la suppression des aires < 1000 m² hors BD IGN (grands cours d'eau < 5m de large) -* 3- Création de points virtuels le long de deux entités hydrographiques : - * Grands cours d'eau (> 5 m de large dans la BD Unis). - * Surfaces planes (mer, lac, étang, etc.) - -### Traitement des grands cours d'eau (> 5 m de large dans la BD Uns). - -Il existe plusieurs étapes intermédiaires : -* 1- création automatique du tronçon hydrographique ("Squelette hydrographique", soit les tronçons hydrographiques dans la BD Unid) à partir de l'emprise du masque hydrographique "écoulement" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle) -* 2- Analyse de la répartition en Z de l'ensemble des points LIDAR "Sol" -* 3- Création de points virtuels nécéssitant plusieurs étapes intermédiaires : - * Associer chaque point virtuel 2D au point le plus proche du squelette hydrographique - * Traitement "Z" du squelette : +1- Création de masques hydrographiques à l'échelle de la dalle LIDAR +2- Création de masques hydrographiques pré-filtrés à l'échelle du chantier, soit : +* la suppression de ces masques dans les zones ZICAD/ZIPVA +* la suppression des aires < 150 m² +* la suppression des aires < 1000 m² hors BD IGN (grands cours d'eaux > 5m de large) + +3- Création de points virtuels le long de deux entités hydrographiques : +* ##### Grands cours d'eaux (> 5 m de large dans la BD Unis). +Il existe plusieurs étapes intermédiaires : +1- création automatique du tronçon hydrographique ("Squelette hydrographique", soit les tronçons hydrographiques dans la BD Unid) à partir de l'emprise du masque hydrographique "écoulement" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle) +2- Analyse de la répartition en Z de l'ensemble des points LIDAR "Sol" +3- Créations de points virtuels nécéssitant plusieurs étapes intermédiaires : +* Associer chaque point virtuel 2D au point le plus proche du squelette hydrographique +* Traitement "Z" du squelette : * Analyser la répartition en Z de l’ensemble des points LIDAR extrait à l’étape précédente afin d’extraire une seule valeur d’altitude au point fictif Z le plus proche du squelette hydrographique. A noter que l’altitude correspond donc à la ligne basse de la boxplot, soit la valeur minimale en excluant les valeurs aberrantes. Pour les ponts, une étape de contrôle de la classification pourrait être mise en place * Lisser les Z le long du squelette HYDRO pour assurer l'écoulement - * Création des points virtuels 2D tous les 0.5 mètres le long des bords du masque hydrographique "écoulement" en cohérence en "Z" avec le squelette hydrographique +* Création des points virtuels 2D tous les 0.5 mètres le long des bords du masque hydrographique "écoulement" en cohérence en "Z" avec le squelette hydrographique -### Traitement des surfaces planes (mer, lac, étang, etc.) +* ##### Surfaces planes (mer, lac, étang, etc.) Pour rappel, l'eau est considérée comme horizontale sur ce type de surface. - -Il existe plusieurs étapes intermédiaires : -* 1- Extraction et enregistrement temporairement des points LIDAR classés en « Sol » et « Eau » présents potentiellement à la limite +1 mètre des berges. Pour cela, on s'appuie sur 'emprise du masque hydrographique "surface plane" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle). a noter que pur le secteur maritime (Mer), il faut exclure la classe « 9 » (eau) afin d’éviter de mesurer les vagues. -* 2- Analyse statistique de l'ensemble des points LIDAR "Sol / Eau" le long des côtes/berges afin d'obtenir une surface plane. - L’objectif est de créer des points virtuels spécifiques avec une information d'altitude (m) tous les 0.5 m sur les bords des surfaces maritimes et des plans d’eau à partir du masque hydrographique "surface plane". Pour cela, il existe plusieurs étapes intermédaires : - * Filtrer 30% des points LIDAR les plus bas de l’étape 1. afin de supprimer les altitudes trop élevées - * Analyser la répartition en Z de ces points afin d’extraire une seule valeur d’altitude selon l’objet hydrographique : +Il existe plusieurs étapes intermédiaires : +1- Extraction et enregistrement temporairement des points LIDAR classés en « Sol » et « Eau » présents potentiellement à la limite +1 mètre des berges. Pour cela, on s'appuie sur 'emprise du masque hydrographique "surface plane" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle). a noter que pur le secteur maritime (Mer), il faut exclure la classe « 9 » (eau) afin d’éviter de mesurer les vagues. +2- Analyse statisque de l'ensemble des points LIDAR "Sol / Eau" le long des côtes/berges afin d'obtenir une surface plane. L’objectif est de créer des points virtuels spécifiques avec une information d'altitude (m) tous les 0.5 m sur les bords des surfaces maritimes et des plans d’eaux à partir du masque hydrographique "surface plane". Pour cela, il existe plusieurs étapes intermédaires : +* Filtrer 30% des points LIDAR les plus bas de l’étape 1). afin de supprimer les altitudes trop élevées +* Analyser la répartition en Z de ces points afin d’extraire une seule valeur d’altitude selon l’objet hydrographique : * Pour les Plans d’eau : l’altitude correspond à la ligne basse de la boxplot, soit la valeur minimale en excluant les valeurs aberrantes, * Pour la Mer : l’altitude correspond à la ligne basse de la boxplot, soit la valeur minimale en excluant les valeurs aberrantes diff --git a/ci/test.sh b/ci/test.sh new file mode 100644 index 00000000..823d3d84 --- /dev/null +++ b/ci/test.sh @@ -0,0 +1 @@ +python -m pytest -s ./test -v \ No newline at end of file diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index a5b68cb5..81261d13 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -19,17 +19,8 @@ io: raster: # size for dilatation - dilation_size: 3 - -vector: - # Filter water's area (m²) - min_water_area: 150 - # Parameters for buffer - buffer_positive: 1 - buffer_negative: -1.5 # negative buffer should be bigger than positive buffer to prevent protruding over the banks - # Tolerance from Douglas-Peucker - tolerance: 1 - + dilatation_size: 3 + filter: # Classes to be considered as "non-water" keep_classes: [0, 1, 2, 3, 4, 5, 6, 17, 64, 65, 66, 67] # All classes diff --git a/lidro/create_mask_hydro/rasters/create_mask_raster.py b/lidro/create_mask_hydro/rasters/create_mask_raster.py index 18fb41e0..d7746510 100644 --- a/lidro/create_mask_hydro/rasters/create_mask_raster.py +++ b/lidro/create_mask_hydro/rasters/create_mask_raster.py @@ -33,7 +33,7 @@ def create_occupancy_map(points: np.array, tile_size: int, pixel_size: float, or return bins -def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, classes: List[int], dilation_size: int): +def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, classes: List[int], dilatation_size: int): """ "Detect hydrographic surfaces in a tile from the classified points of the input pointcloud An hydrographic surface is defined as a surface where there is no points from any class different from water The output hydrographic surface mask is dilated to make sure that the masks are continuous when merged with their @@ -46,10 +46,18 @@ def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, class classification values are ignored) dilation_size (int): size of the structuring element for dilation - Returns: - smoothed_water (np.array): 2D binary array (x, y) of the water presence from the point cloud - pcd_origin (list): top left corner of the tile containing the point cloud - (infered from pointcloud bounding box and input tile size) + Args: + filename (str): input pointcloud + tile_size (int): size of the raster grid (in meters) + pixel_size (float): distance between each node of the raster grid (in meters) + classes (List[int]): List of classes to use for the binarisation (points with other + classification values are ignored) + dilatation_size (int): size for dilatation raster + + Returns: + smoothed_water (np.array): 2D binary array (x, y) of the water presence from the point cloud + pcd_origin (list): top left corner of the tile containing the point cloud + (infered from pointcloud bounding box and input tile size) """ # Read pointcloud, and extract coordinates (X, Y, Z, and classification) of all points array, crs = read_pointcloud(filename) @@ -66,11 +74,9 @@ def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, class # Revert occupancy map to keep pixels where there is no point of the selected classes detected_water = np.logical_not(occupancy) - # Apply a mathematical morphology operations: DILATION - # / ! \ NOT "CLOSING", due to the reduction in the size of hydro masks (tile borders) + # Apply a mathematical morphology operations: DILATATION + # / ! \ NOT "CLOSING", due to the reduction in the size of hydro masks, particularly at the tile borders. # / ! \ WITH "CLOSING" => Masks Hydro are no longer continuous, when they are merged - water_mask = scipy.ndimage.binary_dilation( - detected_water, structure=np.ones((dilation_size, dilation_size)) - ).astype(np.uint8) + morphology_bins = scipy.ndimage.binary_dilation(detected_water, structure=np.ones((dilatation_size, dilatation_size))).astype(np.uint8) return water_mask, pcd_origin diff --git a/lidro/create_mask_hydro/vectors/convert_to_vector.py b/lidro/create_mask_hydro/vectors/convert_to_vector.py index 867e2f36..3b0657d5 100644 --- a/lidro/create_mask_hydro/vectors/convert_to_vector.py +++ b/lidro/create_mask_hydro/vectors/convert_to_vector.py @@ -10,9 +10,7 @@ from lidro.create_mask_hydro.rasters.create_mask_raster import detect_hydro_by_tile -def create_hydro_vector_mask( - filename: str, output: str, pixel_size: float, tile_size: int, classes: list, crs: str, dilatation_size: int -): +def create_hydro_vector_mask(filename: str, output: str, pixel_size: float, tile_size: int, classes: list, crs: str, dilatation_size: int): """Create a vector mask of hydro surfaces in a tile from the points classification of the input LAS/LAZ file, and save it as a GeoJSON file. diff --git a/lidro/main_create_mask.py b/lidro/main_create_mask.py index 3e01fb1c..84313aa8 100644 --- a/lidro/main_create_mask.py +++ b/lidro/main_create_mask.py @@ -46,7 +46,7 @@ def main(config: DictConfig): tile_size = config.io.tile_size crs = CRS.from_user_input(config.io.srid) classe = config.filter.keep_classes - dilation_size = config.raster.dilation_size + dilatation_size = config.raster.dilatation_size def main_on_one_tile(filename): """Lauch main.py on one tile @@ -58,7 +58,7 @@ def main_on_one_tile(filename): input_file = os.path.join(input_dir, filename) # path to the LAS file output_file = os.path.join(output_dir, f"MaskHydro_{tilename}.GeoJSON") # path to the Mask Hydro file logging.info(f"\nCreate Mask Hydro 1 for tile : {tilename}") - create_hydro_vector_mask(input_file, output_file, pixel_size, tile_size, classe, crs, dilation_size) + create_hydro_vector_mask(input_file, output_file, pixel_size, tile_size, classe, crs, dilatation_size) if initial_las_filename: # Lauch creating mask by one tile: @@ -69,6 +69,5 @@ def main_on_one_tile(filename): for file in os.listdir(input_dir): main_on_one_tile(file) - if __name__ == "__main__": main() diff --git a/main_create_mask.log b/main_create_mask.log new file mode 100644 index 00000000..516562ab --- /dev/null +++ b/main_create_mask.log @@ -0,0 +1,44 @@ +[2024-04-26 16:32:39,675][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:32:39,783][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:36:01,953][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:36:02,060][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:38:40,614][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:38:40,720][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:39:34,562][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:39:34,667][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:40:34,669][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:40:34,777][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:41:15,808][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:41:15,915][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:43:02,720][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:43:02,826][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:44:34,947][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:44:35,055][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:45:23,980][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:45:24,089][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:46:39,583][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:46:39,690][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 +[2024-04-26 16:47:36,360][root][INFO] - +Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST +[2024-04-26 16:47:36,469][root][INFO] - +Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 diff --git a/test/rasters/test_create_mask_raster.py b/test/rasters/test_create_mask_raster.py index 078dac2d..33865095 100644 --- a/test/rasters/test_create_mask_raster.py +++ b/test/rasters/test_create_mask_raster.py @@ -7,10 +7,7 @@ import rasterio from rasterio.transform import from_origin -from lidro.create_mask_hydro.rasters.create_mask_raster import ( - create_occupancy_map, - detect_hydro_by_tile, -) +from lidro.create_mask_hydro.rasters.create_mask_raster import create_occupancy_map, detect_hydro_by_tile TMP_PATH = Path("./tmp/create_mask_hydro/rasters/create_mask_raster") @@ -46,9 +43,7 @@ def test_create_occupancy_map_default(): @pytest.mark.returnfile def test_detect_hydro_by_tile_return_file(): output_tif = TMP_PATH / "Semis_2021_0830_6291_LA93_IGN69_size.tif" - array, origin = detect_hydro_by_tile( - LAS_FILE, tile_size, pixel_size, classes=[0, 1, 2, 3, 4, 5, 6, 17, 66], dilation_size=1 - ) + array, origin = detect_hydro_by_tile(LAS_FILE, tile_size, pixel_size, classes=[0, 1, 2, 3, 4, 5, 6, 17, 66], dilatation_size=1) assert isinstance(array, np.ndarray) is True assert list(array.shape) == [tile_size / pixel_size] * 2 diff --git a/test/vectors/test_convert_to_vector.py b/test/vectors/test_convert_to_vector.py index a196b834..dbff5cfe 100644 --- a/test/vectors/test_convert_to_vector.py +++ b/test/vectors/test_convert_to_vector.py @@ -14,6 +14,7 @@ output = "./tmp/create_mask_hydro/vectors/convert_to_vector/MaskHydro_Semis_2021_0830_6291_LA93_IGN69.GeoJSON" + def setup_module(module): if TMP_PATH.is_dir(): shutil.rmtree(TMP_PATH) @@ -31,11 +32,29 @@ def test_create_hydro_vector_mask_default(): create_hydro_vector_mask(las_file, output, pixel_size, tile_size, classes, crs, dilatation_size) assert Path(output).exists() - gdf = gpd.read_file(output) - - assert not gdf.empty # GeoDataFrame shouldn't empty - assert gdf.crs.to_string() == crs # CRS is identical - assert all(isinstance(geom, Polygon) for geom in gdf.geometry) # All geometry should Polygons +def test_check_structure_default(): + # Output + with open(output, "r") as f: + geojson_data = json.load(f) + + with open(output_main, "r") as f: + geojson_data_main = json.load(f) + + # CHECK STRUCTURE + assert "type" in geojson_data + assert geojson_data["type"] == "FeatureCollection" + assert "features" in geojson_data + assert isinstance(geojson_data["features"], list) + + # CHECK POLYGON + for feature in geojson_data["features"]: + geometry = feature["geometry"] + coordinates = geometry["coordinates"] + + for feature in geojson_data_main["features"]: + geometry_main = feature["geometry"] + coordinates_main = geometry_main["coordinates"] + + assert coordinates[0] == coordinates_main[0] + - expected_number_of_geometries = 2820 - assert len(gdf) == expected_number_of_geometries # the number of geometries must be identical From 275b5d1aae41d5cabb71cb42fe2519ed2c54964f Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Mon, 29 Apr 2024 11:49:16 +0200 Subject: [PATCH 19/68] Fix readme layout --- README.md | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 431e7da9..8446a3ab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -### LIDRO +# LIDRO + Lidro (Aplanissement des surfaces d'eaux) est un outil permettant de créer automatiquement des points virtuels le long des surfaces d'eaux afin de créer des modèles numériques cohérents avec les modèles hydrologiques. Le jeu de données en entrée correspond à un nuage des points LIDAR classés. ## Contexte @@ -15,31 +16,36 @@ Les données en entrées : - données vectorielles représentant le réseau hydrographique issu des différentes bases de données IGN (BDUnis, BDTopo, etc.) Trois grands axes du processus à mettre en place en distanguant l'échelle de traitmeent associé : -1- Création de masques hydrographiques à l'échelle de la dalle LIDAR -2- Création de masques hydrographiques pré-filtrés à l'échelle du chantier, soit : -* la suppression de ces masques dans les zones ZICAD/ZIPVA -* la suppression des aires < 150 m² -* la suppression des aires < 1000 m² hors BD IGN (grands cours d'eaux > 5m de large) - -3- Création de points virtuels le long de deux entités hydrographiques : -* ##### Grands cours d'eaux (> 5 m de large dans la BD Unis). -Il existe plusieurs étapes intermédiaires : -1- création automatique du tronçon hydrographique ("Squelette hydrographique", soit les tronçons hydrographiques dans la BD Unid) à partir de l'emprise du masque hydrographique "écoulement" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle) -2- Analyse de la répartition en Z de l'ensemble des points LIDAR "Sol" -3- Créations de points virtuels nécéssitant plusieurs étapes intermédiaires : -* Associer chaque point virtuel 2D au point le plus proche du squelette hydrographique -* Traitement "Z" du squelette : +* 1- Création de masques hydrographiques à l'échelle de la dalle LIDAR +* 2- Création de masques hydrographiques pré-filtrés à l'échelle du chantier, soit : + * la suppression de ces masques dans les zones ZICAD/ZIPVA + * la suppression des aires < 150 m² + * la suppression des aires < 1000 m² hors BD IGN (grands cours d'eau < 5m de large) +* 3- Création de points virtuels le long de deux entités hydrographiques : + * Grands cours d'eau (> 5 m de large dans la BD Unis). + * Surfaces planes (mer, lac, étang, etc.) + +### Traitement des grands cours d'eau (> 5 m de large dans la BD Uns). + +Il existe plusieurs étapes intermédiaires : +* 1- création automatique du tronçon hydrographique ("Squelette hydrographique", soit les tronçons hydrographiques dans la BD Unid) à partir de l'emprise du masque hydrographique "écoulement" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle) +* 2- Analyse de la répartition en Z de l'ensemble des points LIDAR "Sol" +* 3- Création de points virtuels nécéssitant plusieurs étapes intermédiaires : + * Associer chaque point virtuel 2D au point le plus proche du squelette hydrographique + * Traitement "Z" du squelette : * Analyser la répartition en Z de l’ensemble des points LIDAR extrait à l’étape précédente afin d’extraire une seule valeur d’altitude au point fictif Z le plus proche du squelette hydrographique. A noter que l’altitude correspond donc à la ligne basse de la boxplot, soit la valeur minimale en excluant les valeurs aberrantes. Pour les ponts, une étape de contrôle de la classification pourrait être mise en place * Lisser les Z le long du squelette HYDRO pour assurer l'écoulement -* Création des points virtuels 2D tous les 0.5 mètres le long des bords du masque hydrographique "écoulement" en cohérence en "Z" avec le squelette hydrographique + * Création des points virtuels 2D tous les 0.5 mètres le long des bords du masque hydrographique "écoulement" en cohérence en "Z" avec le squelette hydrographique -* ##### Surfaces planes (mer, lac, étang, etc.) +### Traitement des surfaces planes (mer, lac, étang, etc.) Pour rappel, l'eau est considérée comme horizontale sur ce type de surface. -Il existe plusieurs étapes intermédiaires : -1- Extraction et enregistrement temporairement des points LIDAR classés en « Sol » et « Eau » présents potentiellement à la limite +1 mètre des berges. Pour cela, on s'appuie sur 'emprise du masque hydrographique "surface plane" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle). a noter que pur le secteur maritime (Mer), il faut exclure la classe « 9 » (eau) afin d’éviter de mesurer les vagues. -2- Analyse statisque de l'ensemble des points LIDAR "Sol / Eau" le long des côtes/berges afin d'obtenir une surface plane. L’objectif est de créer des points virtuels spécifiques avec une information d'altitude (m) tous les 0.5 m sur les bords des surfaces maritimes et des plans d’eaux à partir du masque hydrographique "surface plane". Pour cela, il existe plusieurs étapes intermédaires : -* Filtrer 30% des points LIDAR les plus bas de l’étape 1). afin de supprimer les altitudes trop élevées -* Analyser la répartition en Z de ces points afin d’extraire une seule valeur d’altitude selon l’objet hydrographique : + +Il existe plusieurs étapes intermédiaires : +* 1- Extraction et enregistrement temporairement des points LIDAR classés en « Sol » et « Eau » présents potentiellement à la limite +1 mètre des berges. Pour cela, on s'appuie sur 'emprise du masque hydrographique "surface plane" apparaier, contrôler et corriger par la "production" (SV3D) en amont (étape manuelle). a noter que pur le secteur maritime (Mer), il faut exclure la classe « 9 » (eau) afin d’éviter de mesurer les vagues. +* 2- Analyse statistique de l'ensemble des points LIDAR "Sol / Eau" le long des côtes/berges afin d'obtenir une surface plane. + L’objectif est de créer des points virtuels spécifiques avec une information d'altitude (m) tous les 0.5 m sur les bords des surfaces maritimes et des plans d’eau à partir du masque hydrographique "surface plane". Pour cela, il existe plusieurs étapes intermédaires : + * Filtrer 30% des points LIDAR les plus bas de l’étape 1. afin de supprimer les altitudes trop élevées + * Analyser la répartition en Z de ces points afin d’extraire une seule valeur d’altitude selon l’objet hydrographique : * Pour les Plans d’eau : l’altitude correspond à la ligne basse de la boxplot, soit la valeur minimale en excluant les valeurs aberrantes, * Pour la Mer : l’altitude correspond à la ligne basse de la boxplot, soit la valeur minimale en excluant les valeurs aberrantes From 8208f0d3782f5cc1604d1be853af857265c7258a Mon Sep 17 00:00:00 2001 From: mdupaysign <162304206+mdupaysign@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:36:29 +0200 Subject: [PATCH 20/68] Update lidro/create_mask_hydro/rasters/create_mask_raster.py Co-authored-by: leavauchier <120112647+leavauchier@users.noreply.github.com> --- lidro/create_mask_hydro/rasters/create_mask_raster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lidro/create_mask_hydro/rasters/create_mask_raster.py b/lidro/create_mask_hydro/rasters/create_mask_raster.py index d7746510..57ed7dec 100644 --- a/lidro/create_mask_hydro/rasters/create_mask_raster.py +++ b/lidro/create_mask_hydro/rasters/create_mask_raster.py @@ -77,6 +77,6 @@ def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, class # Apply a mathematical morphology operations: DILATATION # / ! \ NOT "CLOSING", due to the reduction in the size of hydro masks, particularly at the tile borders. # / ! \ WITH "CLOSING" => Masks Hydro are no longer continuous, when they are merged - morphology_bins = scipy.ndimage.binary_dilation(detected_water, structure=np.ones((dilatation_size, dilatation_size))).astype(np.uint8) + water_mask = scipy.ndimage.binary_dilation(detected_water, structure=np.ones((dilatation_size, dilatation_size))).astype(np.uint8) return water_mask, pcd_origin From 8ec7df39a4ac27592f423d0eaab80a7345e9b1e6 Mon Sep 17 00:00:00 2001 From: mdupaysign <162304206+mdupaysign@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:37:09 +0200 Subject: [PATCH 21/68] Update lidro/create_mask_hydro/rasters/create_mask_raster.py Co-authored-by: leavauchier <120112647+leavauchier@users.noreply.github.com> --- lidro/create_mask_hydro/rasters/create_mask_raster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lidro/create_mask_hydro/rasters/create_mask_raster.py b/lidro/create_mask_hydro/rasters/create_mask_raster.py index 57ed7dec..aaf40458 100644 --- a/lidro/create_mask_hydro/rasters/create_mask_raster.py +++ b/lidro/create_mask_hydro/rasters/create_mask_raster.py @@ -33,7 +33,7 @@ def create_occupancy_map(points: np.array, tile_size: int, pixel_size: float, or return bins -def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, classes: List[int], dilatation_size: int): +def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, classes: List[int], dilation_size: int): """ "Detect hydrographic surfaces in a tile from the classified points of the input pointcloud An hydrographic surface is defined as a surface where there is no points from any class different from water The output hydrographic surface mask is dilated to make sure that the masks are continuous when merged with their From 61729445517e09797041b7ef69d44929ac017c6f Mon Sep 17 00:00:00 2001 From: mdupaysign <162304206+mdupaysign@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:54:20 +0200 Subject: [PATCH 22/68] Update create_mask_raster.py Modify parameters "dilatation_size" to "dilation_size" in this function --- lidro/create_mask_hydro/rasters/create_mask_raster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lidro/create_mask_hydro/rasters/create_mask_raster.py b/lidro/create_mask_hydro/rasters/create_mask_raster.py index aaf40458..c5235ba4 100644 --- a/lidro/create_mask_hydro/rasters/create_mask_raster.py +++ b/lidro/create_mask_hydro/rasters/create_mask_raster.py @@ -52,7 +52,7 @@ def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, class pixel_size (float): distance between each node of the raster grid (in meters) classes (List[int]): List of classes to use for the binarisation (points with other classification values are ignored) - dilatation_size (int): size for dilatation raster + dilation_size (int): size for dilatation raster Returns: smoothed_water (np.array): 2D binary array (x, y) of the water presence from the point cloud @@ -74,9 +74,9 @@ def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, class # Revert occupancy map to keep pixels where there is no point of the selected classes detected_water = np.logical_not(occupancy) - # Apply a mathematical morphology operations: DILATATION - # / ! \ NOT "CLOSING", due to the reduction in the size of hydro masks, particularly at the tile borders. + # Apply a mathematical morphology operations: DILATION + # / ! \ NOT "CLOSING", due to the reduction in the size of hydro masks (tile borders) # / ! \ WITH "CLOSING" => Masks Hydro are no longer continuous, when they are merged - water_mask = scipy.ndimage.binary_dilation(detected_water, structure=np.ones((dilatation_size, dilatation_size))).astype(np.uint8) + water_mask = scipy.ndimage.binary_dilation(detected_water, structure=np.ones((dilation_size, dilation_size))).astype(np.uint8) return water_mask, pcd_origin From 126798a521f0267ba397cbc21306ff9e71e4e9e0 Mon Sep 17 00:00:00 2001 From: mdupaysign <162304206+mdupaysign@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:44:33 +0200 Subject: [PATCH 23/68] Update lidro/create_mask_hydro/rasters/create_mask_raster.py Co-authored-by: leavauchier <120112647+leavauchier@users.noreply.github.com> --- .../create_mask_hydro/rasters/create_mask_raster.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/lidro/create_mask_hydro/rasters/create_mask_raster.py b/lidro/create_mask_hydro/rasters/create_mask_raster.py index c5235ba4..1bf269f6 100644 --- a/lidro/create_mask_hydro/rasters/create_mask_raster.py +++ b/lidro/create_mask_hydro/rasters/create_mask_raster.py @@ -35,17 +35,8 @@ def create_occupancy_map(points: np.array, tile_size: int, pixel_size: float, or def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, classes: List[int], dilation_size: int): """ "Detect hydrographic surfaces in a tile from the classified points of the input pointcloud - An hydrographic surface is defined as a surface where there is no points from any class different from water - The output hydrographic surface mask is dilated to make sure that the masks are continuous when merged with their - neighbours - Args: - filename (str): input pointcloud - tile_size (int): size of the raster grid (in meters) - pixel_size (float): distance between each node of the raster grid (in meters) - classes (List[int]): List of classes to use for the binarisation (points with other - classification values are ignored) - dilation_size (int): size of the structuring element for dilation - + An hydrographic surface is define as a surface where there is no points from any class different from water +The output hydrographic surface mask is dilated to make sure that the masks are continuous when merged with their neighbours Args: filename (str): input pointcloud tile_size (int): size of the raster grid (in meters) From f2f67cf6a76985dc59b8933d91a3fc3eeac728cb Mon Sep 17 00:00:00 2001 From: mdupays Date: Mon, 6 May 2024 14:15:53 +0200 Subject: [PATCH 24/68] update main_merge_mask.py --- .env | 2 + .vscode/launch.json | 35 ++++ .../2024-07-03/10-22-51/.hydra/config.yaml | 20 +++ outputs/2024-07-03/10-22-51/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/10-22-51/.hydra/overrides.yaml | 6 + outputs/2024-07-03/10-22-51/main_skeleton.log | 0 .../2024-07-03/10-36-27/.hydra/config.yaml | 20 +++ outputs/2024-07-03/10-36-27/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/10-36-27/.hydra/overrides.yaml | 6 + outputs/2024-07-03/10-36-27/main_skeleton.log | 0 .../2024-07-03/11-48-08/.hydra/config.yaml | 20 +++ outputs/2024-07-03/11-48-08/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/11-48-08/.hydra/overrides.yaml | 6 + outputs/2024-07-03/11-48-08/main_skeleton.log | 0 .../2024-07-03/11-55-24/.hydra/config.yaml | 20 +++ outputs/2024-07-03/11-55-24/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/11-55-24/.hydra/overrides.yaml | 6 + outputs/2024-07-03/11-55-24/main_skeleton.log | 0 .../2024-07-03/12-10-55/.hydra/config.yaml | 20 +++ outputs/2024-07-03/12-10-55/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/12-10-55/.hydra/overrides.yaml | 6 + outputs/2024-07-03/12-10-55/main_skeleton.log | 0 .../2024-07-03/12-12-11/.hydra/config.yaml | 20 +++ outputs/2024-07-03/12-12-11/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/12-12-11/.hydra/overrides.yaml | 6 + outputs/2024-07-03/12-12-11/main_skeleton.log | 0 .../2024-07-03/14-03-39/.hydra/config.yaml | 20 +++ outputs/2024-07-03/14-03-39/.hydra/hydra.yaml | 163 ++++++++++++++++++ .../2024-07-03/14-03-39/.hydra/overrides.yaml | 6 + outputs/2024-07-03/14-03-39/main_skeleton.log | 0 30 files changed, 1360 insertions(+) create mode 100644 .env create mode 100644 .vscode/launch.json create mode 100644 outputs/2024-07-03/10-22-51/.hydra/config.yaml create mode 100644 outputs/2024-07-03/10-22-51/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/10-22-51/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/10-22-51/main_skeleton.log create mode 100644 outputs/2024-07-03/10-36-27/.hydra/config.yaml create mode 100644 outputs/2024-07-03/10-36-27/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/10-36-27/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/10-36-27/main_skeleton.log create mode 100644 outputs/2024-07-03/11-48-08/.hydra/config.yaml create mode 100644 outputs/2024-07-03/11-48-08/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/11-48-08/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/11-48-08/main_skeleton.log create mode 100644 outputs/2024-07-03/11-55-24/.hydra/config.yaml create mode 100644 outputs/2024-07-03/11-55-24/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/11-55-24/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/11-55-24/main_skeleton.log create mode 100644 outputs/2024-07-03/12-10-55/.hydra/config.yaml create mode 100644 outputs/2024-07-03/12-10-55/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/12-10-55/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/12-10-55/main_skeleton.log create mode 100644 outputs/2024-07-03/12-12-11/.hydra/config.yaml create mode 100644 outputs/2024-07-03/12-12-11/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/12-12-11/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/12-12-11/main_skeleton.log create mode 100644 outputs/2024-07-03/14-03-39/.hydra/config.yaml create mode 100644 outputs/2024-07-03/14-03-39/.hydra/hydra.yaml create mode 100644 outputs/2024-07-03/14-03-39/.hydra/overrides.yaml create mode 100644 outputs/2024-07-03/14-03-39/main_skeleton.log diff --git a/.env b/.env new file mode 100644 index 00000000..58e69023 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +DB_UNI_USER='invite' +DB_UNI_PASSWORD='28de#' \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..76973a6b --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,35 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Skeleton", + "type": "debugpy", + "request": "launch", + "program": "lidro/skeleton/main_skeleton.py", + "console": "integratedTerminal", + "args": [ + "SKELETON.FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson", + "SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson", + "SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson", + "SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson", + "SKELETON.DB_UNI.DB_USER=invite", + "SKELETON.DB_UNI.DB_PASSWORD='28de#'", + ] + }, + { + "name": "unit testing", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "justMyCode": false, + "args": [ + "-s", + "-k", + "test_main_skeleton_2", + ], + }, + ] +} \ No newline at end of file diff --git a/outputs/2024-07-03/10-22-51/.hydra/config.yaml b/outputs/2024-07-03/10-22-51/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/10-22-51/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/10-22-51/.hydra/hydra.yaml b/outputs/2024-07-03/10-22-51/.hydra/hydra.yaml new file mode 100644 index 00000000..3b479709 --- /dev/null +++ b/outputs/2024-07-03/10-22-51/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/10-22-51 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/10-22-51/.hydra/overrides.yaml b/outputs/2024-07-03/10-22-51/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/10-22-51/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/10-22-51/main_skeleton.log b/outputs/2024-07-03/10-22-51/main_skeleton.log new file mode 100644 index 00000000..e69de29b diff --git a/outputs/2024-07-03/10-36-27/.hydra/config.yaml b/outputs/2024-07-03/10-36-27/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/10-36-27/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/10-36-27/.hydra/hydra.yaml b/outputs/2024-07-03/10-36-27/.hydra/hydra.yaml new file mode 100644 index 00000000..2d8d6f93 --- /dev/null +++ b/outputs/2024-07-03/10-36-27/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/10-36-27 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/10-36-27/.hydra/overrides.yaml b/outputs/2024-07-03/10-36-27/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/10-36-27/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/10-36-27/main_skeleton.log b/outputs/2024-07-03/10-36-27/main_skeleton.log new file mode 100644 index 00000000..e69de29b diff --git a/outputs/2024-07-03/11-48-08/.hydra/config.yaml b/outputs/2024-07-03/11-48-08/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/11-48-08/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/11-48-08/.hydra/hydra.yaml b/outputs/2024-07-03/11-48-08/.hydra/hydra.yaml new file mode 100644 index 00000000..177cd2db --- /dev/null +++ b/outputs/2024-07-03/11-48-08/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/11-48-08 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/11-48-08/.hydra/overrides.yaml b/outputs/2024-07-03/11-48-08/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/11-48-08/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/11-48-08/main_skeleton.log b/outputs/2024-07-03/11-48-08/main_skeleton.log new file mode 100644 index 00000000..e69de29b diff --git a/outputs/2024-07-03/11-55-24/.hydra/config.yaml b/outputs/2024-07-03/11-55-24/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/11-55-24/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/11-55-24/.hydra/hydra.yaml b/outputs/2024-07-03/11-55-24/.hydra/hydra.yaml new file mode 100644 index 00000000..849089ba --- /dev/null +++ b/outputs/2024-07-03/11-55-24/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/11-55-24 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/11-55-24/.hydra/overrides.yaml b/outputs/2024-07-03/11-55-24/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/11-55-24/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/11-55-24/main_skeleton.log b/outputs/2024-07-03/11-55-24/main_skeleton.log new file mode 100644 index 00000000..e69de29b diff --git a/outputs/2024-07-03/12-10-55/.hydra/config.yaml b/outputs/2024-07-03/12-10-55/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/12-10-55/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/12-10-55/.hydra/hydra.yaml b/outputs/2024-07-03/12-10-55/.hydra/hydra.yaml new file mode 100644 index 00000000..126a896d --- /dev/null +++ b/outputs/2024-07-03/12-10-55/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/12-10-55 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/12-10-55/.hydra/overrides.yaml b/outputs/2024-07-03/12-10-55/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/12-10-55/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/12-10-55/main_skeleton.log b/outputs/2024-07-03/12-10-55/main_skeleton.log new file mode 100644 index 00000000..e69de29b diff --git a/outputs/2024-07-03/12-12-11/.hydra/config.yaml b/outputs/2024-07-03/12-12-11/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/12-12-11/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/12-12-11/.hydra/hydra.yaml b/outputs/2024-07-03/12-12-11/.hydra/hydra.yaml new file mode 100644 index 00000000..ec0c212a --- /dev/null +++ b/outputs/2024-07-03/12-12-11/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/12-12-11 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/12-12-11/.hydra/overrides.yaml b/outputs/2024-07-03/12-12-11/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/12-12-11/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/12-12-11/main_skeleton.log b/outputs/2024-07-03/12-12-11/main_skeleton.log new file mode 100644 index 00000000..e69de29b diff --git a/outputs/2024-07-03/14-03-39/.hydra/config.yaml b/outputs/2024-07-03/14-03-39/.hydra/config.yaml new file mode 100644 index 00000000..89be70a7 --- /dev/null +++ b/outputs/2024-07-03/14-03-39/.hydra/config.yaml @@ -0,0 +1,20 @@ +work_dir: ${hydra:runtime.cwd} +FILE_PATH: + MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson + GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson + GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson +MAX_GAP_WIDTH: 200 +MAX_BRIDGES: 1 +GAP_WIDTH_CHECK_DB: 30 +RATIO_GAP: 0.25 +DB_UNI: + DB_NAME: bduni_france_consultation + DB_HOST: bduni_consult.ign.fr + DB_USER: invite + DB_PASSWORD: 28de# + DB_PORT: 5432 +BRANCH: + VORONOI_MAX_LENGTH: 2 + WATER_MIN_SIZE: 20 + MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/14-03-39/.hydra/hydra.yaml b/outputs/2024-07-03/14-03-39/.hydra/hydra.yaml new file mode 100644 index 00000000..adc5eccd --- /dev/null +++ b/outputs/2024-07-03/14-03-39/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson + - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson + - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson + - DB_UNI.DB_USER=invite + - DB_UNI.DB_PASSWORD='28de#' + job: + name: main_skeleton + chdir: null + override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson + id: ??? + num: ??? + config_name: configs_skeleton.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.2' + cwd: /home/MDaab/code/lidro + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/MDaab/code/lidro/configs + schema: file + provider: main + - path: hydra_plugins.hydra_colorlog.conf + schema: pkg + provider: hydra-colorlog + - path: '' + schema: structured + provider: schema + output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/14-03-39 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/outputs/2024-07-03/14-03-39/.hydra/overrides.yaml b/outputs/2024-07-03/14-03-39/.hydra/overrides.yaml new file mode 100644 index 00000000..907f19b5 --- /dev/null +++ b/outputs/2024-07-03/14-03-39/.hydra/overrides.yaml @@ -0,0 +1,6 @@ +- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson +- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson +- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson +- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson +- DB_UNI.DB_USER=invite +- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/14-03-39/main_skeleton.log b/outputs/2024-07-03/14-03-39/main_skeleton.log new file mode 100644 index 00000000..e69de29b From 690461af86f3fc5917e76e48ed452d30bfbf3ee3 Mon Sep 17 00:00:00 2001 From: mdupays Date: Tue, 14 May 2024 14:55:52 +0200 Subject: [PATCH 25/68] refacto with Lea --- configs/configs_lidro.yaml | 12 +++++++++++- lidro/merge_mask_hydro/vectors/close_holes.py | 2 +- test/vectors/test_check_rectify_geometry.py | 10 +++++++++- test/vectors/test_close_holes.py | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 81261d13..00fe7672 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -20,7 +20,17 @@ io: raster: # size for dilatation dilatation_size: 3 - + + +vector: + # Filter water's area (m²) + min_water_area: 150 + # Parameters for buffer + buffer_positive: 1 + buffer_negative: -1.5 # negative buffer should be bigger than positive buffer to prevent protruding over the banks + # Tolerance from Douglas-Peucker + tolerance: 1 + filter: # Classes to be considered as "non-water" keep_classes: [0, 1, 2, 3, 4, 5, 6, 17, 64, 65, 66, 67] # All classes diff --git a/lidro/merge_mask_hydro/vectors/close_holes.py b/lidro/merge_mask_hydro/vectors/close_holes.py index 7b88a910..4f4ad0e4 100644 --- a/lidro/merge_mask_hydro/vectors/close_holes.py +++ b/lidro/merge_mask_hydro/vectors/close_holes.py @@ -4,7 +4,7 @@ from shapely.ops import unary_union -def close_holes(polygon: Polygon, min_hole_area)-> Polygon: +def close_holes(polygon: Polygon, min_hole_area): """Remove small holes (surface < 100 m²) Args: diff --git a/test/vectors/test_check_rectify_geometry.py b/test/vectors/test_check_rectify_geometry.py index 01719ffa..ef5c7bf2 100644 --- a/test/vectors/test_check_rectify_geometry.py +++ b/test/vectors/test_check_rectify_geometry.py @@ -16,7 +16,6 @@ def test_apply_buffers_to_geometry_default(): def test_fix_topology_default(): - input = "./data/merge_mask_hydro/MaskHydro_merge.geojson" # Load each GeoJSON file as GeoDataFrame geojson = gpd.read_file(input) check_geom = fix_topology(geojson) @@ -25,6 +24,7 @@ def test_fix_topology_default(): assert check_geom.geometry.dtype == "geometry" +<<<<<<< HEAD # Check not duplicates in the data assert not check_geom.duplicated().any(), "There are duplicates in the data" @@ -46,3 +46,11 @@ def test_fix_topology_error(): # # Check geometry assert check_geom.geometry.is_valid.all(), "Geometry no-valid" +======= + # duplicates in the data + assert not geojson.duplicated().any(), "There are duplicates in the data" + + # Check geometry + assert geojson["geometry"].is_valid.all(), "Geometry no-valid" + +>>>>>>> ebdf035 (refacto with Lea) diff --git a/test/vectors/test_close_holes.py b/test/vectors/test_close_holes.py index 1376cc04..b77ebb4d 100644 --- a/test/vectors/test_close_holes.py +++ b/test/vectors/test_close_holes.py @@ -9,7 +9,7 @@ def test_close_holes_default(): # Load each GeoJSON file as GeoDataFrame geojson = gpd.read_file(input) - geometry = geojson["geometry"][1] # with contain severals holes + geometry = geojson["geometry"][0] mask_without_hole = close_holes(geometry, 100) assert isinstance(mask_without_hole, Polygon) From 8e09393c2b2d25f0f1e74d4167b788566f7bd41d Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Thu, 25 Jul 2024 14:57:03 +0200 Subject: [PATCH 26/68] manual correction for rebase --- configs/configs_lidro.yaml | 44 +++++++++++++-------- test/vectors/test_check_rectify_geometry.py | 12 +----- test/vectors/test_convert_to_vector.py | 33 ++++------------ 3 files changed, 37 insertions(+), 52 deletions(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 00fe7672..8cb947ca 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -8,6 +8,8 @@ work_dir: ${hydra:runtime.cwd} io: input_filename: null + input_mask_hydro: null + input_skeleton: null input_dir: null output_dir: null srid: 2154 @@ -17,23 +19,33 @@ io: no_data_value: -9999 tile_size: 1000 -raster: - # size for dilatation - dilatation_size: 3 +mask_generation: + raster: + # size for dilatation + dilation_size: 3 + filter: + # Classes to be considered as "non-water" + keep_classes: [0, 1, 2, 3, 4, 5, 6, 17, 64, 65, 66, 67] # All classes + vector: + # Filter water's area (m²) + min_water_area: 150 + # Parameters for buffer + buffer_positive: 1 + buffer_negative: -1.5 # negative buffer should be bigger than positive buffer to prevent protruding over the banks + # Tolerance from Douglas-Peucker + tolerance: 1 - -vector: - # Filter water's area (m²) - min_water_area: 150 - # Parameters for buffer - buffer_positive: 1 - buffer_negative: -1.5 # negative buffer should be bigger than positive buffer to prevent protruding over the banks - # Tolerance from Douglas-Peucker - tolerance: 1 - -filter: - # Classes to be considered as "non-water" - keep_classes: [0, 1, 2, 3, 4, 5, 6, 17, 64, 65, 66, 67] # All classes +virtual_point: + filter: + # Keep ground and water pointclouds between Hydro Mask and Hydro Mask buffer + keep_neighbors_classes: [2, 9] + vector: + # distance in meters between 2 consecutive points + distance_meters: 2 + # buffer for searching the points classification (default. "2") of the input LAS/LAZ file + buffer: 2 + # The number of nearest neighbors to find with KNeighbors + k: 20 SKELETON: FILE_PATH: diff --git a/test/vectors/test_check_rectify_geometry.py b/test/vectors/test_check_rectify_geometry.py index ef5c7bf2..bbba49d3 100644 --- a/test/vectors/test_check_rectify_geometry.py +++ b/test/vectors/test_check_rectify_geometry.py @@ -16,6 +16,7 @@ def test_apply_buffers_to_geometry_default(): def test_fix_topology_default(): + input = "./data/merge_mask_hydro/MaskHydro_merge.geojson" # Load each GeoJSON file as GeoDataFrame geojson = gpd.read_file(input) check_geom = fix_topology(geojson) @@ -24,7 +25,6 @@ def test_fix_topology_default(): assert check_geom.geometry.dtype == "geometry" -<<<<<<< HEAD # Check not duplicates in the data assert not check_geom.duplicated().any(), "There are duplicates in the data" @@ -45,12 +45,4 @@ def test_fix_topology_error(): assert not check_geom.duplicated().any(), "There are duplicates in the data" # # Check geometry - assert check_geom.geometry.is_valid.all(), "Geometry no-valid" -======= - # duplicates in the data - assert not geojson.duplicated().any(), "There are duplicates in the data" - - # Check geometry - assert geojson["geometry"].is_valid.all(), "Geometry no-valid" - ->>>>>>> ebdf035 (refacto with Lea) + assert check_geom.geometry.is_valid.all(), "Geometry no-valid" \ No newline at end of file diff --git a/test/vectors/test_convert_to_vector.py b/test/vectors/test_convert_to_vector.py index dbff5cfe..3de41bd4 100644 --- a/test/vectors/test_convert_to_vector.py +++ b/test/vectors/test_convert_to_vector.py @@ -14,7 +14,6 @@ output = "./tmp/create_mask_hydro/vectors/convert_to_vector/MaskHydro_Semis_2021_0830_6291_LA93_IGN69.GeoJSON" - def setup_module(module): if TMP_PATH.is_dir(): shutil.rmtree(TMP_PATH) @@ -32,29 +31,11 @@ def test_create_hydro_vector_mask_default(): create_hydro_vector_mask(las_file, output, pixel_size, tile_size, classes, crs, dilatation_size) assert Path(output).exists() -def test_check_structure_default(): - # Output - with open(output, "r") as f: - geojson_data = json.load(f) - - with open(output_main, "r") as f: - geojson_data_main = json.load(f) - - # CHECK STRUCTURE - assert "type" in geojson_data - assert geojson_data["type"] == "FeatureCollection" - assert "features" in geojson_data - assert isinstance(geojson_data["features"], list) - - # CHECK POLYGON - for feature in geojson_data["features"]: - geometry = feature["geometry"] - coordinates = geometry["coordinates"] - - for feature in geojson_data_main["features"]: - geometry_main = feature["geometry"] - coordinates_main = geometry_main["coordinates"] - - assert coordinates[0] == coordinates_main[0] - + gdf = gpd.read_file(output) + + assert not gdf.empty # GeoDataFrame shouldn't empty + assert gdf.crs.to_string() == crs # CRS is identical + assert all(isinstance(geom, Polygon) for geom in gdf.geometry) # All geometry should Polygons + expected_number_of_geometries = 2820 + assert len(gdf) == expected_number_of_geometries # the number of geometries must be identical \ No newline at end of file From 6c9ad27a80c7916de4fd2289eff71a4e2beed17c Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 26 Jul 2024 10:06:03 +0200 Subject: [PATCH 27/68] move main_skeleton into lidro --- lidro/main_create_skeleton_lines.py | 59 +++++++++++++++++++ ...n_skeleton.py => create_skeleton_lines.py} | 4 +- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 lidro/main_create_skeleton_lines.py rename lidro/skeleton/{main_skeleton.py => create_skeleton_lines.py} (99%) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py new file mode 100644 index 00000000..811b4921 --- /dev/null +++ b/lidro/main_create_skeleton_lines.py @@ -0,0 +1,59 @@ +""" Main script for creating skeleton lines inside masks +""" +import hydra +from omegaconf import DictConfig + +import geopandas as gpd +import pandas as pd +from shapely.geometry import Point + +from skeleton.create_skeleton_lines import create_branches_list, create_branches_pair, select_candidates +from skeleton.branch import Branch + + +@hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_lidro.yaml") +def run(config: DictConfig): + """ + Get whole hydrographic skeleton lines, based on + a geojson file containing multiple river contours. + The result file will contain "skeleton" lines running in the middle of + those contours to describe the flow of the rivers, and lines crossing the contours + to join with the skeleton lines, as bridge and other elements may "break" river contours + args: + - config (DictConfig): the config dict from hydra + """ + + gdf_hydro_global_mask = gpd.read_file(config.SKELETON.FILE_PATH.MASK_INPUT_PATH) + crs = gdf_hydro_global_mask.crs # Load a crs from input + + branches_list = create_branches_list(config, gdf_hydro_global_mask, crs) + branches_pair_list = create_branches_pair(config, branches_list) + validated_candidates = select_candidates(config, branches_pair_list) + + # create the gap lines from the selected candidates + gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] + if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: + gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) + gdf_gap_lines.to_file(config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') + + # add the extremities used on each branch to close a gap to the list of gap_point of that branch, + # to create a new line toward that extremity and know not to remove it during the "simplify" + for candidate in validated_candidates: + candidate.branch_1.gap_points.append(Point(candidate.extremity_1)) + candidate.branch_2.gap_points.append(Point(candidate.extremity_2)) + + # get skeletons for all branches + for branch in branches_list: + branch: Branch + branch.create_skeleton() + branch.simplify() + + # putting all skeleton lines together, and save them if there is a path + branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] + gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) + if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: + gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') + + +if __name__ == "__main__": + run() diff --git a/lidro/skeleton/main_skeleton.py b/lidro/skeleton/create_skeleton_lines.py similarity index 99% rename from lidro/skeleton/main_skeleton.py rename to lidro/skeleton/create_skeleton_lines.py index 92d024ac..30d0464f 100644 --- a/lidro/skeleton/main_skeleton.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -11,8 +11,8 @@ import psycopg2 from shapely.geometry import Point -from branch import Branch, Candidate -from group_maker import GroupMaker +from skeleton.branch import Branch, Candidate +from skeleton.group_maker import GroupMaker def db_connector(config: DictConfig): From 7a523c0bbd290a0c7bdffd0c4d45611e7f7d3e9d Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 26 Jul 2024 10:25:23 +0200 Subject: [PATCH 28/68] merge skeleton lines at the end --- lidro/main_create_skeleton_lines.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 811b4921..91c815dc 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -8,7 +8,7 @@ from shapely.geometry import Point from skeleton.create_skeleton_lines import create_branches_list, create_branches_pair, select_candidates -from skeleton.branch import Branch +from skeleton.branch import Branch, line_merge @hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_lidro.yaml") @@ -54,6 +54,11 @@ def run(config: DictConfig): if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') + # saving all lines + branch_lines_list.append(gdf_gap_lines) + gdf_global_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) + gdf_global_lines = line_merge(gdf_global_lines, crs) # merge lines into polylines + gdf_global_lines.to_file(config.SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') if __name__ == "__main__": run() From e578b359e9294eadfde713541df2908adaaaba58 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 26 Jul 2024 10:54:59 +0200 Subject: [PATCH 29/68] fix final merge lines --- lidro/main_create_skeleton_lines.py | 5 +-- lidro/skeleton/create_skeleton_lines.py | 53 ------------------------- 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 91c815dc..3b96c6ee 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -32,8 +32,8 @@ def run(config: DictConfig): # create the gap lines from the selected candidates gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] + gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: - gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) gdf_gap_lines.to_file(config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, @@ -55,8 +55,7 @@ def run(config: DictConfig): gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') # saving all lines - branch_lines_list.append(gdf_gap_lines) - gdf_global_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) + gdf_global_lines = gpd.GeoDataFrame(pd.concat([gdf_branch_lines, gdf_gap_lines], ignore_index=True)) gdf_global_lines = line_merge(gdf_global_lines, crs) # merge lines into polylines gdf_global_lines.to_file(config.SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index 30d0464f..11152657 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -177,56 +177,3 @@ def create_branches_pair(config: DictConfig, branches_list: List[Branch]) -> Lis if distance < config.SKELETON.MAX_GAP_WIDTH: branches_pair_list.append((branch_a, branch_b, distance)) return branches_pair_list - - -@hydra.main(version_base="1.2", config_path="../../configs/", config_name="configs_lidro.yaml") -def run(config: DictConfig): - """ - Get whole hydrographic skeleton lines, based on - a geojson file containing multiple river contours. - The result file will contain "skeleton" lines running in the middle of - those contours to describe the flow of the rivers, and lines crossing the contours - to join with the skeleton lines, as bridge and other elements may "break" river contours - args: - - config (DictConfig): the config dict from hydra - """ - - gdf_hydro_global_mask = gpd.read_file(config.SKELETON.FILE_PATH.MASK_INPUT_PATH) - crs = gdf_hydro_global_mask.crs # Load a crs from input - - branches_list = create_branches_list(config, gdf_hydro_global_mask, crs) - branches_pair_list = create_branches_pair(config, branches_list) - validated_candidates = select_candidates(config, branches_pair_list) - - # create the gap lines from the selected candidates - gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] - if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: - gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) - gdf_gap_lines.to_file(config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') - - # add the extremities used on each branch to close a gap to the list of gap_point of that branch, - # to create a new line toward that extremity and know not to remove it during the "simplify" - for candidate in validated_candidates: - candidate.branch_1.gap_points.append(Point(candidate.extremity_1)) - candidate.branch_2.gap_points.append(Point(candidate.extremity_2)) - - # get skeletons for all branches - for branch in branches_list: - branch: Branch - branch.create_skeleton() - branch.simplify() - - # putting all skeleton lines together, and save them if there is a path - branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] - gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: - gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') - - # saving all lines - branch_lines_list.append(gdf_gap_lines) - gdf_global_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - gdf_global_lines.to_file(config.SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') - - -if __name__ == "__main__": - run() From 162992edc83ad512cf48b1200c4dd85a6acafd81 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 26 Jul 2024 14:31:05 +0200 Subject: [PATCH 30/68] adding some tests --- .vscode/launch.json | 4 +-- lidro/skeleton/create_skeleton_lines.py | 12 ++++----- test/skeleton/test_branch.py | 33 +++++++++++++++++++++++-- test/skeleton/test_main_skeleton.py | 4 +-- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 76973a6b..96a952ad 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "Python Debugger: Skeleton", "type": "debugpy", "request": "launch", - "program": "lidro/skeleton/main_skeleton.py", + "program": "lidro/main_create_skeleton_lines.py", "console": "integratedTerminal", "args": [ "SKELETON.FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson", @@ -28,7 +28,7 @@ "args": [ "-s", "-k", - "test_main_skeleton_2", + "test_get_df_points_from_gdf", ], }, ] diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index 11152657..18baf22a 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -1,18 +1,16 @@ from typing import List, Tuple +import sys + +sys.path.append('../lidro') -import hydra from omegaconf import DictConfig from pyproj.crs.crs import CRS -import geopandas as gpd -import pandas as pd -from pandas import DataFrame as df from geopandas.geodataframe import GeoDataFrame import psycopg2 -from shapely.geometry import Point -from skeleton.branch import Branch, Candidate -from skeleton.group_maker import GroupMaker +from lidro.skeleton.branch import Branch, Candidate +from lidro.skeleton.group_maker import GroupMaker def db_connector(config: DictConfig): diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index 5e33a72f..88621f70 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -3,8 +3,9 @@ from hydra import compose, initialize import geopandas as gpd from omegaconf import DictConfig +from shapely import LineString -from lidro.skeleton.branch import Branch, get_vertices_dict +from lidro.skeleton.branch import Branch, get_vertices_dict, get_df_points_from_gdf, line_merge sys.path.append('lidro/skeleton') @@ -18,7 +19,34 @@ def read_branch(config: DictConfig, branch_path: str, branch_name: str) -> Branc return Branch(config, branch_name, mask_branch, crs) -def test_branch_1(): +def test_get_df_points_from_gdf(): + """test of 'get_df_points_from_gdf'""" + gdf_branch_mask = gpd.read_file(BRANCH_TEST_1_PATH) + df_all_coords = get_df_points_from_gdf(gdf_branch_mask) + assert len(df_all_coords) == 235 + + +# def test_line_merge(): +# line_1 = LineString([0, 0], [0, 1], [0, 2]) +# line_2 = LineString([0, 2], [0, 3], [0, 4]) +# line_3 = LineString([0, 4], [0, 5]) +# line_4 = LineString([0, 5], [0, 6]) +# line_5 = LineString([0, 4], [1, 4]) +# line_6 = LineString([1, 4], [2, 4]) +# line_list = [line_1, +# line_2, +# line_3, +# line_4, +# line_5, +# line_6 +# ] + +# # gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] +# # gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) +# pass + + +def test_creation_skeleton_lines(): """test creation/simplification of skeleton lines for a branch""" with initialize(version_base="1.2", config_path="../../configs"): config = compose( @@ -38,3 +66,4 @@ def test_branch_1(): extremities_cpt += 1 assert extremities_cpt == 3 # check that this branch's skeleton has exactly 3 extremities + diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index 68da856a..974d8101 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -5,8 +5,8 @@ import geopandas as gpd from dotenv import load_dotenv -from lidro.skeleton.main_skeleton import create_branches_list, create_branches_pair -from lidro.skeleton.main_skeleton import select_candidates, query_db_for_bridge_across_gap +from lidro.skeleton.create_skeleton_lines import create_branches_list, create_branches_pair +from lidro.skeleton.create_skeleton_lines import select_candidates, query_db_for_bridge_across_gap from lidro.skeleton.branch import Candidate from test.skeleton.test_branch import read_branch From 973e3beab46c1f79467cb1de9cd351d444831a7d Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 26 Jul 2024 14:33:46 +0200 Subject: [PATCH 31/68] remove launch.json --- .vscode/launch.json | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 96a952ad..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python Debugger: Skeleton", - "type": "debugpy", - "request": "launch", - "program": "lidro/main_create_skeleton_lines.py", - "console": "integratedTerminal", - "args": [ - "SKELETON.FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson", - "SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson", - "SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson", - "SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson", - "SKELETON.DB_UNI.DB_USER=invite", - "SKELETON.DB_UNI.DB_PASSWORD='28de#'", - ] - }, - { - "name": "unit testing", - "type": "debugpy", - "request": "launch", - "module": "pytest", - "justMyCode": false, - "args": [ - "-s", - "-k", - "test_get_df_points_from_gdf", - ], - }, - ] -} \ No newline at end of file From 400e84dd7d32254ec52318aab973e9022cca2b71 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 26 Jul 2024 15:28:34 +0200 Subject: [PATCH 32/68] better path (...maybe ???) --- lidro/main_create_skeleton_lines.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 3b96c6ee..1cf523c3 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -1,5 +1,7 @@ """ Main script for creating skeleton lines inside masks """ +import sys + import hydra from omegaconf import DictConfig @@ -7,8 +9,11 @@ import pandas as pd from shapely.geometry import Point -from skeleton.create_skeleton_lines import create_branches_list, create_branches_pair, select_candidates -from skeleton.branch import Branch, line_merge +sys.path.append('../lidro') + +from lidro.skeleton.create_skeleton_lines import create_branches_list, create_branches_pair, select_candidates +from lidro.skeleton.branch import Branch, line_merge + @hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_lidro.yaml") From e9bc6a0ff60fd8629f546294c3f5e53a4f953ab8 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 29 Jul 2024 08:43:52 +0200 Subject: [PATCH 33/68] specify the WATER_MIN_SIZE parameter --- configs/configs_lidro.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 8cb947ca..8f94c5cb 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -70,7 +70,7 @@ SKELETON: BRANCH: VORONOI_MAX_LENGTH: 2 # max size of a voronoi line - WATER_MIN_SIZE: 20 # min size of a skeleton line to be sure not to be removed + WATER_MIN_SIZE: 200 # min size of a skeleton line to be sure not to be removed MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches hydra: From f6f8aa10866b83c14e60678069ea403a69335ec5 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 29 Jul 2024 08:52:11 +0200 Subject: [PATCH 34/68] more explanation for WATER_MIN_SIZE parameter --- configs/configs_lidro.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 8f94c5cb..1a35b022 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -70,7 +70,7 @@ SKELETON: BRANCH: VORONOI_MAX_LENGTH: 2 # max size of a voronoi line - WATER_MIN_SIZE: 200 # min size of a skeleton line to be sure not to be removed + WATER_MIN_SIZE: 200 # min size of a skeleton line to be sure not to be removed (should be at least more than half the max river width) MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches hydra: From 6a50fd55a39dc154b027e65455d412169866c5bf Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 29 Jul 2024 13:23:27 +0200 Subject: [PATCH 35/68] To create skeleton lines, we only care about the exterior polygon --- configs/configs_lidro.yaml | 4 ++-- lidro/skeleton/branch.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 1a35b022..f020c57a 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -56,7 +56,7 @@ SKELETON: MAX_GAP_WIDTH: 200 # distance max in meter of any gap between 2 branches we will try to close with a line MAX_BRIDGES: 1 # number max of bridges that can be created between 2 branches - GAP_WIDTH_CHECK_DB: 30 # for a gap at least this wide, we check with DB_UNI. Smaller than that, we assume a line automatically exists + GAP_WIDTH_CHECK_DB: 20 # for a gap at least this wide, we check with DB_UNI if there is a bridge. Smaller than that, we assume a line automatically exists RATIO_GAP: 0.25 # when checking if a candidate line to close a gap intersects a bridge in BD UNI, that's the ratio ]0.1] of the line to consider. # we only check part of the line, because when the line is really long it can intersect with bridges not corresponding to that line, # so we make the line smaller @@ -70,7 +70,7 @@ SKELETON: BRANCH: VORONOI_MAX_LENGTH: 2 # max size of a voronoi line - WATER_MIN_SIZE: 200 # min size of a skeleton line to be sure not to be removed (should be at least more than half the max river width) + WATER_MIN_SIZE: 500 # min size of a skeleton line to be sure not to be removed (should be at least more than half the max river width) MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches hydra: diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index be3a4e36..a4b2ca78 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -69,7 +69,8 @@ def set_gdf_branch_mask(self, branch_mask: GeoDataFrame): gdf_valid.geometry = gdf_valid.geometry.apply( lambda geom: fix_invalid_geometry(geom) ) - self.gdf_branch_mask = gdf_valid + # # we keep only the exterior ring (that should be unique) as our polygon, to simplify the result + self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[Polygon(gdf_valid.exterior[0].coords)], crs=self.crs) def create_skeleton(self): """ @@ -264,7 +265,7 @@ def create_voronoi_lines(self) -> GeoDataFrame: # remove Voronoi lines exterior to the mask geometry = gpd.GeoSeries(regions.geoms, crs=self.crs).explode(index_parts=False) df = gpd.GeoDataFrame(geometry=geometry, crs=self.crs) - lines_filter = df.sjoin(self.gdf_branch_mask, predicate='within') # Opération spatiale "Within" + lines_filter = df.sjoin(self.gdf_branch_mask, predicate='within') # only keeps lines "Within" gdf_branch_mask # save Voronoi lines lines_filter = lines_filter.reset_index(drop=True) # Réinitialiser l'index lines_filter = lines_filter.drop(columns=['index_right']) # Supprimer la colonne 'index_right' From 6cc70f65eb8d7aac17468d4aff97359eceb06317 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 29 Jul 2024 15:13:27 +0200 Subject: [PATCH 36/68] fix to tests to match the changes of limiting skeleton_lines to the exterior ring only --- lidro/main_create_skeleton_lines.py | 8 +-- lidro/skeleton/branch.py | 10 ++-- lidro/skeleton/create_skeleton_lines.py | 4 +- test/skeleton/test_branch.py | 70 ++++++++++++++++++------- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 1cf523c3..ccbfac36 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -9,11 +9,10 @@ import pandas as pd from shapely.geometry import Point -sys.path.append('../lidro') - from lidro.skeleton.create_skeleton_lines import create_branches_list, create_branches_pair, select_candidates from lidro.skeleton.branch import Branch, line_merge +sys.path.append('../lidro') @hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_lidro.yaml") @@ -21,7 +20,7 @@ def run(config: DictConfig): """ Get whole hydrographic skeleton lines, based on a geojson file containing multiple river contours. - The result file will contain "skeleton" lines running in the middle of + The result file will contain "skeleton" lines running in the middle of those contours to describe the flow of the rivers, and lines crossing the contours to join with the skeleton lines, as bridge and other elements may "break" river contours args: @@ -61,8 +60,9 @@ def run(config: DictConfig): # saving all lines gdf_global_lines = gpd.GeoDataFrame(pd.concat([gdf_branch_lines, gdf_gap_lines], ignore_index=True)) - gdf_global_lines = line_merge(gdf_global_lines, crs) # merge lines into polylines + gdf_global_lines = line_merge(gdf_global_lines, crs) # merge lines into polylines gdf_global_lines.to_file(config.SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') + if __name__ == "__main__": run() diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index a4b2ca78..a9a5ce34 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -137,7 +137,8 @@ def get_candidates(self, other_branch: 'Branch') -> List[Candidate]: if index >= self.config.SKELETON.BRANCH.MAX_GAP_CANDIDATES: break # stop if the following candidates - if distance_squared[other_index][self_index] > self.config.SKELETON.MAX_GAP_WIDTH * self.config.SKELETON.MAX_GAP_WIDTH: + if distance_squared[other_index][self_index] \ + > self.config.SKELETON.MAX_GAP_WIDTH * self.config.SKELETON.MAX_GAP_WIDTH: break candidates.append( @@ -234,7 +235,8 @@ def can_line_be_removed(self, line: Geometry, vertices_dict: Dict[Point, List[Li Returns true if a line can be removed Args: - line (Geometry) : the line to test - - vertices_dict (Dict[Point, List[LineString]]) : a dictionary off all the lines with a point as an extremity + - vertices_dict (Dict[Point, List[LineString]]) : a dictionary off all the + lines with a point as an extremity """ if line.length > self.config.SKELETON.BRANCH.WATER_MIN_SIZE: return False @@ -275,7 +277,7 @@ def create_voronoi_lines(self) -> GeoDataFrame: def get_df_points_from_gdf(gdf: GeoDataFrame) -> pd.DataFrame: """ - Return a 2-columns dataframe (col x, y), containing the coords of + Return a 2-columns dataframe (col x, y), containing the coords of all the points in a geodataframe -Args : - gdf (GeoDataFrame) : the geodataframe we want the points' coordinates from @@ -301,7 +303,7 @@ def get_df_points_from_gdf(gdf: GeoDataFrame) -> pd.DataFrame: def fix_invalid_geometry(geometry): - """ + """ return the geometry, fixed Args: - gdf_lines:geodataframe containing a list of lines diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index 18baf22a..c530563d 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -1,8 +1,6 @@ from typing import List, Tuple import sys -sys.path.append('../lidro') - from omegaconf import DictConfig from pyproj.crs.crs import CRS @@ -12,6 +10,8 @@ from lidro.skeleton.branch import Branch, Candidate from lidro.skeleton.group_maker import GroupMaker +sys.path.append('../lidro') + def db_connector(config: DictConfig): """ diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index 88621f70..4049993b 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -10,6 +10,8 @@ sys.path.append('lidro/skeleton') BRANCH_TEST_1_PATH = "test_files/90.geojson" +CRS_FOR_TEST = 2145 +WATER_MIN_SIZE_TEST = 20 def read_branch(config: DictConfig, branch_path: str, branch_name: str) -> Branch: @@ -26,24 +28,51 @@ def test_get_df_points_from_gdf(): assert len(df_all_coords) == 235 -# def test_line_merge(): -# line_1 = LineString([0, 0], [0, 1], [0, 2]) -# line_2 = LineString([0, 2], [0, 3], [0, 4]) -# line_3 = LineString([0, 4], [0, 5]) -# line_4 = LineString([0, 5], [0, 6]) -# line_5 = LineString([0, 4], [1, 4]) -# line_6 = LineString([1, 4], [2, 4]) -# line_list = [line_1, -# line_2, -# line_3, -# line_4, -# line_5, -# line_6 -# ] - -# # gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] -# # gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) -# pass +def test_line_merge(): + """test of line_merge""" + point_0_0 = [0, 0] + point_0_1 = [0, 1] + point_0_2 = [0, 2] + point_0_3 = [0, 3] + point_0_4 = [0, 4] + point_0_5 = [0, 5] + point_0_6 = [0, 6] + point_1_4 = [1, 4] + point_2_4 = [2, 4] + + line_1 = LineString([point_0_0, point_0_1, point_0_2]) + line_2 = LineString([point_0_2, point_0_3, point_0_4]) + line_3 = LineString([point_0_4, point_0_5]) + line_4 = LineString([point_0_5, point_0_6]) + line_5 = LineString([point_0_4, point_1_4]) + line_6 = LineString([point_1_4, point_2_4]) + + line_list = [line_1, + line_2, + line_3, + line_4, + line_5, + line_6 + ] + + gdf_gap_lines = gpd.GeoDataFrame(geometry=line_list).set_crs(CRS_FOR_TEST, allow_override=True) + gdf_merged = line_merge(gdf_gap_lines, CRS_FOR_TEST) + assert len(gdf_merged) == 3 + for line in gdf_merged['geometry']: + if point_0_0 in line.coords: + assert len(line.coords) == 5 + assert point_0_1 in line.coords + assert point_0_2 in line.coords + assert point_0_3 in line.coords + assert point_0_4 in line.coords + if point_0_6 in line.coords: + assert len(line.coords) == 3 + assert point_0_4 in line.coords + assert point_0_5 in line.coords + if point_2_4 in line.coords: + assert len(line.coords) == 3 + assert point_1_4 in line.coords + assert point_0_4 in line.coords def test_creation_skeleton_lines(): @@ -51,6 +80,10 @@ def test_creation_skeleton_lines(): with initialize(version_base="1.2", config_path="../../configs"): config = compose( config_name="configs_lidro.yaml", + overrides=[ + f"SKELETON.BRANCH.WATER_MIN_SIZE={WATER_MIN_SIZE_TEST}", + ] + ) branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") @@ -66,4 +99,3 @@ def test_creation_skeleton_lines(): extremities_cpt += 1 assert extremities_cpt == 3 # check that this branch's skeleton has exactly 3 extremities - From 4154e7d1461dab7964304c41c0f9245ae756e4b7 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 30 Jul 2024 15:16:32 +0200 Subject: [PATCH 37/68] small fix in case there is no bridge --- lidro/main_create_skeleton_lines.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index ccbfac36..5f0dcd1a 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -9,11 +9,11 @@ import pandas as pd from shapely.geometry import Point +sys.path.append('../lidro') + from lidro.skeleton.create_skeleton_lines import create_branches_list, create_branches_pair, select_candidates from lidro.skeleton.branch import Branch, line_merge -sys.path.append('../lidro') - @hydra.main(version_base="1.2", config_path="../configs/", config_name="configs_lidro.yaml") def run(config: DictConfig): @@ -37,7 +37,7 @@ def run(config: DictConfig): # create the gap lines from the selected candidates gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) - if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: + if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH and validated_candidates: gdf_gap_lines.to_file(config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, From 8cf1c15de82c44513c88482a2a9db4809d282768 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Thu, 1 Aug 2024 09:55:57 +0200 Subject: [PATCH 38/68] fix a rare case --- lidro/skeleton/branch.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index a9a5ce34..5226e024 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -78,6 +78,10 @@ def create_skeleton(self): """ voronoi_lines = self.create_voronoi_lines() + if len(voronoi_lines) == 0: + self.gdf_skeleton_lines = gpd.GeoDataFrame(geometry=[], crs=self.crs) + return + # draw a new line for each point added to closes gaps to the nearest points on voronoi_lines np_points = get_df_points_from_gdf(voronoi_lines).to_numpy().transpose() for gap_point in self.gap_points: @@ -85,6 +89,7 @@ def create_skeleton(self): min_index = np.unravel_index(np.argmin(distance_squared, axis=None), distance_squared.shape)[0] line_to_close_the_gap = LineString([gap_point, Point(np_points[0][min_index], np_points[1][min_index])]) voronoi_lines.loc[len(voronoi_lines)] = line_to_close_the_gap + self.gdf_skeleton_lines = line_merge(voronoi_lines, self.crs) def simplify(self): From fc9925fb7d9a8ce104280485a8508f3b02659a10 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Thu, 1 Aug 2024 14:44:45 +0200 Subject: [PATCH 39/68] change request to database to be sligthly faster --- environment.yml | 2 +- lidro/skeleton/create_skeleton_lines.py | 42 ++++++++++++++----------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/environment.yml b/environment.yml index 0899ee3d..49c68131 100644 --- a/environment.yml +++ b/environment.yml @@ -16,7 +16,7 @@ dependencies: - pdal>=2.6 - python-pdal>=3.2.1 - shapely>=2.0.3 - - psycopg2-binary + - psycopg[binary] - python-dotenv # --------- hydra configs --------- # - hydra-core diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index c530563d..a61226d2 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -5,7 +5,7 @@ from pyproj.crs.crs import CRS from geopandas.geodataframe import GeoDataFrame -import psycopg2 +import psycopg from lidro.skeleton.branch import Branch, Candidate from lidro.skeleton.group_maker import GroupMaker @@ -19,12 +19,20 @@ def db_connector(config: DictConfig): args: - config (DictConfig): the config dict from hydra """ - return psycopg2.connect( - database=config.SKELETON.DB_UNI.DB_NAME, - host=config.SKELETON.DB_UNI.DB_HOST, - user=config.SKELETON.DB_UNI.DB_USER, - password=config.SKELETON.DB_UNI.DB_PASSWORD, - port=config.SKELETON.DB_UNI.DB_PORT + # return psycopg.connect( + # database=config.SKELETON.DB_UNI.DB_NAME, + # host=config.SKELETON.DB_UNI.DB_HOST, + # user=config.SKELETON.DB_UNI.DB_USER, + # password=config.SKELETON.DB_UNI.DB_PASSWORD, + # port=config.SKELETON.DB_UNI.DB_PORT + # ) + + return psycopg.connect( + f"dbname={config.SKELETON.DB_UNI.DB_NAME} \ + host={config.SKELETON.DB_UNI.DB_HOST} \ + user={config.SKELETON.DB_UNI.DB_USER} \ + password={config.SKELETON.DB_UNI.DB_PASSWORD} \ + port={config.SKELETON.DB_UNI.DB_PORT}" ) @@ -63,17 +71,15 @@ def query_db_for_bridge_across_gap(config: DictConfig, candidate: Candidate) -> # execution of queries with db_connector(config) as db_conn: with db_conn.cursor() as db_cursor: - db_cursor.execute(query_linear) - results = db_cursor.fetchall() - # if no result with linear bridge, maybe with area bridge... - if not results: - db_cursor.execute(query_area) - results = db_cursor.fetchall() - - if results: - pass - - return True if results else False + whole_query = query_linear + query_area + db_cursor.execute(whole_query) + + result_linear = db_cursor.fetchone() + db_cursor.nextset() + result_area = db_cursor.fetchone() + if result_linear or result_area: + return True + return False def select_candidates( From 74e5627f692a8b8f34d0d84512a8ecba7584257f Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Thu, 1 Aug 2024 15:39:16 +0200 Subject: [PATCH 40/68] add a test for the db query, only played locally, not by github action --- test/skeleton/test_main_skeleton.py | 37 ++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index 974d8101..9f0dbc29 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -4,6 +4,7 @@ import pandas as pd import geopandas as gpd from dotenv import load_dotenv +import pytest from lidro.skeleton.create_skeleton_lines import create_branches_list, create_branches_pair from lidro.skeleton.create_skeleton_lines import select_candidates, query_db_for_bridge_across_gap @@ -12,6 +13,8 @@ load_dotenv() +IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" # check if tests runs under github_action + DB_UNI_USER = os.getenv('DB_UNI_USER') DB_UNI_PASSWORD = os.getenv('DB_UNI_PASSWORD') CRS = 2154 @@ -45,20 +48,22 @@ def test_main_skeleton_1(): assert (candidate_2.squared_distance < 75) -# def test_main_skeleton_2(): -# """Test : query_db_for_bridge_across_gap """ -# with initialize(version_base="1.2", config_path="../../configs"): -# config = compose( -# config_name="configs_lidro.yaml", -# overrides=[ -# f"SKELETON.DB_UNI.DB_USER={DB_UNI_USER}", -# f'SKELETON.DB_UNI.DB_PASSWORD="{DB_UNI_PASSWORD}"', -# ], -# ) -# dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) -# dummy_candidate_2 = Candidate(None, None, (689272.5, 6760595.5), (689322.5, 6760553.5), 0) +# do that test only if we are not on github action, since github can't connect to BD UNI +@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="BD UNI not reachable from github action") +def test_main_skeleton_2(): + """Test : query_db_for_bridge_across_gap """ + with initialize(version_base="1.2", config_path="../../configs"): + config = compose( + config_name="configs_lidro.yaml", + overrides=[ + f"SKELETON.DB_UNI.DB_USER={DB_UNI_USER}", + f'SKELETON.DB_UNI.DB_PASSWORD="{DB_UNI_PASSWORD}"', + ], + ) + dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) + dummy_candidate_2 = Candidate(None, None, (689272.5, 6760595.5), (689322.5, 6760553.5), 0) -# is_bridge_1 = query_db_for_bridge_across_gap(config, dummy_candidate_1) -# is_bridge_2 = query_db_for_bridge_across_gap(config, dummy_candidate_2) -# assert not is_bridge_1 -# assert is_bridge_2 + is_bridge_1 = query_db_for_bridge_across_gap(config, dummy_candidate_1) + is_bridge_2 = query_db_for_bridge_across_gap(config, dummy_candidate_2) + assert not is_bridge_1 + assert is_bridge_2 From a2b2ca839e7e855522c343d3ffbaeaf22cf25f41 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 2 Aug 2024 09:12:47 +0200 Subject: [PATCH 41/68] adding an example for create skeleton lines --- scripts/example_create_skeleton_lines.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 scripts/example_create_skeleton_lines.sh diff --git a/scripts/example_create_skeleton_lines.sh b/scripts/example_create_skeleton_lines.sh new file mode 100644 index 00000000..cb9d375e --- /dev/null +++ b/scripts/example_create_skeleton_lines.sh @@ -0,0 +1,16 @@ +# For lauching create skeleton lines +python lidro/main_create_skeleton_lines.py \ +SKELETON.FILE_PATH.MASK_INPUT_PATH=[input_filepath] \ +SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=[output_filepath] \ +SKELETON.DB_UNI.DB_USER=[user_name] \ +"SKELETON.DB_UNI.DB_PASSWORD=[password]" \ +SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH=[out_filepath(optional)] \ +SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH=[out_filepath(optional)] + +# SKELETON.FILE_PATH.MASK_INPUT_PATH : input path to the .geojson with the water masks +# SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH : output path for all the skeletons +# SKELETON.DB_UNI.DB_USER : username for "bduni_france_consultation" +# SKELETON.DB_UNI.DB_PASSWORD : password for "bduni_france_consultation". WARNING ! If there is a special character in the password, +# the line must be written like this : "SKELETON.DB_UNI.DB_PASSWORD='$tr@ng€_ch@r@ct€r$'" (note the " and the ') +# SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH : (optional) output path for only the skeleton inside water beds defined by the input masks +# SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file From 5677001e88a28e58b30233a10ff8eba8f2dadea3 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 2 Aug 2024 11:41:51 +0200 Subject: [PATCH 42/68] removed a forgotten comment --- lidro/skeleton/create_skeleton_lines.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index a61226d2..909e4397 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -19,14 +19,6 @@ def db_connector(config: DictConfig): args: - config (DictConfig): the config dict from hydra """ - # return psycopg.connect( - # database=config.SKELETON.DB_UNI.DB_NAME, - # host=config.SKELETON.DB_UNI.DB_HOST, - # user=config.SKELETON.DB_UNI.DB_USER, - # password=config.SKELETON.DB_UNI.DB_PASSWORD, - # port=config.SKELETON.DB_UNI.DB_PORT - # ) - return psycopg.connect( f"dbname={config.SKELETON.DB_UNI.DB_NAME} \ host={config.SKELETON.DB_UNI.DB_HOST} \ From 5d7230f5e6282d40e37fd1287c034c2f5d87aaa3 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 5 Aug 2024 15:09:23 +0200 Subject: [PATCH 43/68] changes according to Palvina's code review --- README.md | 25 +++++++++++++++++++++++++ lidro/skeleton/group_maker.py | 4 ++-- test/skeleton/test_branch.py | 2 +- test/skeleton/test_main_skeleton.py | 6 +++--- test_files/40.geojson | 7 ------- test_files/43.geojson | 7 ------- test_files/44.geojson | 7 ------- test_files/90.geojson | 7 ------- 8 files changed, 31 insertions(+), 34 deletions(-) delete mode 100644 test_files/40.geojson delete mode 100644 test_files/43.geojson delete mode 100644 test_files/44.geojson delete mode 100644 test_files/90.geojson diff --git a/README.md b/README.md index 8446a3ab..995b971a 100644 --- a/README.md +++ b/README.md @@ -108,3 +108,28 @@ Pour lancer les tests : ``` python -m pytest -s ``` +### paramètres pour créer les squelettes des cours d'eau +Pour fonctionner, la création de squelettes a besoin d'une série de paramètres, certains ayant une valeur par défaut, d'autres non. Les paramètres se trouvent dans le fichier configs/configs_lidro.yaml. On peut soit les y modifier, soit les modifer en ligne de commande lors de l'exécution du script avec : +``` +python lidro/main_create_skeleton_lines.py [nom_paramètre_1]=[valeur_du_paramètre_1] [nom_paramètre_2]=[valeur_du_paramètre_2] +``` +ces paramètres sont : +SKELETON.FILE_PATH.MASK_INPUT_PATH : Le chemin d'entrée des masques des cours d'eau +SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) +SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) +SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH : Le chemin de sortie des lignes et des squelettes ensemble + +SKELETON.MAX_GAP_WIDTH : La distance maximale envisagée pour franchir des ponts +SKELETON.MAX_BRIDGES : Le nombre maximal de ponts entre deux bras séparés de cours d'eau +SKELETON.GAP_WIDTH_CHECK_DB : La distance à partir de laquelle on vérifie via la base de données s'il y a bien un pont +SKELETON.RATIO_GAP : la proportion de la ligne franchissant un pont qui est comparé en base pour voir s'il y a bien un pont (trop grande et on pourrait trouver un pont qui ne correspond pas) + +SKELETON.DB_UNI.DB_NAME : Le nom de la base de données +SKELETON.DB_UNI.DB_HOST : l'adresse de la base de données +SKELETON.DB_UNI.DB_USER : L'utilisateur de la base de données +SKELETON.DB_UNI.DB_PASSWORD : Le mot de passe de l'utilisateur +SKELETON.DB_UNI.DB_PORT : La port de connexion avec la base de données + +SKELETON.BRANCH.VORONOI_MAX_LENGTH : LA longuer maximum des lignes individuelles des squelettes +SKELETON.BRANCH.WATER_MIN_SIZE : La longueur minimal à partir de laquelle une ligne de squelette sera automatiquement gardée (trop petite, et il y aura des sortes "d'aiguilles" qui apparaitront. Trop grande, et certains afluents ne seront pas détectés) +SKELETON.BRANCH.MAX_GAP_CANDIDATES : Le nombre maximum de candidats pour envisager de franchir des ponts entre deux bras \ No newline at end of file diff --git a/lidro/skeleton/group_maker.py b/lidro/skeleton/group_maker.py index 80260a0b..fa459d25 100644 --- a/lidro/skeleton/group_maker.py +++ b/lidro/skeleton/group_maker.py @@ -1,8 +1,8 @@ class GroupMaker: """ From a list of elements, GroupMaker is used to - group sets of elements according to single element from - the sets. For example: + put together groups of elements, according to single + element from each group. For example: At the beginning, we have {A}, {B}, {C}, {D}, {E} put_together(A, B) -> {A, B}, {C}, {D}, {E} put_together(C, D) -> {A, B}, {C, D}, {E} diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index 4049993b..da58f2fd 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -9,7 +9,7 @@ sys.path.append('lidro/skeleton') -BRANCH_TEST_1_PATH = "test_files/90.geojson" +BRANCH_TEST_1_PATH = "data/skeleton_hydro/test_files/90.geojson" CRS_FOR_TEST = 2145 WATER_MIN_SIZE_TEST = 20 diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index 9f0dbc29..9be7508b 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -18,9 +18,9 @@ DB_UNI_USER = os.getenv('DB_UNI_USER') DB_UNI_PASSWORD = os.getenv('DB_UNI_PASSWORD') CRS = 2154 -MAIN_SKELETON_TEST_1_1_PATH = "test_files/40.geojson" -MAIN_SKELETON_TEST_1_2_PATH = "test_files/43.geojson" -MAIN_SKELETON_TEST_1_3_PATH = "test_files/44.geojson" +MAIN_SKELETON_TEST_1_1_PATH = "data/skeleton_hydro/test_files/40.geojson" +MAIN_SKELETON_TEST_1_2_PATH = "data/skeleton_hydro/test_files/43.geojson" +MAIN_SKELETON_TEST_1_3_PATH = "data/skeleton_hydro/test_files/44.geojson" def test_main_skeleton_1(): diff --git a/test_files/40.geojson b/test_files/40.geojson deleted file mode 100644 index 7cccd82c..00000000 --- a/test_files/40.geojson +++ /dev/null @@ -1,7 +0,0 @@ -{ -"type": "FeatureCollection", -"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, -"features": [ -{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 699065.413934385520406, 6759448.5 ], [ 699063.5, 6759461.5 ], [ 699066.262332465150394, 6759463.18974278960377 ], [ 699081.333572532981634, 6759487.313280043192208 ], [ 699081.5, 6759490.586065614596009 ], [ 699083.472945803543553, 6759492.716397003270686 ], [ 699083.5, 6759501.5 ], [ 699085.471177920582704, 6759502.707364517264068 ], [ 699084.472945803543553, 6759521.283602996729314 ], [ 699078.5, 6759537.5 ], [ 699080.5, 6759537.5 ], [ 699082.716397003736347, 6759533.527054196223617 ], [ 699086.435427015880123, 6759533.564589496701956 ], [ 699087.5, 6759535.5 ], [ 699089.5, 6759535.5 ], [ 699089.5, 6759533.5 ], [ 699087.56458949635271, 6759532.435427015647292 ], [ 699087.926403762074187, 6759529.952435625717044 ], [ 699093.5, 6759531.5 ], [ 699092.506778866751119, 6759526.857554757036269 ], [ 699095.5, 6759508.586179815232754 ], [ 699092.516235235147178, 6759498.220095711760223 ], [ 699093.5, 6759494.5 ], [ 699091.752795581589453, 6759493.833355349488556 ], [ 699088.607240879791789, 6759486.55697579216212 ], [ 699087.5, 6759480.413934385403991 ], [ 699082.614180701202713, 6759474.574025148525834 ], [ 699081.5, 6759467.5 ], [ 699078.820317181060091, 6759465.926470963284373 ], [ 699075.607240879791789, 6759460.55697579216212 ], [ 699075.5, 6759454.5 ], [ 699069.5, 6759449.5 ], [ 699065.413934385520406, 6759448.5 ] ] ] } } -] -} diff --git a/test_files/43.geojson b/test_files/43.geojson deleted file mode 100644 index 1308a6ef..00000000 --- a/test_files/43.geojson +++ /dev/null @@ -1,7 +0,0 @@ -{ -"type": "FeatureCollection", -"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, -"features": [ -{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 698272.483764764852822, 6759201.220095711760223 ], [ 698270.5, 6759205.5 ], [ 698280.686719955410808, 6759208.666427466087043 ], [ 698291.413820184767246, 6759215.5 ], [ 698297.283602996263653, 6759215.527054196223617 ], [ 698304.413820184767246, 6759218.5 ], [ 698319.283602996263653, 6759218.527054196223617 ], [ 698327.413820184767246, 6759222.5 ], [ 698332.574025148525834, 6759222.614180700853467 ], [ 698333.5, 6759224.5 ], [ 698338.574025148525834, 6759224.614180700853467 ], [ 698339.5, 6759226.5 ], [ 698348.413934385520406, 6759230.5 ], [ 698358.686719955410808, 6759231.666427466087043 ], [ 698374.247204418410547, 6759248.166644650511444 ], [ 698375.5, 6759255.586065614596009 ], [ 698381.159515680046752, 6759261.048410073854029 ], [ 698381.5, 6759263.5 ], [ 698384.926470962003805, 6759263.820317179895937 ], [ 698389.413934385520406, 6759269.5 ], [ 698394.926470962003805, 6759271.820317179895937 ], [ 698400.5, 6759277.5 ], [ 698407.413820184767246, 6759279.5 ], [ 698429.586179815232754, 6759279.5 ], [ 698446.586179815232754, 6759278.5 ], [ 698459.0, 6759275.5 ], [ 698498.142445242963731, 6759276.50677886698395 ], [ 698513.556975790997967, 6759280.607240878976882 ], [ 698527.413820184767246, 6759287.5 ], [ 698536.413820184767246, 6759289.5 ], [ 698551.073601511539891, 6759288.501806816086173 ], [ 698554.413820184767246, 6759290.5 ], [ 698562.142445242963731, 6759289.50677886698395 ], [ 698571.413820184767246, 6759292.5 ], [ 698581.283602996263653, 6759292.527054196223617 ], [ 698586.413934385520406, 6759295.5 ], [ 698592.707095105201006, 6759296.677118103019893 ], [ 698593.5, 6759298.5 ], [ 698599.833355349488556, 6759299.752795581705868 ], [ 698600.5, 6759304.5 ], [ 698609.262332465150394, 6759314.18974278960377 ], [ 698615.5, 6759326.5 ], [ 698618.951589926262386, 6759326.840484320186079 ], [ 698620.5, 6759330.586065614596009 ], [ 698632.413934385520406, 6759341.5 ], [ 698643.142445242963731, 6759341.50677886698395 ], [ 698657.413820184767246, 6759345.5 ], [ 698664.435427015880123, 6759345.564589496701956 ], [ 698665.5, 6759348.5 ], [ 698673.707095105201006, 6759350.677118103019893 ], [ 698675.413934385520406, 6759353.5 ], [ 698683.707095105201006, 6759354.677118103019893 ], [ 698685.5, 6759358.5 ], [ 698689.810257209115662, 6759359.737667533569038 ], [ 698691.5, 6759362.5 ], [ 698697.926470962003805, 6759365.820317179895937 ], [ 698701.247204418410547, 6759369.166644650511444 ], [ 698701.5, 6759371.5 ], [ 698706.060660171788186, 6759373.939339828677475 ], [ 698706.5, 6759376.586065614596009 ], [ 698717.086370622622781, 6759385.965689181350172 ], [ 698717.5, 6759388.5 ], [ 698722.247204418410547, 6759389.166644650511444 ], [ 698723.483764764852822, 6759392.220095711760223 ], [ 698721.262332465150394, 6759398.81025721039623 ], [ 698714.810257209115662, 6759405.262332466430962 ], [ 698711.5, 6759405.5 ], [ 698708.5, 6759409.5 ], [ 698710.5, 6759417.0 ], [ 698707.5, 6759435.586179815232754 ], [ 698714.392759120208211, 6759443.44302420783788 ], [ 698715.5, 6759450.586179815232754 ], [ 698718.472945803543553, 6759455.716397003270686 ], [ 698717.5, 6759462.5 ], [ 698732.686719955410808, 6759467.666427466087043 ], [ 698737.4393572693225, 6759473.577802591957152 ], [ 698737.247204418410547, 6759476.833355349488556 ], [ 698727.5, 6759488.5 ], [ 698726.5, 6759501.5 ], [ 698730.5, 6759500.586065614596009 ], [ 698728.655030296300538, 6759496.335879155434668 ], [ 698737.5, 6759484.586065614596009 ], [ 698738.840484319953248, 6759480.048410073854029 ], [ 698741.5, 6759479.5 ], [ 698741.5, 6759475.413820184767246 ], [ 698736.586065614479594, 6759464.5 ], [ 698733.586065614479594, 6759462.5 ], [ 698727.443024209002033, 6759462.392759121023118 ], [ 698723.820317181060091, 6759458.926470963284373 ], [ 698723.5, 6759455.5 ], [ 698721.528822079417296, 6759454.292635482735932 ], [ 698720.5, 6759443.413934385403991 ], [ 698717.5, 6759439.5 ], [ 698713.913629377377219, 6759438.034310818649828 ], [ 698711.607240879791789, 6759433.55697579216212 ], [ 698711.5606427306775, 6759426.577802591957152 ], [ 698715.5, 6759421.586179815232754 ], [ 698714.666427467018366, 6759411.313280043192208 ], [ 698720.073529037996195, 6759405.820317179895937 ], [ 698727.56603432178963, 6759401.566034321673214 ], [ 698731.313280044589192, 6759394.666427466087043 ], [ 698739.586179815232754, 6759395.5 ], [ 698743.5, 6759394.5 ], [ 698744.564572984119877, 6759392.564589496701956 ], [ 698749.5, 6759393.5 ], [ 698750.425974851474166, 6759391.614180700853467 ], [ 698756.5, 6759390.5 ], [ 698761.292904894798994, 6759385.677118103019893 ], [ 698768.586065614479594, 6759384.5 ], [ 698777.189742790884338, 6759377.737667533569038 ], [ 698779.586065614479594, 6759377.5 ], [ 698783.166644650511444, 6759371.752795581705868 ], [ 698789.5, 6759371.5 ], [ 698792.048410073737614, 6759366.840484320186079 ], [ 698795.586065614479594, 6759366.5 ], [ 698799.5, 6759362.586065614596009 ], [ 698799.752795581589453, 6759359.166644650511444 ], [ 698801.5, 6759358.5 ], [ 698801.5, 6759355.5 ], [ 698798.500451771891676, 6759354.036811839789152 ], [ 698800.752795581589453, 6759347.166644650511444 ], [ 698809.5, 6759338.586065614596009 ], [ 698810.048410073737614, 6759333.840484320186079 ], [ 698813.577802593121305, 6759331.56064273044467 ], [ 698820.5, 6759330.5 ], [ 698821.666427467018366, 6759325.313280043192208 ], [ 698824.5, 6759323.5 ], [ 698826.707364517031237, 6759318.52882207930088 ], [ 698834.707095105201006, 6759320.677118103019893 ], [ 698840.413934385520406, 6759325.5 ], [ 698852.833355349488556, 6759330.752795581705868 ], [ 698853.5, 6759332.5 ], [ 698863.686719955410808, 6759333.666427466087043 ], [ 698870.413934385520406, 6759339.5 ], [ 698876.686719955410808, 6759341.666427466087043 ], [ 698890.413934385520406, 6759350.5 ], [ 698893.707095105201006, 6759350.677118103019893 ], [ 698894.5, 6759352.5 ], [ 698902.686719955410808, 6759355.666427466087043 ], [ 698920.413934385520406, 6759366.5 ], [ 698926.556975790997967, 6759367.607240878976882 ], [ 698931.413934385520406, 6759371.5 ], [ 698937.686719955410808, 6759372.666427466087043 ], [ 698942.413934385520406, 6759376.5 ], [ 698970.556975790997967, 6759384.607240878976882 ], [ 698985.810257209115662, 6759391.737667533569038 ], [ 698995.159515680046752, 6759401.048410073854029 ], [ 698996.5, 6759406.5 ], [ 699000.159515680046752, 6759408.048410073854029 ], [ 699003.5, 6759414.5 ], [ 699007.247204418410547, 6759417.166644650511444 ], [ 699008.5, 6759421.586065614596009 ], [ 699014.086370622622781, 6759425.965689181350172 ], [ 699018.5, 6759432.586065614596009 ], [ 699021.413934385520406, 6759435.5 ], [ 699023.926470962003805, 6759435.820317179895937 ], [ 699026.5, 6759439.5 ], [ 699031.5, 6759440.5 ], [ 699031.5, 6759438.5 ], [ 699021.840484319953248, 6759431.951589926145971 ], [ 699020.544953120406717, 6759428.364470270462334 ], [ 699022.635529730119742, 6759425.544953119941056 ], [ 699028.5, 6759426.5 ], [ 699027.5, 6759422.5 ], [ 699018.507222909945995, 6759417.147025710903108 ], [ 699018.527054196456447, 6759407.716397003270686 ], [ 699022.048410073737614, 6759400.840484320186079 ], [ 699028.422197406878695, 6759399.56064273044467 ], [ 699032.413934385520406, 6759404.5 ], [ 699034.5, 6759404.5 ], [ 699034.5, 6759401.413934385403991 ], [ 699027.701729563763365, 6759396.751328073441982 ], [ 699027.5, 6759392.5 ], [ 699022.443024209002033, 6759392.392759121023118 ], [ 698983.586179815232754, 6759375.5 ], [ 698975.577802593121305, 6759374.43935726955533 ], [ 698954.443024209002033, 6759367.392759121023118 ], [ 698949.586065614479594, 6759363.5 ], [ 698942.425974851474166, 6759363.385819299146533 ], [ 698934.586065614479594, 6759356.5 ], [ 698930.425974851474166, 6759356.385819299146533 ], [ 698928.586065614479594, 6759353.5 ], [ 698921.425974851474166, 6759352.385819299146533 ], [ 698920.5, 6759350.5 ], [ 698915.292904894798994, 6759350.322881896980107 ], [ 698909.5, 6759344.5 ], [ 698883.292904894798994, 6759331.322881896980107 ], [ 698877.586065614479594, 6759326.5 ], [ 698872.313280044589192, 6759325.333572533912957 ], [ 698861.586179815232754, 6759318.5 ], [ 698856.443024209002033, 6759318.392759121023118 ], [ 698848.586179815232754, 6759313.5 ], [ 698836.586179815232754, 6759310.5 ], [ 698824.5, 6759310.5 ], [ 698822.159515680046752, 6759318.951589926145971 ], [ 698815.5, 6759320.5 ], [ 698814.060660171788186, 6759324.060660171322525 ], [ 698810.833355349488556, 6759327.247204418294132 ], [ 698808.5, 6759327.5 ], [ 698806.086370622622781, 6759333.034310818649828 ], [ 698801.5, 6759335.413934385403991 ], [ 698801.262332465150394, 6759337.81025721039623 ], [ 698793.5, 6759345.5 ], [ 698791.159515680046752, 6759350.951589926145971 ], [ 698788.5, 6759351.5 ], [ 698789.426442500087433, 6759355.463963138870895 ], [ 698786.5, 6759356.5 ], [ 698785.412305920384824, 6759361.5 ], [ 698788.488999214023352, 6759365.181332128122449 ], [ 698785.729315414093435, 6759369.310762764886022 ], [ 698781.725387349841185, 6759366.790798705071211 ], [ 698783.587694079615176, 6759362.5 ], [ 698781.5, 6759362.5 ], [ 698775.810257209115662, 6759370.262332466430962 ], [ 698771.413820184767246, 6759370.5 ], [ 698762.422197406878695, 6759375.43935726955533 ], [ 698758.413934385520406, 6759375.5 ], [ 698752.556975790997967, 6759381.392759121023118 ], [ 698741.142445242963731, 6759384.49322113301605 ], [ 698729.716397003736347, 6759383.472945803776383 ], [ 698724.820317181060091, 6759379.926470963284373 ], [ 698718.5, 6759369.5 ], [ 698715.586065614479594, 6759366.5 ], [ 698711.965689182863571, 6759366.086370624601841 ], [ 698707.5, 6759359.5 ], [ 698701.048410073737614, 6759356.159515679813921 ], [ 698698.586065614479594, 6759352.5 ], [ 698695.048410073737614, 6759352.159515679813921 ], [ 698694.5, 6759349.5 ], [ 698690.189742790884338, 6759348.262332466430962 ], [ 698686.5, 6759344.5 ], [ 698683.166644650511444, 6759344.247204418294132 ], [ 698677.586065614479594, 6759338.5 ], [ 698664.577802593121305, 6759335.43935726955533 ], [ 698656.586179815232754, 6759331.5 ], [ 698641.577802593121305, 6759330.43935726955533 ], [ 698632.073529037996195, 6759325.179682820104063 ], [ 698623.820317181060091, 6759316.926470963284373 ], [ 698623.5, 6759314.5 ], [ 698620.737667534849606, 6759312.81025721039623 ], [ 698613.5, 6759300.413934385403991 ], [ 698609.737667534849606, 6759297.81025721039623 ], [ 698609.5, 6759295.5 ], [ 698606.939339828211814, 6759295.060660171322525 ], [ 698599.586065614479594, 6759285.5 ], [ 698596.443024209002033, 6759285.392759121023118 ], [ 698591.586179815232754, 6759282.5 ], [ 698580.0, 6759283.5 ], [ 698565.857554757036269, 6759282.49322113301605 ], [ 698527.716397003736347, 6759277.472945803776383 ], [ 698522.425974851474166, 6759276.385819299146533 ], [ 698521.5, 6759274.5 ], [ 698516.425974851474166, 6759273.385819299146533 ], [ 698512.586179815232754, 6759270.5 ], [ 698495.586179815232754, 6759266.5 ], [ 698464.413820184767246, 6759265.5 ], [ 698458.413934385520406, 6759266.5 ], [ 698456.283602996263653, 6759268.472945803776383 ], [ 698446.142445242963731, 6759269.49322113301605 ], [ 698438.413820184767246, 6759268.5 ], [ 698411.577802593121305, 6759270.43935726955533 ], [ 698393.939339828211814, 6759261.060660171322525 ], [ 698388.752795581589453, 6759255.833355349488556 ], [ 698388.5, 6759253.413934385403991 ], [ 698384.840484319953248, 6759250.951589926145971 ], [ 698380.5, 6759243.5 ], [ 698377.737667534849606, 6759241.81025721039623 ], [ 698375.5, 6759236.5 ], [ 698373.614180701202713, 6759235.574025148525834 ], [ 698373.5, 6759231.5 ], [ 698369.939339828211814, 6759230.060660171322525 ], [ 698369.5, 6759227.5 ], [ 698363.292904894798994, 6759226.322881896980107 ], [ 698359.5, 6759222.5 ], [ 698352.716397003736347, 6759222.472945803776383 ], [ 698340.443024209002033, 6759219.392759121023118 ], [ 698335.5, 6759215.5 ], [ 698329.048410073737614, 6759213.159515679813921 ], [ 698328.5, 6759210.5 ], [ 698316.852974289562553, 6759210.492777089588344 ], [ 698315.5, 6759208.5 ], [ 698313.5, 6759208.5 ], [ 698313.159515680046752, 6759210.951589926145971 ], [ 698310.857554757036269, 6759211.49322113301605 ], [ 698306.292904894798994, 6759210.322881896980107 ], [ 698301.586179815232754, 6759206.5 ], [ 698298.292904894798994, 6759206.322881896980107 ], [ 698297.5, 6759204.5 ], [ 698291.536036859615706, 6759206.426442499272525 ], [ 698289.586065614479594, 6759203.5 ], [ 698281.716397003736347, 6759203.472945803776383 ], [ 698275.292904894798994, 6759201.322881896980107 ], [ 698274.5, 6759197.5 ], [ 698271.5, 6759197.5 ], [ 698272.483764764852822, 6759201.220095711760223 ] ] ] } } -] -} diff --git a/test_files/44.geojson b/test_files/44.geojson deleted file mode 100644 index 0618305b..00000000 --- a/test_files/44.geojson +++ /dev/null @@ -1,7 +0,0 @@ -{ -"type": "FeatureCollection", -"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, -"features": [ -{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 699038.5, 6759434.5 ], [ 699036.5, 6759442.5 ], [ 699042.413820184767246, 6759446.5 ], [ 699045.833355349488556, 6759446.752795581705868 ], [ 699046.5, 6759449.5 ], [ 699051.951589926262386, 6759451.840484320186079 ], [ 699054.5, 6759455.5 ], [ 699056.5, 6759455.5 ], [ 699058.5, 6759443.5 ], [ 699053.292904894798994, 6759443.322881896980107 ], [ 699051.586065614479594, 6759439.5 ], [ 699040.586179815232754, 6759434.5 ], [ 699038.5, 6759434.5 ] ] ] } } -] -} diff --git a/test_files/90.geojson b/test_files/90.geojson deleted file mode 100644 index e2ae0971..00000000 --- a/test_files/90.geojson +++ /dev/null @@ -1,7 +0,0 @@ -{ -"type": "FeatureCollection", -"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::2154" } }, -"features": [ -{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 706730.5, 6754661.5 ], [ 706730.5, 6754664.5 ], [ 706737.5, 6754663.5 ], [ 706740.5, 6754654.586179815232754 ], [ 706739.516235235147178, 6754649.779904288239777 ], [ 706743.5, 6754637.586179815232754 ], [ 706743.5606427306775, 6754632.577802591957152 ], [ 706746.752795581589453, 6754625.166644650511444 ], [ 706748.5, 6754624.5 ], [ 706747.501806815736927, 6754620.926398488692939 ], [ 706748.528822079417296, 6754605.707364517264068 ], [ 706750.5, 6754604.5 ], [ 706754.56458949635271, 6754579.564572984352708 ], [ 706757.5, 6754578.5 ], [ 706755.381012363242917, 6754573.279904288239777 ], [ 706759.607240879791789, 6754559.44302420783788 ], [ 706761.5, 6754557.586065614596009 ], [ 706760.506778866751119, 6754551.857554757036269 ], [ 706764.5, 6754538.586179815232754 ], [ 706764.5606427306775, 6754533.577802591957152 ], [ 706766.5, 6754530.586179815232754 ], [ 706766.527054196456447, 6754516.716397003270686 ], [ 706769.5, 6754510.586179815232754 ], [ 706772.5, 6754497.586179815232754 ], [ 706772.56458949635271, 6754488.564572984352708 ], [ 706775.5, 6754487.5 ], [ 706774.501806815736927, 6754479.926398488692939 ], [ 706775.5606427306775, 6754473.577802591957152 ], [ 706777.666427467018366, 6754467.313280043192208 ], [ 706780.5, 6754464.586065614596009 ], [ 706780.528822079417296, 6754458.707364517264068 ], [ 706782.5, 6754457.5 ], [ 706783.666427467018366, 6754451.313280043192208 ], [ 706787.5, 6754449.5 ], [ 706788.677118103485554, 6754441.292904894798994 ], [ 706793.5, 6754438.5 ], [ 706794.048410073737614, 6754432.840484320186079 ], [ 706800.5, 6754431.5 ], [ 706800.5, 6754429.5 ], [ 706791.5, 6754429.5 ], [ 706790.036811842699535, 6754431.49954822845757 ], [ 706787.913629377377219, 6754430.034310818649828 ], [ 706788.5, 6754421.413820184767246 ], [ 706786.501806815736927, 6754417.073601511307061 ], [ 706789.5, 6754405.586179815232754 ], [ 706789.5606427306775, 6754395.577802591957152 ], [ 706790.913629377377219, 6754391.965689181350172 ], [ 706794.5, 6754390.5 ], [ 706794.5, 6754388.5 ], [ 706792.528822079417296, 6754387.292635482735932 ], [ 706792.5, 6754380.0 ], [ 706795.5, 6754378.5 ], [ 706793.5, 6754375.0 ], [ 706793.506778866751119, 6754364.857554757036269 ], [ 706795.5, 6754357.586179815232754 ], [ 706794.501806815736927, 6754349.926398488692939 ], [ 706796.5, 6754339.586179815232754 ], [ 706797.528822079417296, 6754315.707364517264068 ], [ 706800.5, 6754313.586065614596009 ], [ 706800.5, 6754311.413934385403991 ], [ 706797.614180701202713, 6754309.574025148525834 ], [ 706796.5, 6754303.0 ], [ 706798.5, 6754295.586179815232754 ], [ 706797.528822079417296, 6754286.707364517264068 ], [ 706799.5, 6754285.5 ], [ 706799.5, 6754283.5 ], [ 706796.500451771891676, 6754282.036811839789152 ], [ 706796.527054196456447, 6754268.716397003270686 ], [ 706799.5, 6754265.5 ], [ 706796.5, 6754258.0 ], [ 706796.5, 6754233.0 ], [ 706798.5, 6754209.413934385403991 ], [ 706795.5606427306775, 6754205.422197408042848 ], [ 706795.5, 6754195.413934385403991 ], [ 706793.516235235147178, 6754193.220095711760223 ], [ 706794.5, 6754164.413820184767246 ], [ 706789.506778866751119, 6754141.142445242963731 ], [ 706789.506778866751119, 6754128.857554757036269 ], [ 706794.5, 6754116.586179815232754 ], [ 706797.614180701202713, 6754103.425974851474166 ], [ 706799.5, 6754102.5 ], [ 706799.939339828211814, 6754098.939339828677475 ], [ 706803.5, 6754097.5 ], [ 706801.5, 6754094.0 ], [ 706803.5, 6754075.586179815232754 ], [ 706803.5, 6754059.413820184767246 ], [ 706801.501806815736927, 6754045.073601511307061 ], [ 706802.5, 6754038.413934385403991 ], [ 706799.527054196456447, 6754032.283602996729314 ], [ 706798.5, 6754020.5 ], [ 706795.666427467018366, 6754018.686719956807792 ], [ 706792.5, 6754010.0 ], [ 706793.607240879791789, 6753999.44302420783788 ], [ 706798.5, 6753998.5 ], [ 706799.5, 6753992.5 ], [ 706795.292904894798994, 6753992.322881896980107 ], [ 706793.527054196456447, 6753989.283602996729314 ], [ 706793.506778866751119, 6753977.857554757036269 ], [ 706795.5, 6753969.5 ], [ 706793.5, 6753969.5 ], [ 706793.254821590846404, 6753971.821841089054942 ], [ 706791.606587939313613, 6753972.447489879094064 ], [ 706787.501806815736927, 6753966.926398488692939 ], [ 706791.577802593121305, 6753962.56064273044467 ], [ 706800.5, 6753963.5 ], [ 706802.073529037996195, 6753960.820317179895937 ], [ 706805.5, 6753959.5 ], [ 706805.5, 6753957.413820184767246 ], [ 706804.5, 6753951.5 ], [ 706802.5, 6753951.5 ], [ 706801.435427015880123, 6753954.435410503298044 ], [ 706798.507222909945995, 6753953.147025710903108 ], [ 706802.5, 6753946.586179815232754 ], [ 706800.501806815736927, 6753938.073601511307061 ], [ 706800.516235235147178, 6753931.779904288239777 ], [ 706802.5, 6753927.5 ], [ 706787.425974851474166, 6753922.385819299146533 ], [ 706784.586179815232754, 6753919.5 ], [ 706782.5, 6753919.5 ], [ 706781.5, 6753922.413820184767246 ], [ 706781.5, 6753926.5 ], [ 706785.220095711643808, 6753925.516235235147178 ], [ 706786.633416056167334, 6753927.426398488692939 ], [ 706786.472945803543553, 6753934.283602996729314 ], [ 706784.5, 6753936.413934385403991 ], [ 706783.472945803543553, 6753954.283602996729314 ], [ 706781.5, 6753958.413820184767246 ], [ 706782.389315363252535, 6753965.565511114895344 ], [ 706780.468754711793736, 6753966.402775265276432 ], [ 706777.5, 6753963.5 ], [ 706773.5, 6753962.5 ], [ 706774.471177920582704, 6753967.292635482735932 ], [ 706772.5, 6753968.5 ], [ 706771.5, 6753974.5 ], [ 706774.951589926262386, 6753974.840484320186079 ], [ 706775.322881896514446, 6753977.707095105201006 ], [ 706768.5, 6753978.5 ], [ 706770.5, 6753983.0 ], [ 706768.5, 6753985.413934385403991 ], [ 706768.247204418410547, 6753989.833355349488556 ], [ 706763.5, 6753990.5 ], [ 706761.5, 6753992.5 ], [ 706758.262332465150394, 6753999.81025721039623 ], [ 706751.5, 6754008.413934385403991 ], [ 706750.262332465150394, 6754012.81025721039623 ], [ 706745.5, 6754018.413934385403991 ], [ 706745.221054494613782, 6754022.871220936998725 ], [ 706742.5, 6754023.5 ], [ 706734.5, 6754032.5 ], [ 706734.5, 6754040.5 ], [ 706739.147025710437447, 6754039.507222910411656 ], [ 706740.5, 6754042.5 ], [ 706750.5, 6754042.5 ], [ 706753.677118103485554, 6754034.292904894798994 ], [ 706755.5, 6754033.5 ], [ 706755.671733715804294, 6754027.303071970120072 ], [ 706760.5, 6754027.5 ], [ 706762.614180701202713, 6754018.425974851474166 ], [ 706764.5, 6754017.5 ], [ 706766.752795581589453, 6754011.166644650511444 ], [ 706771.166644650511444, 6754006.752795581705868 ], [ 706778.852388428989798, 6754003.765725327655673 ], [ 706780.5, 6754010.5 ], [ 706784.4393572693225, 6754014.577802591957152 ], [ 706786.5, 6754031.586179815232754 ], [ 706790.471177920582704, 6754037.707364517264068 ], [ 706790.5, 6754050.586179815232754 ], [ 706792.493221133248881, 6754054.857554757036269 ], [ 706793.5, 6754074.0 ], [ 706791.472945803543553, 6754092.283602996729314 ], [ 706784.5, 6754108.413820184767246 ], [ 706780.5, 6754130.413820184767246 ], [ 706780.5, 6754141.586179815232754 ], [ 706783.472945803543553, 6754148.716397003270686 ], [ 706787.493221133248881, 6754173.857554757036269 ], [ 706789.498193184263073, 6754203.926398488692939 ], [ 706788.060660171788186, 6754208.060660171322525 ], [ 706783.5, 6754209.5 ], [ 706784.413934385520406, 6754212.5 ], [ 706788.385819298797287, 6754213.425974851474166 ], [ 706789.5, 6754223.0 ], [ 706788.385819298797287, 6754233.574025148525834 ], [ 706784.5, 6754235.5 ], [ 706784.5, 6754237.5 ], [ 706788.471177920582704, 6754238.707364517264068 ], [ 706789.483764764852822, 6754245.220095711760223 ], [ 706788.43541050364729, 6754253.435427015647292 ], [ 706786.5, 6754254.5 ], [ 706786.5, 6754256.5 ], [ 706788.247204418410547, 6754257.166644650511444 ], [ 706790.5, 6754262.0 ], [ 706788.5, 6754265.413934385403991 ], [ 706789.5, 6754286.0 ], [ 706787.5, 6754297.413820184767246 ], [ 706788.493221133248881, 6754305.142445242963731 ], [ 706786.5, 6754310.413820184767246 ], [ 706786.5, 6754312.5 ], [ 706788.5, 6754314.0 ], [ 706786.5, 6754348.5 ], [ 706788.496103194658644, 6754350.108051982708275 ], [ 706784.5, 6754371.413820184767246 ], [ 706784.471177920582704, 6754381.292635482735932 ], [ 706782.5, 6754382.5 ], [ 706783.633416056167334, 6754389.573601511307061 ], [ 706781.471177920582704, 6754404.292635482735932 ], [ 706779.5, 6754405.5 ], [ 706780.455046879593283, 6754409.364470270462334 ], [ 706777.5, 6754415.413820184767246 ], [ 706778.493221133248881, 6754426.142445242963731 ], [ 706776.5, 6754430.413820184767246 ], [ 706775.472945803543553, 6754445.283602996729314 ], [ 706772.5, 6754449.413820184767246 ], [ 706774.472945803543553, 6754456.283602996729314 ], [ 706772.951589926262386, 6754459.159515679813921 ], [ 706770.5, 6754459.5 ], [ 706768.471177920582704, 6754480.292635482735932 ], [ 706766.5, 6754481.5 ], [ 706765.5, 6754486.413820184767246 ], [ 706765.43541050364729, 6754496.435427015647292 ], [ 706761.5, 6754498.413934385403991 ], [ 706763.493221133248881, 6754504.142445242963731 ], [ 706761.5, 6754506.413934385403991 ], [ 706760.4393572693225, 6754518.422197408042848 ], [ 706758.5, 6754520.413934385403991 ], [ 706759.493221133248881, 6754525.142445242963731 ], [ 706756.5, 6754535.413820184767246 ], [ 706756.472945803543553, 6754545.283602996729314 ], [ 706750.5, 6754565.413820184767246 ], [ 706745.4393572693225, 6754595.422197408042848 ], [ 706742.5, 6754600.413820184767246 ], [ 706740.472945803543553, 6754620.283602996729314 ], [ 706736.5, 6754631.413820184767246 ], [ 706735.4393572693225, 6754642.422197408042848 ], [ 706732.5, 6754648.413820184767246 ], [ 706732.471177920582704, 6754660.292635482735932 ], [ 706730.5, 6754661.5 ] ] ] } } -] -} From 5331b8aecafd7dd56697cb144776534cfea3be59 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Thu, 8 Aug 2024 09:10:22 +0200 Subject: [PATCH 44/68] removing files coming from God knows where --- Dockerfile | 16 -- Makefile | 40 ----- .../2024-07-03/10-22-51/.hydra/config.yaml | 20 --- outputs/2024-07-03/10-22-51/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/10-22-51/.hydra/overrides.yaml | 6 - outputs/2024-07-03/10-22-51/main_skeleton.log | 0 .../2024-07-03/10-36-27/.hydra/config.yaml | 20 --- outputs/2024-07-03/10-36-27/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/10-36-27/.hydra/overrides.yaml | 6 - outputs/2024-07-03/10-36-27/main_skeleton.log | 0 .../2024-07-03/11-48-08/.hydra/config.yaml | 20 --- outputs/2024-07-03/11-48-08/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/11-48-08/.hydra/overrides.yaml | 6 - outputs/2024-07-03/11-48-08/main_skeleton.log | 0 .../2024-07-03/11-55-24/.hydra/config.yaml | 20 --- outputs/2024-07-03/11-55-24/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/11-55-24/.hydra/overrides.yaml | 6 - outputs/2024-07-03/11-55-24/main_skeleton.log | 0 .../2024-07-03/12-10-55/.hydra/config.yaml | 20 --- outputs/2024-07-03/12-10-55/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/12-10-55/.hydra/overrides.yaml | 6 - outputs/2024-07-03/12-10-55/main_skeleton.log | 0 .../2024-07-03/12-12-11/.hydra/config.yaml | 20 --- outputs/2024-07-03/12-12-11/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/12-12-11/.hydra/overrides.yaml | 6 - outputs/2024-07-03/12-12-11/main_skeleton.log | 0 .../2024-07-03/14-03-39/.hydra/config.yaml | 20 --- outputs/2024-07-03/14-03-39/.hydra/hydra.yaml | 163 ------------------ .../2024-07-03/14-03-39/.hydra/overrides.yaml | 6 - outputs/2024-07-03/14-03-39/main_skeleton.log | 0 30 files changed, 1379 deletions(-) delete mode 100644 Dockerfile delete mode 100644 Makefile delete mode 100644 outputs/2024-07-03/10-22-51/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/10-22-51/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/10-22-51/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/10-22-51/main_skeleton.log delete mode 100644 outputs/2024-07-03/10-36-27/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/10-36-27/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/10-36-27/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/10-36-27/main_skeleton.log delete mode 100644 outputs/2024-07-03/11-48-08/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/11-48-08/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/11-48-08/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/11-48-08/main_skeleton.log delete mode 100644 outputs/2024-07-03/11-55-24/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/11-55-24/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/11-55-24/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/11-55-24/main_skeleton.log delete mode 100644 outputs/2024-07-03/12-10-55/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/12-10-55/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/12-10-55/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/12-10-55/main_skeleton.log delete mode 100644 outputs/2024-07-03/12-12-11/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/12-12-11/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/12-12-11/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/12-12-11/main_skeleton.log delete mode 100644 outputs/2024-07-03/14-03-39/.hydra/config.yaml delete mode 100644 outputs/2024-07-03/14-03-39/.hydra/hydra.yaml delete mode 100644 outputs/2024-07-03/14-03-39/.hydra/overrides.yaml delete mode 100644 outputs/2024-07-03/14-03-39/main_skeleton.log diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e69d4f0a..00000000 --- a/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM mambaorg/micromamba:latest as mamba_pdal -COPY environment.yml /environment.yml -USER root -RUN micromamba env create -n lidro -f /environment.yml - -FROM debian:bullseye-slim - - -WORKDIR /lidro -RUN mkdir tmp -COPY lidro lidro -COPY test test -COPY configs configs - -# Copy test data that are stored directly in the lidro-data repository ("http://gitlab.forge-idi.ign.fr/Lidar/lidro-data.git") -COPY data/pointcloud data/pointcloud \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 001ed431..00000000 --- a/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -# Makefile to manage main tasks -# cf. https://blog.ianpreston.ca/conda/python/bash/2020/05/13/conda_envs.html#makefile - -# Oneshell means I can run multiple lines in a recipe in the same shell, so I don't have to -# chain commands together with semicolon -.ONESHELL: -SHELL = /bin/bash -install: - pip install -e . - -install-precommit: - pre-commit install - -testing: - ./ci/test.sh - -mamba-env-create: - mamba env create -n lidro -f environment.yml - -mamba-env-update: - mamba env update -n lidro -f environment.yml - -############################## -# Docker -############################## - -PROJECT_NAME=ignimagelidar/lidro -VERSION=`python -m lidro.version` - -docker-build: - docker build -t ${PROJECT_NAME}:${VERSION} -f Dockerfile . - -docker-test: - docker run --rm -it ${PROJECT_NAME}:${VERSION} python -m pytest -s - -docker-remove: - docker rmi -f `docker images | grep ${PROJECT_NAME} | tr -s ' ' | cut -d ' ' -f 3` - -docker-deploy: - docker push ${PROJECT_NAME}:${VERSION} \ No newline at end of file diff --git a/outputs/2024-07-03/10-22-51/.hydra/config.yaml b/outputs/2024-07-03/10-22-51/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/10-22-51/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/10-22-51/.hydra/hydra.yaml b/outputs/2024-07-03/10-22-51/.hydra/hydra.yaml deleted file mode 100644 index 3b479709..00000000 --- a/outputs/2024-07-03/10-22-51/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/10-22-51 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/10-22-51/.hydra/overrides.yaml b/outputs/2024-07-03/10-22-51/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/10-22-51/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/10-22-51/main_skeleton.log b/outputs/2024-07-03/10-22-51/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 diff --git a/outputs/2024-07-03/10-36-27/.hydra/config.yaml b/outputs/2024-07-03/10-36-27/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/10-36-27/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/10-36-27/.hydra/hydra.yaml b/outputs/2024-07-03/10-36-27/.hydra/hydra.yaml deleted file mode 100644 index 2d8d6f93..00000000 --- a/outputs/2024-07-03/10-36-27/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/10-36-27 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/10-36-27/.hydra/overrides.yaml b/outputs/2024-07-03/10-36-27/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/10-36-27/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/10-36-27/main_skeleton.log b/outputs/2024-07-03/10-36-27/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 diff --git a/outputs/2024-07-03/11-48-08/.hydra/config.yaml b/outputs/2024-07-03/11-48-08/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/11-48-08/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/11-48-08/.hydra/hydra.yaml b/outputs/2024-07-03/11-48-08/.hydra/hydra.yaml deleted file mode 100644 index 177cd2db..00000000 --- a/outputs/2024-07-03/11-48-08/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/11-48-08 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/11-48-08/.hydra/overrides.yaml b/outputs/2024-07-03/11-48-08/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/11-48-08/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/11-48-08/main_skeleton.log b/outputs/2024-07-03/11-48-08/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 diff --git a/outputs/2024-07-03/11-55-24/.hydra/config.yaml b/outputs/2024-07-03/11-55-24/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/11-55-24/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/11-55-24/.hydra/hydra.yaml b/outputs/2024-07-03/11-55-24/.hydra/hydra.yaml deleted file mode 100644 index 849089ba..00000000 --- a/outputs/2024-07-03/11-55-24/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/11-55-24 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/11-55-24/.hydra/overrides.yaml b/outputs/2024-07-03/11-55-24/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/11-55-24/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/11-55-24/main_skeleton.log b/outputs/2024-07-03/11-55-24/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 diff --git a/outputs/2024-07-03/12-10-55/.hydra/config.yaml b/outputs/2024-07-03/12-10-55/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/12-10-55/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/12-10-55/.hydra/hydra.yaml b/outputs/2024-07-03/12-10-55/.hydra/hydra.yaml deleted file mode 100644 index 126a896d..00000000 --- a/outputs/2024-07-03/12-10-55/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/12-10-55 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/12-10-55/.hydra/overrides.yaml b/outputs/2024-07-03/12-10-55/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/12-10-55/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/12-10-55/main_skeleton.log b/outputs/2024-07-03/12-10-55/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 diff --git a/outputs/2024-07-03/12-12-11/.hydra/config.yaml b/outputs/2024-07-03/12-12-11/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/12-12-11/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/12-12-11/.hydra/hydra.yaml b/outputs/2024-07-03/12-12-11/.hydra/hydra.yaml deleted file mode 100644 index ec0c212a..00000000 --- a/outputs/2024-07-03/12-12-11/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/12-12-11 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/12-12-11/.hydra/overrides.yaml b/outputs/2024-07-03/12-12-11/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/12-12-11/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/12-12-11/main_skeleton.log b/outputs/2024-07-03/12-12-11/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 diff --git a/outputs/2024-07-03/14-03-39/.hydra/config.yaml b/outputs/2024-07-03/14-03-39/.hydra/config.yaml deleted file mode 100644 index 89be70a7..00000000 --- a/outputs/2024-07-03/14-03-39/.hydra/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -FILE_PATH: - MASK_INPUT_PATH: /home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - SKELETON_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/skeleton_lines.geojson - GAP_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/gap_lines.geojson - GLOBAL_LINES_OUTPUT_PATH: /home/MDaab/code/lidro/global_lines.geojson -MAX_GAP_WIDTH: 200 -MAX_BRIDGES: 1 -GAP_WIDTH_CHECK_DB: 30 -RATIO_GAP: 0.25 -DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: invite - DB_PASSWORD: 28de# - DB_PORT: 5432 -BRANCH: - VORONOI_MAX_LENGTH: 2 - WATER_MIN_SIZE: 20 - MAX_GAP_CANDIDATES: 3 diff --git a/outputs/2024-07-03/14-03-39/.hydra/hydra.yaml b/outputs/2024-07-03/14-03-39/.hydra/hydra.yaml deleted file mode 100644 index adc5eccd..00000000 --- a/outputs/2024-07-03/14-03-39/.hydra/hydra.yaml +++ /dev/null @@ -1,163 +0,0 @@ -hydra: - run: - dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - params: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - mode: RUN - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: - - hydra.mode=RUN - task: - - FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson - - FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - - FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson - - FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson - - DB_UNI.DB_USER=invite - - DB_UNI.DB_PASSWORD='28de#' - job: - name: main_skeleton - chdir: null - override_dirname: DB_UNI.DB_PASSWORD='28de#',DB_UNI.DB_USER=invite,FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson,FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson,FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson,FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson - id: ??? - num: ??? - config_name: configs_skeleton.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.3.2 - version_base: '1.2' - cwd: /home/MDaab/code/lidro - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /home/MDaab/code/lidro/configs - schema: file - provider: main - - path: hydra_plugins.hydra_colorlog.conf - schema: pkg - provider: hydra-colorlog - - path: '' - schema: structured - provider: schema - output_dir: /home/MDaab/code/lidro/outputs/2024-07-03/14-03-39 - choices: - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/outputs/2024-07-03/14-03-39/.hydra/overrides.yaml b/outputs/2024-07-03/14-03-39/.hydra/overrides.yaml deleted file mode 100644 index 907f19b5..00000000 --- a/outputs/2024-07-03/14-03-39/.hydra/overrides.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- FILE_PATH.MASK_INPUT_PATH=/home/MDaab/data/squelette_test/Mask_Hydro_without_vegetation.geojson -- FILE_PATH.SKELETON_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/skeleton_lines.geojson -- FILE_PATH.GAP_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/gap_lines.geojson -- FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=/home/MDaab/code/lidro/global_lines.geojson -- DB_UNI.DB_USER=invite -- DB_UNI.DB_PASSWORD='28de#' diff --git a/outputs/2024-07-03/14-03-39/main_skeleton.log b/outputs/2024-07-03/14-03-39/main_skeleton.log deleted file mode 100644 index e69de29b..00000000 From 7c09772a8bd88b5e2eef788baff7e544e476809b Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Fri, 9 Aug 2024 11:28:24 +0200 Subject: [PATCH 45/68] removing .env --- .env | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 58e69023..00000000 --- a/.env +++ /dev/null @@ -1,2 +0,0 @@ -DB_UNI_USER='invite' -DB_UNI_PASSWORD='28de#' \ No newline at end of file From 7c69a401f479e1b23dd4bea78dfa2a13f7a819dd Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 12 Aug 2024 08:30:20 +0200 Subject: [PATCH 46/68] adding "null" to some config values to be consistent --- configs/configs_lidro.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index f020c57a..da709cf2 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -49,10 +49,10 @@ virtual_point: SKELETON: FILE_PATH: - MASK_INPUT_PATH: - SKELETON_LINES_OUTPUT_PATH: # only branches' skeleton (if empty, no file is created) - GAP_LINES_OUTPUT_PATH: # only new lines to close gaps (if empty, no file is created) - GLOBAL_LINES_OUTPUT_PATH: # lines from both the skeletons and the gap lines + MASK_INPUT_PATH: null + SKELETON_LINES_OUTPUT_PATH: null # only branches' skeleton (if empty, no file is created) + GAP_LINES_OUTPUT_PATH: null # only new lines to close gaps (if empty, no file is created) + GLOBAL_LINES_OUTPUT_PATH: null # lines from both the skeletons and the gap lines MAX_GAP_WIDTH: 200 # distance max in meter of any gap between 2 branches we will try to close with a line MAX_BRIDGES: 1 # number max of bridges that can be created between 2 branches From 2fe5a401ac7fdc7f5e80e792da1b71a281ff307a Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 12 Aug 2024 08:33:42 +0200 Subject: [PATCH 47/68] add null to config str to be consistent --- configs/configs_lidro.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index da709cf2..58d95250 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -64,8 +64,8 @@ SKELETON: DB_UNI: DB_NAME: bduni_france_consultation DB_HOST: bduni_consult.ign.fr - DB_USER: - DB_PASSWORD: + DB_USER: null + DB_PASSWORD: null DB_PORT: 5432 BRANCH: From ca9c0b9560aba431dff85cc93ad328e71efd2705 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 12 Aug 2024 08:55:37 +0200 Subject: [PATCH 48/68] a strange case of problem with git merge, manually solved --- lidro/create_mask_hydro/vectors/convert_to_vector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lidro/create_mask_hydro/vectors/convert_to_vector.py b/lidro/create_mask_hydro/vectors/convert_to_vector.py index 3b0657d5..867e2f36 100644 --- a/lidro/create_mask_hydro/vectors/convert_to_vector.py +++ b/lidro/create_mask_hydro/vectors/convert_to_vector.py @@ -10,7 +10,9 @@ from lidro.create_mask_hydro.rasters.create_mask_raster import detect_hydro_by_tile -def create_hydro_vector_mask(filename: str, output: str, pixel_size: float, tile_size: int, classes: list, crs: str, dilatation_size: int): +def create_hydro_vector_mask( + filename: str, output: str, pixel_size: float, tile_size: int, classes: list, crs: str, dilatation_size: int +): """Create a vector mask of hydro surfaces in a tile from the points classification of the input LAS/LAZ file, and save it as a GeoJSON file. From edc195b1508b6e4bc54a51735554eef0a5161e97 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 12 Aug 2024 10:31:15 +0200 Subject: [PATCH 49/68] moved the "make valid" at the beginning, to have valid polygons right at the start Fix the test to have the skeleton lines output --- lidro/main_create_skeleton_lines.py | 2 +- lidro/skeleton/branch.py | 22 +--------------------- lidro/skeleton/create_skeleton_lines.py | 23 ++++++++++++++++++++++- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 5f0dcd1a..fbd4803b 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -55,7 +55,7 @@ def run(config: DictConfig): # putting all skeleton lines together, and save them if there is a path branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH: + if config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH: gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') # saving all lines diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index 5226e024..052230aa 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -61,16 +61,8 @@ def set_gdf_branch_mask(self, branch_mask: GeoDataFrame): """ raw_gdf_branch_mask = gpd.GeoDataFrame(geometry=[branch_mask], crs=self.crs) - # create simple geometry - gdf_simple = raw_gdf_branch_mask.explode(ignore_index=True) - gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) - # remove invalid polygons - gdf_valid = gdf_without_duplicates.copy() - gdf_valid.geometry = gdf_valid.geometry.apply( - lambda geom: fix_invalid_geometry(geom) - ) # # we keep only the exterior ring (that should be unique) as our polygon, to simplify the result - self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[Polygon(gdf_valid.exterior[0].coords)], crs=self.crs) + self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[Polygon(raw_gdf_branch_mask.exterior[0].coords)], crs=self.crs) def create_skeleton(self): """ @@ -307,18 +299,6 @@ def get_df_points_from_gdf(gdf: GeoDataFrame) -> pd.DataFrame: 'y': [point[1] for point in all_points], }) -def fix_invalid_geometry(geometry): - """ - return the geometry, fixed - Args: - - gdf_lines:geodataframe containing a list of lines - """ - if not geometry.is_valid: - return make_valid(geometry) - else: - return geometry - - def get_vertices_dict(gdf_lines: GeoDataFrame) -> Dict[Point, List[LineString]]: """ get a dictionary of vertices listing all the lines having a specific vertex as one of its extremities diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index 909e4397..b0f30fbf 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -6,6 +6,7 @@ from geopandas.geodataframe import GeoDataFrame import psycopg +from shapely import make_valid from lidro.skeleton.branch import Branch, Candidate from lidro.skeleton.group_maker import GroupMaker @@ -138,6 +139,16 @@ def select_candidates( break return validated_candidates +def fix_invalid_geometry(geometry): + """ + return the geometry, fixed + Args: + - gdf_lines:geodataframe containing a list of lines + """ + if not geometry.is_valid: + return make_valid(geometry, "structure") + else: + return geometry def create_branches_list(config: DictConfig, gdf_hydro_global_mask: GeoDataFrame, crs: CRS) -> List[Branch]: """ @@ -149,8 +160,18 @@ def create_branches_list(config: DictConfig, gdf_hydro_global_mask: GeoDataFrame - crs (CRS); the crs of gdf_hydro_global_mask """ + # create simple geometry + gdf_simple = gdf_hydro_global_mask.explode(ignore_index=True) + gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) + + # make geometry valid and remove unwanted "lone" lines + gdf_valid = gdf_without_duplicates.copy() + gdf_valid.geometry = gdf_valid.geometry.apply( + lambda geom: fix_invalid_geometry(geom) + ) + branches_list = [] - for index_branch, branch_mask_row in gdf_hydro_global_mask.iterrows(): + for index_branch, branch_mask_row in gdf_valid.iterrows(): mask_branch = branch_mask_row["geometry"] new_branch = Branch(config, index_branch, mask_branch, crs) branches_list.append(new_branch) From 80f3c4180c624c1ed8441c259d12189e7fb301f2 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 12 Aug 2024 11:09:18 +0200 Subject: [PATCH 50/68] removed an unnecessary copy --- lidro/skeleton/create_skeleton_lines.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index b0f30fbf..c023dbe0 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -4,6 +4,7 @@ from omegaconf import DictConfig from pyproj.crs.crs import CRS +import geopandas as gpd from geopandas.geodataframe import GeoDataFrame import psycopg from shapely import make_valid @@ -146,7 +147,7 @@ def fix_invalid_geometry(geometry): - gdf_lines:geodataframe containing a list of lines """ if not geometry.is_valid: - return make_valid(geometry, "structure") + return make_valid(geometry, method="structure", keep_collapsed=False) else: return geometry @@ -165,10 +166,9 @@ def create_branches_list(config: DictConfig, gdf_hydro_global_mask: GeoDataFrame gdf_without_duplicates = gdf_simple.drop_duplicates(ignore_index=True) # make geometry valid and remove unwanted "lone" lines - gdf_valid = gdf_without_duplicates.copy() - gdf_valid.geometry = gdf_valid.geometry.apply( - lambda geom: fix_invalid_geometry(geom) - ) + # gdf_valid = gdf_without_duplicates.copy() + valid_geometry = gdf_without_duplicates.geometry.apply(lambda geom: fix_invalid_geometry(geom)) + gdf_valid = gpd.GeoDataFrame(geometry=valid_geometry) branches_list = [] for index_branch, branch_mask_row in gdf_valid.iterrows(): From 76487b7d108137df1f65a0596c58a4acefaa8da6 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 12 Aug 2024 11:18:27 +0200 Subject: [PATCH 51/68] removed a typo and a log file that has no business being here --- lidro/skeleton/branch.py | 2 +- main_create_mask.log | 44 ---------------------------------------- 2 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 main_create_mask.log diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index 052230aa..a0f3f806 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -74,7 +74,7 @@ def create_skeleton(self): self.gdf_skeleton_lines = gpd.GeoDataFrame(geometry=[], crs=self.crs) return - # draw a new line for each point added to closes gaps to the nearest points on voronoi_lines + # draw a new line for each point added to close gaps to the nearest points on voronoi_lines np_points = get_df_points_from_gdf(voronoi_lines).to_numpy().transpose() for gap_point in self.gap_points: distance_squared = (np_points[0] - gap_point.x)**2 + (np_points[1] - gap_point.y)**2 diff --git a/main_create_mask.log b/main_create_mask.log deleted file mode 100644 index 516562ab..00000000 --- a/main_create_mask.log +++ /dev/null @@ -1,44 +0,0 @@ -[2024-04-26 16:32:39,675][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:32:39,783][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:36:01,953][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:36:02,060][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:38:40,614][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:38:40,720][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:39:34,562][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:39:34,667][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:40:34,669][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:40:34,777][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:41:15,808][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:41:15,915][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:43:02,720][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:43:02,826][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:44:34,947][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:44:35,055][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:45:23,980][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:45:24,089][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:46:39,583][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:46:39,690][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 -[2024-04-26 16:47:36,360][root][INFO] - -Create Mask Hydro 1 for tile : LHD_FXX_0706_6627_PTS_C_LAMB93_IGN69_TEST -[2024-04-26 16:47:36,469][root][INFO] - -Create Mask Hydro 1 for tile : Semis_2021_0830_6291_LA93_IGN69 From eaeb81bdee5f705592362c8547f468a621370538 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 13 Aug 2024 11:13:50 +0200 Subject: [PATCH 52/68] Set the UPPERCASE of configs to lowercase --- README.md | 38 +++++++++++----------- configs/configs_lidro.yaml | 40 ++++++++++++------------ lidro/main_create_skeleton_lines.py | 12 +++---- lidro/skeleton/branch.py | 10 +++--- lidro/skeleton/create_skeleton_lines.py | 24 +++++++------- scripts/example_create_skeleton_lines.sh | 26 +++++++-------- test/skeleton/test_branch.py | 3 +- test/skeleton/test_main_skeleton.py | 4 +-- 8 files changed, 78 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 995b971a..4666d9c3 100644 --- a/README.md +++ b/README.md @@ -114,22 +114,22 @@ Pour fonctionner, la création de squelettes a besoin d'une série de paramètre python lidro/main_create_skeleton_lines.py [nom_paramètre_1]=[valeur_du_paramètre_1] [nom_paramètre_2]=[valeur_du_paramètre_2] ``` ces paramètres sont : -SKELETON.FILE_PATH.MASK_INPUT_PATH : Le chemin d'entrée des masques des cours d'eau -SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) -SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) -SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH : Le chemin de sortie des lignes et des squelettes ensemble - -SKELETON.MAX_GAP_WIDTH : La distance maximale envisagée pour franchir des ponts -SKELETON.MAX_BRIDGES : Le nombre maximal de ponts entre deux bras séparés de cours d'eau -SKELETON.GAP_WIDTH_CHECK_DB : La distance à partir de laquelle on vérifie via la base de données s'il y a bien un pont -SKELETON.RATIO_GAP : la proportion de la ligne franchissant un pont qui est comparé en base pour voir s'il y a bien un pont (trop grande et on pourrait trouver un pont qui ne correspond pas) - -SKELETON.DB_UNI.DB_NAME : Le nom de la base de données -SKELETON.DB_UNI.DB_HOST : l'adresse de la base de données -SKELETON.DB_UNI.DB_USER : L'utilisateur de la base de données -SKELETON.DB_UNI.DB_PASSWORD : Le mot de passe de l'utilisateur -SKELETON.DB_UNI.DB_PORT : La port de connexion avec la base de données - -SKELETON.BRANCH.VORONOI_MAX_LENGTH : LA longuer maximum des lignes individuelles des squelettes -SKELETON.BRANCH.WATER_MIN_SIZE : La longueur minimal à partir de laquelle une ligne de squelette sera automatiquement gardée (trop petite, et il y aura des sortes "d'aiguilles" qui apparaitront. Trop grande, et certains afluents ne seront pas détectés) -SKELETON.BRANCH.MAX_GAP_CANDIDATES : Le nombre maximum de candidats pour envisager de franchir des ponts entre deux bras \ No newline at end of file +skeleton.file_path.mask_input_path : Le chemin d'entrée des masques des cours d'eau +skeleton.file_path.skeleton_lines_ouput_path : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) +skeleton.file_path.gap_lines_ouput_path : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) +skeleton.file_path.global_lines_output_path : Le chemin de sortie des lignes et des squelettes ensemble + +skeleton.max_gap_width : La distance maximale envisagée pour franchir des ponts +skeleton.max_bridges : Le nombre maximal de ponts entre deux bras séparés de cours d'eau +skeleton.gap_width_check_db : La distance à partir de laquelle on vérifie via la base de données s'il y a bien un pont +skeleton.ratio_gap : la proportion de la ligne franchissant un pont qui est comparé en base pour voir s'il y a bien un pont (trop grande et on pourrait trouver un pont qui ne correspond pas) + +skeleton.db_uni.db_name : Le nom de la base de données +skeleton.db_uni.db_host : l'adresse de la base de données +skeleton.db_uni.db_user : L'utilisateur de la base de données +skeleton.db_uni.db_password : Le mot de passe de l'utilisateur +skeleton.db_uni.db_port : La port de connexion avec la base de données + +skeleton.branch.voronoi_max_length : LA longuer maximum des lignes individuelles des squelettes +skeleton.branch.water_min_size : La longueur minimal à partir de laquelle une ligne de squelette sera automatiquement gardée (trop petite, et il y aura des sortes "d'aiguilles" qui apparaitront. Trop grande, et certains afluents ne seront pas détectés) +skeleton.branch.max_gap_candidates : Le nombre maximum de candidats pour envisager de franchir des ponts entre deux bras \ No newline at end of file diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 58d95250..545beaaa 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -47,31 +47,31 @@ virtual_point: # The number of nearest neighbors to find with KNeighbors k: 20 -SKELETON: - FILE_PATH: - MASK_INPUT_PATH: null - SKELETON_LINES_OUTPUT_PATH: null # only branches' skeleton (if empty, no file is created) - GAP_LINES_OUTPUT_PATH: null # only new lines to close gaps (if empty, no file is created) - GLOBAL_LINES_OUTPUT_PATH: null # lines from both the skeletons and the gap lines +skeleton: + file_path: + mask_input_path: null + skeleton_lines_ouput_path: null # only branches' skeleton (if empty, no file is created) + gap_lines_ouput_path: null # only new lines to close gaps (if empty, no file is created) + global_lines_output_path: null # lines from both the skeletons and the gap lines - MAX_GAP_WIDTH: 200 # distance max in meter of any gap between 2 branches we will try to close with a line - MAX_BRIDGES: 1 # number max of bridges that can be created between 2 branches - GAP_WIDTH_CHECK_DB: 20 # for a gap at least this wide, we check with DB_UNI if there is a bridge. Smaller than that, we assume a line automatically exists - RATIO_GAP: 0.25 # when checking if a candidate line to close a gap intersects a bridge in BD UNI, that's the ratio ]0.1] of the line to consider. + max_gap_width: 200 # distance max in meter of any gap between 2 branches we will try to close with a line + max_bridges: 1 # number max of bridges that can be created between 2 branches + gap_width_check_db: 20 # for a gap at least this wide, we check with db_uni if there is a bridge. Smaller than that, we assume a line automatically exists + ratio_gap: 0.25 # when checking if a candidate line to close a gap intersects a bridge in BD UNI, that's the ratio ]0.1] of the line to consider. # we only check part of the line, because when the line is really long it can intersect with bridges not corresponding to that line, # so we make the line smaller - DB_UNI: - DB_NAME: bduni_france_consultation - DB_HOST: bduni_consult.ign.fr - DB_USER: null - DB_PASSWORD: null - DB_PORT: 5432 + db_uni: + db_name: bduni_france_consultation + db_host: bduni_consult.ign.fr + db_user: null + db_password: null + db_port: 5432 - BRANCH: - VORONOI_MAX_LENGTH: 2 # max size of a voronoi line - WATER_MIN_SIZE: 500 # min size of a skeleton line to be sure not to be removed (should be at least more than half the max river width) - MAX_GAP_CANDIDATES: 3 # max number of candidates to close a gap between 2 branches + branch: + voronoi_max_length: 2 # max size of a voronoi line + water_min_size: 500 # min size of a skeleton line to be sure not to be removed (should be at least more than half the max river width) + max_gap_candidates: 3 # max number of candidates to close a gap between 2 branches hydra: output_subdir: null diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index fbd4803b..399c7832 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -27,7 +27,7 @@ def run(config: DictConfig): - config (DictConfig): the config dict from hydra """ - gdf_hydro_global_mask = gpd.read_file(config.SKELETON.FILE_PATH.MASK_INPUT_PATH) + gdf_hydro_global_mask = gpd.read_file(config.skeleton.file_path.mask_input_path) crs = gdf_hydro_global_mask.crs # Load a crs from input branches_list = create_branches_list(config, gdf_hydro_global_mask, crs) @@ -37,8 +37,8 @@ def run(config: DictConfig): # create the gap lines from the selected candidates gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) - if config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH and validated_candidates: - gdf_gap_lines.to_file(config.SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH, driver='GeoJSON') + if config.skeleton.file_path.gap_lines_ouput_path and validated_candidates: + gdf_gap_lines.to_file(config.skeleton.file_path.gap_lines_ouput_path, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, # to create a new line toward that extremity and know not to remove it during the "simplify" @@ -55,13 +55,13 @@ def run(config: DictConfig): # putting all skeleton lines together, and save them if there is a path branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - if config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH: - gdf_branch_lines.to_file(config.SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH, driver='GeoJSON') + if config.skeleton.file_path.skeleton_lines_ouput_path: + gdf_branch_lines.to_file(config.skeleton.file_path.skeleton_lines_ouput_path, driver='GeoJSON') # saving all lines gdf_global_lines = gpd.GeoDataFrame(pd.concat([gdf_branch_lines, gdf_gap_lines], ignore_index=True)) gdf_global_lines = line_merge(gdf_global_lines, crs) # merge lines into polylines - gdf_global_lines.to_file(config.SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH, driver='GeoJSON') + gdf_global_lines.to_file(config.skeleton.file_path.global_lines_output_path, driver='GeoJSON') if __name__ == "__main__": diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index a0f3f806..b5f875f7 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -109,7 +109,7 @@ def distance_to_a_branch(self, other_branch: 'Branch') -> float: def get_candidates(self, other_branch: 'Branch') -> List[Candidate]: """ - Returns all the possible candidates (up to MAX_GAP_CANDIDATES) to close the gap with the other branch + Returns all the possible candidates (up to max_gap_candidates) to close the gap with the other branch Args: - other_branch (Branch): the other branch we want the distance to """ @@ -131,11 +131,11 @@ def get_candidates(self, other_branch: 'Branch') -> List[Candidate]: candidates = [] for index, (other_index, self_index) in enumerate(zip(indexes[0], indexes[1])): # stop if we have enough candidates - if index >= self.config.SKELETON.BRANCH.MAX_GAP_CANDIDATES: + if index >= self.config.skeleton.branch.max_gap_candidates: break # stop if the following candidates if distance_squared[other_index][self_index] \ - > self.config.SKELETON.MAX_GAP_WIDTH * self.config.SKELETON.MAX_GAP_WIDTH: + > self.config.skeleton.max_gap_width * self.config.skeleton.max_gap_width: break candidates.append( @@ -235,7 +235,7 @@ def can_line_be_removed(self, line: Geometry, vertices_dict: Dict[Point, List[Li - vertices_dict (Dict[Point, List[LineString]]) : a dictionary off all the lines with a point as an extremity """ - if line.length > self.config.SKELETON.BRANCH.WATER_MIN_SIZE: + if line.length > self.config.skeleton.branch.water_min_size: return False point_a, point_b = line.boundary.geoms[0], line.boundary.geoms[1] @@ -257,7 +257,7 @@ def create_voronoi_lines(self) -> GeoDataFrame: """ # divide geometry into segments no longer than max_segment_length united_geom = self.gdf_branch_mask['geometry'].unary_union - segmentize_geom = united_geom.segmentize(max_segment_length=self.config.SKELETON.BRANCH.VORONOI_MAX_LENGTH) + segmentize_geom = united_geom.segmentize(max_segment_length=self.config.skeleton.branch.voronoi_max_length) # Create the voronoi diagram and only keep polygon regions = voronoi_diagram(segmentize_geom, envelope=segmentize_geom, tolerance=0.0, edges=True) diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index c023dbe0..d6e6e4ef 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -22,11 +22,11 @@ def db_connector(config: DictConfig): - config (DictConfig): the config dict from hydra """ return psycopg.connect( - f"dbname={config.SKELETON.DB_UNI.DB_NAME} \ - host={config.SKELETON.DB_UNI.DB_HOST} \ - user={config.SKELETON.DB_UNI.DB_USER} \ - password={config.SKELETON.DB_UNI.DB_PASSWORD} \ - port={config.SKELETON.DB_UNI.DB_PORT}" + f"dbname={config.skeleton.db_uni.db_name} \ + host={config.skeleton.db_uni.db_host} \ + user={config.skeleton.db_uni.db_user} \ + password={config.skeleton.db_uni.db_password} \ + port={config.skeleton.db_uni.db_port}" ) @@ -42,10 +42,10 @@ def query_db_for_bridge_across_gap(config: DictConfig, candidate: Candidate) -> # bridge from another area) middle_x = (candidate.extremity_1[0] + candidate.extremity_2[0]) / 2 middle_y = (candidate.extremity_1[1] + candidate.extremity_2[1]) / 2 - new_ext_1_x = (candidate.extremity_1[0] - middle_x) * config.SKELETON.RATIO_GAP + middle_x - new_ext_1_y = (candidate.extremity_1[1] - middle_y) * config.SKELETON.RATIO_GAP + middle_y - new_ext_2_x = (candidate.extremity_2[0] - middle_x) * config.SKELETON.RATIO_GAP + middle_x - new_ext_2_y = (candidate.extremity_2[1] - middle_y) * config.SKELETON.RATIO_GAP + middle_y + new_ext_1_x = (candidate.extremity_1[0] - middle_x) * config.skeleton.ratio_gap + middle_x + new_ext_1_y = (candidate.extremity_1[1] - middle_y) * config.skeleton.ratio_gap + middle_y + new_ext_2_x = (candidate.extremity_2[0] - middle_x) * config.skeleton.ratio_gap + middle_x + new_ext_2_y = (candidate.extremity_2[1] - middle_y) * config.skeleton.ratio_gap + middle_y # creation of queries line = f"LINESTRING({new_ext_1_x} {new_ext_1_y}, {new_ext_2_x} {new_ext_2_y})" @@ -123,7 +123,7 @@ def select_candidates( # if the gap is wide enough, we check with DB_Uni to see if there is a bridge # On the other hand, if it's small enough the candidate is automatically validated - if candidate.squared_distance > config.SKELETON.GAP_WIDTH_CHECK_DB * config.SKELETON.GAP_WIDTH_CHECK_DB: + if candidate.squared_distance > config.skeleton.gap_width_check_db * config.skeleton.gap_width_check_db: is_bridge = query_db_for_bridge_across_gap(config, candidate) # if the line does not cross any bridge, we don't validate that candidate if not is_bridge: @@ -136,7 +136,7 @@ def select_candidates( # a candidate has been validated between A and B, so we put together A and B branch_group.put_together(branch_a, branch_b) nb_bridges_crossed += 1 - if nb_bridges_crossed >= config.SKELETON.MAX_BRIDGES: # max bridges reached between those 2 branches + if nb_bridges_crossed >= config.skeleton.max_bridges: # max bridges reached between those 2 branches break return validated_candidates @@ -191,6 +191,6 @@ def create_branches_pair(config: DictConfig, branches_list: List[Branch]) -> Lis for index, branch_a in enumerate(branches_list[:-1]): for branch_b in branches_list[index + 1:]: distance = branch_a.distance_to_a_branch(branch_b) - if distance < config.SKELETON.MAX_GAP_WIDTH: + if distance < config.skeleton.max_gap_width: branches_pair_list.append((branch_a, branch_b, distance)) return branches_pair_list diff --git a/scripts/example_create_skeleton_lines.sh b/scripts/example_create_skeleton_lines.sh index cb9d375e..61cd52cb 100644 --- a/scripts/example_create_skeleton_lines.sh +++ b/scripts/example_create_skeleton_lines.sh @@ -1,16 +1,16 @@ # For lauching create skeleton lines python lidro/main_create_skeleton_lines.py \ -SKELETON.FILE_PATH.MASK_INPUT_PATH=[input_filepath] \ -SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH=[output_filepath] \ -SKELETON.DB_UNI.DB_USER=[user_name] \ -"SKELETON.DB_UNI.DB_PASSWORD=[password]" \ -SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH=[out_filepath(optional)] \ -SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH=[out_filepath(optional)] +skeleton.file_path.mask_input_path=[input_filepath] \ +skeleton.file_path.global_lines_output_path=[output_filepath] \ +skeleton.db_uni.db_user=[user_name] \ +"skeleton.db_uni.db_password=[password]" \ +skeleton.file_path.skeleton_lines_ouput_path=[out_filepath(optional)] \ +skeleton.file_path.gap_lines_ouput_path=[out_filepath(optional)] -# SKELETON.FILE_PATH.MASK_INPUT_PATH : input path to the .geojson with the water masks -# SKELETON.FILE_PATH.GLOBAL_LINES_OUTPUT_PATH : output path for all the skeletons -# SKELETON.DB_UNI.DB_USER : username for "bduni_france_consultation" -# SKELETON.DB_UNI.DB_PASSWORD : password for "bduni_france_consultation". WARNING ! If there is a special character in the password, -# the line must be written like this : "SKELETON.DB_UNI.DB_PASSWORD='$tr@ng€_ch@r@ct€r$'" (note the " and the ') -# SKELETON.FILE_PATH.SKELETON_LINES_OUTPUT_PATH : (optional) output path for only the skeleton inside water beds defined by the input masks -# SKELETON.FILE_PATH.GAP_LINES_OUTPUT_PATH : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file +# skeleton.file_path.mask_input_path : input path to the .geojson with the water masks +# skeleton.file_path.global_lines_output_path : output path for all the skeletons +# skeleton.db_uni.db_user : username for "bduni_france_consultation" +# skeleton.db_uni.db_password : password for "bduni_france_consultation". WARNING ! If there is a special character in the password, +# the line must be written like this : "skeleton.db_uni.db_password='$tr@ng€_ch@r@ct€r$'" (note the " and the ') +# skeleton.file_path.skeleton_lines_ouput_path : (optional) output path for only the skeleton inside water beds defined by the input masks +# skeleton.file_path.gap_lines_ouput_path : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index da58f2fd..bedbfa3d 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -81,9 +81,8 @@ def test_creation_skeleton_lines(): config = compose( config_name="configs_lidro.yaml", overrides=[ - f"SKELETON.BRANCH.WATER_MIN_SIZE={WATER_MIN_SIZE_TEST}", + f"skeleton.branch.water_min_size={WATER_MIN_SIZE_TEST}", ] - ) branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index 9be7508b..02b482ce 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -56,8 +56,8 @@ def test_main_skeleton_2(): config = compose( config_name="configs_lidro.yaml", overrides=[ - f"SKELETON.DB_UNI.DB_USER={DB_UNI_USER}", - f'SKELETON.DB_UNI.DB_PASSWORD="{DB_UNI_PASSWORD}"', + f"skeleton.db_uni.db_user={DB_UNI_USER}", + f'skeleton.db_uni.db_password="{DB_UNI_PASSWORD}"', ], ) dummy_candidate_1 = Candidate(None, None, (687575.5, 6748540.586179815), (687594.5, 6748515.586065615), 0) From 8e9bda86a02ac17cb1a6fddf3e65d2c517b771cf Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 13 Aug 2024 13:32:51 +0200 Subject: [PATCH 53/68] changes to code from dev that have been wrongly merged --- lidro/main_create_mask.py | 3 ++- test/rasters/test_create_mask_raster.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lidro/main_create_mask.py b/lidro/main_create_mask.py index d4167265..6c4927f5 100644 --- a/lidro/main_create_mask.py +++ b/lidro/main_create_mask.py @@ -58,7 +58,7 @@ def main_on_one_tile(filename): input_file = os.path.join(input_dir, filename) # path to the LAS file output_file = os.path.join(output_dir, f"MaskHydro_{tilename}.GeoJSON") # path to the Mask Hydro file logging.info(f"\nCreate Mask Hydro 1 for tile : {tilename}") - create_hydro_vector_mask(input_file, output_file, pixel_size, tile_size, classe, crs, dilatation_size) + create_hydro_vector_mask(input_file, output_file, pixel_size, tile_size, classe, crs, dilation_size) if initial_las_filename: # Lauch creating mask by one tile: @@ -69,5 +69,6 @@ def main_on_one_tile(filename): for file in os.listdir(input_dir): main_on_one_tile(file) + if __name__ == "__main__": main() diff --git a/test/rasters/test_create_mask_raster.py b/test/rasters/test_create_mask_raster.py index 33865095..56bde175 100644 --- a/test/rasters/test_create_mask_raster.py +++ b/test/rasters/test_create_mask_raster.py @@ -43,7 +43,7 @@ def test_create_occupancy_map_default(): @pytest.mark.returnfile def test_detect_hydro_by_tile_return_file(): output_tif = TMP_PATH / "Semis_2021_0830_6291_LA93_IGN69_size.tif" - array, origin = detect_hydro_by_tile(LAS_FILE, tile_size, pixel_size, classes=[0, 1, 2, 3, 4, 5, 6, 17, 66], dilatation_size=1) + array, origin = detect_hydro_by_tile(LAS_FILE, tile_size, pixel_size, classes=[0, 1, 2, 3, 4, 5, 6, 17, 66], dilation_size=1) assert isinstance(array, np.ndarray) is True assert list(array.shape) == [tile_size / pixel_size] * 2 From 77d99a0ac9bd0d71816bb80eaa6ada9d7f3a7c54 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 13 Aug 2024 14:16:20 +0200 Subject: [PATCH 54/68] Update of the readme.md for the location of data --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4666d9c3..4382c68e 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ pre-commit install ``` ## Données de test -Les données de test se trouvent dans un autre projet ici : http://gitlab.forge-idi.ign.fr/Lidar/lidro-data +Les données de test se trouvent dans un autre projet ici : https://github.com/IGNF/lidro-data Ce projet est un sous module git, qui sera téléchargé dans le dossier `data`, via la commande: From 7e2c66debc9e0bb514cfc57dda2ef30717456997 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 13 Aug 2024 15:05:56 +0200 Subject: [PATCH 55/68] get the right submodule data --- data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data b/data index c9398e09..19f969cd 160000 --- a/data +++ b/data @@ -1 +1 @@ -Subproject commit c9398e09dde30bdb446044d6b4c65f0befdf6674 +Subproject commit 19f969cd0f938dcd5ca3d7019578fa74785c9934 From d9b9f7d6edbc2e944275eadde1817b48f27a313b Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 14 Aug 2024 10:59:49 +0200 Subject: [PATCH 56/68] removing "ci" directory, that has no business being here --- ci/test.sh | 1 - 1 file changed, 1 deletion(-) delete mode 100644 ci/test.sh diff --git a/ci/test.sh b/ci/test.sh deleted file mode 100644 index 823d3d84..00000000 --- a/ci/test.sh +++ /dev/null @@ -1 +0,0 @@ -python -m pytest -s ./test -v \ No newline at end of file From 58c64fcf25dc963db70e0fdf87efd2630cfdcda1 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 14 Aug 2024 11:26:46 +0200 Subject: [PATCH 57/68] changing test name to be more explicit --- test/skeleton/test_main_skeleton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/skeleton/test_main_skeleton.py b/test/skeleton/test_main_skeleton.py index 02b482ce..7d47f35f 100644 --- a/test/skeleton/test_main_skeleton.py +++ b/test/skeleton/test_main_skeleton.py @@ -50,7 +50,7 @@ def test_main_skeleton_1(): # do that test only if we are not on github action, since github can't connect to BD UNI @pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="BD UNI not reachable from github action") -def test_main_skeleton_2(): +def test_query_db_for_bridge_across_gap(): """Test : query_db_for_bridge_across_gap """ with initialize(version_base="1.2", config_path="../../configs"): config = compose( From 04521c3f52ab29b01186fcd1fa7a3439626bd111 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 14 Aug 2024 11:52:19 +0200 Subject: [PATCH 58/68] correction of a modification on test_close_holes_default that shouldn't be here --- test/vectors/test_close_holes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/vectors/test_close_holes.py b/test/vectors/test_close_holes.py index b77ebb4d..1376cc04 100644 --- a/test/vectors/test_close_holes.py +++ b/test/vectors/test_close_holes.py @@ -9,7 +9,7 @@ def test_close_holes_default(): # Load each GeoJSON file as GeoDataFrame geojson = gpd.read_file(input) - geometry = geojson["geometry"][0] + geometry = geojson["geometry"][1] # with contain severals holes mask_without_hole = close_holes(geometry, 100) assert isinstance(mask_without_hole, Polygon) From 34684de89545819656ef404da1be6600555f2f18 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 19 Aug 2024 10:20:36 +0200 Subject: [PATCH 59/68] add a test to branch --- lidro/skeleton/branch.py | 6 ++-- lidro/skeleton/create_skeleton_lines.py | 2 ++ test/skeleton/test_branch.py | 40 ++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lidro/skeleton/branch.py b/lidro/skeleton/branch.py index b5f875f7..59fda867 100644 --- a/lidro/skeleton/branch.py +++ b/lidro/skeleton/branch.py @@ -9,7 +9,6 @@ import numpy as np from shapely import LineString, Point, Geometry from shapely.geometry import Polygon, MultiLineString -from shapely.validation import make_valid from shapely.ops import voronoi_diagram, linemerge from pyproj.crs.crs import CRS @@ -62,7 +61,8 @@ def set_gdf_branch_mask(self, branch_mask: GeoDataFrame): raw_gdf_branch_mask = gpd.GeoDataFrame(geometry=[branch_mask], crs=self.crs) # # we keep only the exterior ring (that should be unique) as our polygon, to simplify the result - self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[Polygon(raw_gdf_branch_mask.exterior[0].coords)], crs=self.crs) + exterior_ring = Polygon(raw_gdf_branch_mask.exterior[0].coords) + self.gdf_branch_mask = gpd.GeoDataFrame(geometry=[exterior_ring], crs=self.crs) def create_skeleton(self): """ @@ -70,7 +70,7 @@ def create_skeleton(self): """ voronoi_lines = self.create_voronoi_lines() - if len(voronoi_lines) == 0: + if len(voronoi_lines) == 0: self.gdf_skeleton_lines = gpd.GeoDataFrame(geometry=[], crs=self.crs) return diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index d6e6e4ef..3e8276d8 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -140,6 +140,7 @@ def select_candidates( break return validated_candidates + def fix_invalid_geometry(geometry): """ return the geometry, fixed @@ -151,6 +152,7 @@ def fix_invalid_geometry(geometry): else: return geometry + def create_branches_list(config: DictConfig, gdf_hydro_global_mask: GeoDataFrame, crs: CRS) -> List[Branch]: """ create the list of branches from the global mask diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index bedbfa3d..444cd611 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -3,7 +3,7 @@ from hydra import compose, initialize import geopandas as gpd from omegaconf import DictConfig -from shapely import LineString +from shapely import LineString, Point from lidro.skeleton.branch import Branch, get_vertices_dict, get_df_points_from_gdf, line_merge @@ -98,3 +98,41 @@ def test_creation_skeleton_lines(): extremities_cpt += 1 assert extremities_cpt == 3 # check that this branch's skeleton has exactly 3 extremities + + +def test_get_vertices_dict(): + point_0_0 = [0, 0] + point_0_1 = [0, 1] + point_0_2 = [0, 2] + point_0_3 = [0, 3] + point_0_4 = [0, 4] + point_0_5 = [0, 5] + point_0_6 = [0, 6] + point_1_4 = [1, 4] + point_2_4 = [2, 4] + + line_1 = LineString([point_0_0, point_0_1, point_0_2]) + line_2 = LineString([point_0_2, point_0_3, point_0_4]) + line_3 = LineString([point_0_4, point_0_5]) + line_4 = LineString([point_0_5, point_0_6]) + line_5 = LineString([point_0_4, point_1_4]) + line_6 = LineString([point_1_4, point_2_4]) + + line_list = [line_1, + line_2, + line_3, + line_4, + line_5, + line_6 + ] + + gdf_gap_lines = gpd.GeoDataFrame(geometry=line_list).set_crs(CRS_FOR_TEST, allow_override=True) + + vertices_dict = get_vertices_dict(gdf_gap_lines) + + assert len(vertices_dict) == 7 + assert vertices_dict[Point(point_0_0)] == [line_1] + assert vertices_dict[Point(point_0_6)] == [line_4] + assert vertices_dict[Point(point_2_4)] == [line_6] + assert len(vertices_dict[Point(point_0_2)]) == 2 + assert len(vertices_dict[Point(point_0_4)]) == 3 From ac91fa196c8d718a7af3b450f9bc3b27a223f607 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Mon, 19 Aug 2024 10:42:32 +0200 Subject: [PATCH 60/68] revert some changes that a misuse of git created --- .../rasters/create_mask_raster.py | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/lidro/create_mask_hydro/rasters/create_mask_raster.py b/lidro/create_mask_hydro/rasters/create_mask_raster.py index 1bf269f6..8d31097f 100644 --- a/lidro/create_mask_hydro/rasters/create_mask_raster.py +++ b/lidro/create_mask_hydro/rasters/create_mask_raster.py @@ -35,20 +35,21 @@ def create_occupancy_map(points: np.array, tile_size: int, pixel_size: float, or def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, classes: List[int], dilation_size: int): """ "Detect hydrographic surfaces in a tile from the classified points of the input pointcloud - An hydrographic surface is define as a surface where there is no points from any class different from water -The output hydrographic surface mask is dilated to make sure that the masks are continuous when merged with their neighbours - Args: - filename (str): input pointcloud - tile_size (int): size of the raster grid (in meters) - pixel_size (float): distance between each node of the raster grid (in meters) - classes (List[int]): List of classes to use for the binarisation (points with other - classification values are ignored) - dilation_size (int): size for dilatation raster + An hydrographic surface is defined as a surface where there is no points from any class different from water + The output hydrographic surface mask is dilated to make sure that the masks are continuous when merged with their + neighbours + Args: + filename (str): input pointcloud + tile_size (int): size of the raster grid (in meters) + pixel_size (float): distance between each node of the raster grid (in meters) + classes (List[int]): List of classes to use for the binarisation (points with other + classification values are ignored) + dilation_size (int): size of the structuring element for dilation - Returns: - smoothed_water (np.array): 2D binary array (x, y) of the water presence from the point cloud - pcd_origin (list): top left corner of the tile containing the point cloud - (infered from pointcloud bounding box and input tile size) + Returns: + smoothed_water (np.array): 2D binary array (x, y) of the water presence from the point cloud + pcd_origin (list): top left corner of the tile containing the point cloud + (infered from pointcloud bounding box and input tile size) """ # Read pointcloud, and extract coordinates (X, Y, Z, and classification) of all points array, crs = read_pointcloud(filename) @@ -68,6 +69,8 @@ def detect_hydro_by_tile(filename: str, tile_size: int, pixel_size: float, class # Apply a mathematical morphology operations: DILATION # / ! \ NOT "CLOSING", due to the reduction in the size of hydro masks (tile borders) # / ! \ WITH "CLOSING" => Masks Hydro are no longer continuous, when they are merged - water_mask = scipy.ndimage.binary_dilation(detected_water, structure=np.ones((dilation_size, dilation_size))).astype(np.uint8) + water_mask = scipy.ndimage.binary_dilation( + detected_water, structure=np.ones((dilation_size, dilation_size)) + ).astype(np.uint8) - return water_mask, pcd_origin + return water_mask, pcd_origin \ No newline at end of file From 89e7085ba8a448318e8559585681dc3b9d233db2 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 20 Aug 2024 07:11:18 +0200 Subject: [PATCH 61/68] fix a typo --- README.md | 4 ++-- configs/configs_lidro.yaml | 4 ++-- lidro/main_create_skeleton_lines.py | 8 ++++---- scripts/example_create_skeleton_lines.sh | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4382c68e..f72b5d8b 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,8 @@ python lidro/main_create_skeleton_lines.py [nom_paramètre_1]=[valeur_du_paramè ``` ces paramètres sont : skeleton.file_path.mask_input_path : Le chemin d'entrée des masques des cours d'eau -skeleton.file_path.skeleton_lines_ouput_path : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) -skeleton.file_path.gap_lines_ouput_path : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) +skeleton.file_path.skeleton_lines_output_path : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) +skeleton.file_path.gap_lines_output_path : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) skeleton.file_path.global_lines_output_path : Le chemin de sortie des lignes et des squelettes ensemble skeleton.max_gap_width : La distance maximale envisagée pour franchir des ponts diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index 545beaaa..dc8f5d23 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -50,8 +50,8 @@ virtual_point: skeleton: file_path: mask_input_path: null - skeleton_lines_ouput_path: null # only branches' skeleton (if empty, no file is created) - gap_lines_ouput_path: null # only new lines to close gaps (if empty, no file is created) + skeleton_lines_output_path: null # only branches' skeleton (if empty, no file is created) + gap_lines_output_path: null # only new lines to close gaps (if empty, no file is created) global_lines_output_path: null # lines from both the skeletons and the gap lines max_gap_width: 200 # distance max in meter of any gap between 2 branches we will try to close with a line diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 399c7832..023c4aa3 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -37,8 +37,8 @@ def run(config: DictConfig): # create the gap lines from the selected candidates gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) - if config.skeleton.file_path.gap_lines_ouput_path and validated_candidates: - gdf_gap_lines.to_file(config.skeleton.file_path.gap_lines_ouput_path, driver='GeoJSON') + if config.skeleton.file_path.gap_lines_output_path and validated_candidates: + gdf_gap_lines.to_file(config.skeleton.file_path.gap_lines_output_path, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, # to create a new line toward that extremity and know not to remove it during the "simplify" @@ -55,8 +55,8 @@ def run(config: DictConfig): # putting all skeleton lines together, and save them if there is a path branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - if config.skeleton.file_path.skeleton_lines_ouput_path: - gdf_branch_lines.to_file(config.skeleton.file_path.skeleton_lines_ouput_path, driver='GeoJSON') + if config.skeleton.file_path.skeleton_lines_output_path: + gdf_branch_lines.to_file(config.skeleton.file_path.skeleton_lines_output_path, driver='GeoJSON') # saving all lines gdf_global_lines = gpd.GeoDataFrame(pd.concat([gdf_branch_lines, gdf_gap_lines], ignore_index=True)) diff --git a/scripts/example_create_skeleton_lines.sh b/scripts/example_create_skeleton_lines.sh index 61cd52cb..10fc5848 100644 --- a/scripts/example_create_skeleton_lines.sh +++ b/scripts/example_create_skeleton_lines.sh @@ -4,13 +4,13 @@ skeleton.file_path.mask_input_path=[input_filepath] \ skeleton.file_path.global_lines_output_path=[output_filepath] \ skeleton.db_uni.db_user=[user_name] \ "skeleton.db_uni.db_password=[password]" \ -skeleton.file_path.skeleton_lines_ouput_path=[out_filepath(optional)] \ -skeleton.file_path.gap_lines_ouput_path=[out_filepath(optional)] +skeleton.file_path.skeleton_lines_output_path=[out_filepath(optional)] \ +skeleton.file_path.gap_lines_output_path=[out_filepath(optional)] # skeleton.file_path.mask_input_path : input path to the .geojson with the water masks # skeleton.file_path.global_lines_output_path : output path for all the skeletons # skeleton.db_uni.db_user : username for "bduni_france_consultation" # skeleton.db_uni.db_password : password for "bduni_france_consultation". WARNING ! If there is a special character in the password, # the line must be written like this : "skeleton.db_uni.db_password='$tr@ng€_ch@r@ct€r$'" (note the " and the ') -# skeleton.file_path.skeleton_lines_ouput_path : (optional) output path for only the skeleton inside water beds defined by the input masks -# skeleton.file_path.gap_lines_ouput_path : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file +# skeleton.file_path.skeleton_lines_output_path : (optional) output path for only the skeleton inside water beds defined by the input masks +# skeleton.file_path.gap_lines_output_path : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file From 86e40508d9b44fa2852f64a0ac86d26255489671 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 20 Aug 2024 08:58:23 +0200 Subject: [PATCH 62/68] fix a previous unwanted change --- lidro/merge_mask_hydro/vectors/close_holes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lidro/merge_mask_hydro/vectors/close_holes.py b/lidro/merge_mask_hydro/vectors/close_holes.py index 4f4ad0e4..d28d95d7 100644 --- a/lidro/merge_mask_hydro/vectors/close_holes.py +++ b/lidro/merge_mask_hydro/vectors/close_holes.py @@ -4,7 +4,7 @@ from shapely.ops import unary_union -def close_holes(polygon: Polygon, min_hole_area): +def close_holes(polygon: Polygon, min_hole_area)-> Polygon: """Remove small holes (surface < 100 m²) Args: @@ -26,4 +26,4 @@ def close_holes(polygon: Polygon, min_hole_area): shell=polygon.exterior, holes=interior_holes_filter) - return final_polygon + return final_polygon \ No newline at end of file From ef1aa3ea4a2a3d5ed8b1f73bfbd4cadaa5255ca5 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 20 Aug 2024 09:01:12 +0200 Subject: [PATCH 63/68] fix another unwanted change --- test/rasters/test_create_mask_raster.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/rasters/test_create_mask_raster.py b/test/rasters/test_create_mask_raster.py index 56bde175..541e1216 100644 --- a/test/rasters/test_create_mask_raster.py +++ b/test/rasters/test_create_mask_raster.py @@ -7,7 +7,10 @@ import rasterio from rasterio.transform import from_origin -from lidro.create_mask_hydro.rasters.create_mask_raster import create_occupancy_map, detect_hydro_by_tile +from lidro.create_mask_hydro.rasters.create_mask_raster import ( + create_occupancy_map, + detect_hydro_by_tile, +) TMP_PATH = Path("./tmp/create_mask_hydro/rasters/create_mask_raster") @@ -43,7 +46,9 @@ def test_create_occupancy_map_default(): @pytest.mark.returnfile def test_detect_hydro_by_tile_return_file(): output_tif = TMP_PATH / "Semis_2021_0830_6291_LA93_IGN69_size.tif" - array, origin = detect_hydro_by_tile(LAS_FILE, tile_size, pixel_size, classes=[0, 1, 2, 3, 4, 5, 6, 17, 66], dilation_size=1) + array, origin = detect_hydro_by_tile( + LAS_FILE, tile_size, pixel_size, classes=[0, 1, 2, 3, 4, 5, 6, 17, 66], dilation_size=1 + ) assert isinstance(array, np.ndarray) is True assert list(array.shape) == [tile_size / pixel_size] * 2 @@ -67,4 +72,4 @@ def test_detect_hydro_by_tile_return_file(): ) as dst: dst.write(array, 1) - assert Path(output_tif).exists() + assert Path(output_tif).exists() \ No newline at end of file From f94c7eea18912b648040066284f392d549715d85 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 21 Aug 2024 10:56:59 +0200 Subject: [PATCH 64/68] changes so all submodules can be run with python [modulefilename] and not only with python -m [modulefilename] --- lidro/main_create_mask.py | 3 +++ lidro/main_create_virtual_point.py | 3 +++ lidro/main_merge_mask.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/lidro/main_create_mask.py b/lidro/main_create_mask.py index 6c4927f5..260f30ba 100644 --- a/lidro/main_create_mask.py +++ b/lidro/main_create_mask.py @@ -3,11 +3,14 @@ import logging import os +import sys import hydra from omegaconf import DictConfig from pyproj import CRS +sys.path.append('../lidro') + from lidro.create_mask_hydro.vectors.convert_to_vector import create_hydro_vector_mask diff --git a/lidro/main_create_virtual_point.py b/lidro/main_create_virtual_point.py index fc610166..2ead2845 100644 --- a/lidro/main_create_virtual_point.py +++ b/lidro/main_create_virtual_point.py @@ -3,6 +3,7 @@ import logging import os +import sys import geopandas as gpd import hydra @@ -10,6 +11,8 @@ from omegaconf import DictConfig from pyproj import CRS +sys.path.append('../lidro') + from lidro.create_virtual_point.vectors.extract_points_around_skeleton import ( extract_points_around_skeleton_points_one_tile, ) diff --git a/lidro/main_merge_mask.py b/lidro/main_merge_mask.py index 49854413..ab95d952 100644 --- a/lidro/main_merge_mask.py +++ b/lidro/main_merge_mask.py @@ -3,11 +3,14 @@ import logging import os +import sys import hydra from omegaconf import DictConfig from pyproj import CRS +sys.path.append('../lidro') + from lidro.merge_mask_hydro.vectors.merge_vector import merge_geom From d977f10be2932ee0db07948494ad03a47e705b1f Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 21 Aug 2024 11:23:21 +0200 Subject: [PATCH 65/68] moving input/ouptut files from skeleton.filepth to io.skeleton in configs_lidro.yaml --- README.md | 8 ++++---- configs/configs_lidro.yaml | 12 ++++++------ lidro/main_create_skeleton_lines.py | 12 ++++++------ scripts/example_create_skeleton_lines.sh | 16 ++++++++-------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index f72b5d8b..b9818717 100644 --- a/README.md +++ b/README.md @@ -114,10 +114,10 @@ Pour fonctionner, la création de squelettes a besoin d'une série de paramètre python lidro/main_create_skeleton_lines.py [nom_paramètre_1]=[valeur_du_paramètre_1] [nom_paramètre_2]=[valeur_du_paramètre_2] ``` ces paramètres sont : -skeleton.file_path.mask_input_path : Le chemin d'entrée des masques des cours d'eau -skeleton.file_path.skeleton_lines_output_path : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) -skeleton.file_path.gap_lines_output_path : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) -skeleton.file_path.global_lines_output_path : Le chemin de sortie des lignes et des squelettes ensemble +io.skeleton.mask_input_path : Le chemin d'entrée des masques des cours d'eau +io.skeleton.skeleton_lines_output_path : Le chemin de sortie des squelettes uniquement (pas de fichier de sortie si laissé à vide) +io.skeleton.gap_lines_output_path : Le chemin de sortie des lignes franchissant des ponts uniquement (pas de fichier de sortie si laissé à vide) +io.skeleton.global_lines_output_path : Le chemin de sortie des lignes et des squelettes ensemble skeleton.max_gap_width : La distance maximale envisagée pour franchir des ponts skeleton.max_bridges : Le nombre maximal de ponts entre deux bras séparés de cours d'eau diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index dc8f5d23..e0d109c3 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -19,6 +19,12 @@ io: no_data_value: -9999 tile_size: 1000 + skeleton: + mask_input_path: null + skeleton_lines_output_path: null # only branches' skeleton (if empty, no file is created) + gap_lines_output_path: null # only new lines to close gaps (if empty, no file is created) + global_lines_output_path: null # lines from both the skeletons and the gap lines + mask_generation: raster: # size for dilatation @@ -48,12 +54,6 @@ virtual_point: k: 20 skeleton: - file_path: - mask_input_path: null - skeleton_lines_output_path: null # only branches' skeleton (if empty, no file is created) - gap_lines_output_path: null # only new lines to close gaps (if empty, no file is created) - global_lines_output_path: null # lines from both the skeletons and the gap lines - max_gap_width: 200 # distance max in meter of any gap between 2 branches we will try to close with a line max_bridges: 1 # number max of bridges that can be created between 2 branches gap_width_check_db: 20 # for a gap at least this wide, we check with db_uni if there is a bridge. Smaller than that, we assume a line automatically exists diff --git a/lidro/main_create_skeleton_lines.py b/lidro/main_create_skeleton_lines.py index 023c4aa3..85fd203d 100644 --- a/lidro/main_create_skeleton_lines.py +++ b/lidro/main_create_skeleton_lines.py @@ -27,7 +27,7 @@ def run(config: DictConfig): - config (DictConfig): the config dict from hydra """ - gdf_hydro_global_mask = gpd.read_file(config.skeleton.file_path.mask_input_path) + gdf_hydro_global_mask = gpd.read_file(config.io.skeleton.mask_input_path) crs = gdf_hydro_global_mask.crs # Load a crs from input branches_list = create_branches_list(config, gdf_hydro_global_mask, crs) @@ -37,8 +37,8 @@ def run(config: DictConfig): # create the gap lines from the selected candidates gap_lines_list = [validated_candidate.line for validated_candidate in validated_candidates] gdf_gap_lines = gpd.GeoDataFrame(geometry=gap_lines_list).set_crs(crs, allow_override=True) - if config.skeleton.file_path.gap_lines_output_path and validated_candidates: - gdf_gap_lines.to_file(config.skeleton.file_path.gap_lines_output_path, driver='GeoJSON') + if config.io.skeleton.gap_lines_output_path and validated_candidates: + gdf_gap_lines.to_file(config.io.skeleton.gap_lines_output_path, driver='GeoJSON') # add the extremities used on each branch to close a gap to the list of gap_point of that branch, # to create a new line toward that extremity and know not to remove it during the "simplify" @@ -55,13 +55,13 @@ def run(config: DictConfig): # putting all skeleton lines together, and save them if there is a path branch_lines_list = [branch.gdf_skeleton_lines for branch in branches_list] gdf_branch_lines = gpd.GeoDataFrame(pd.concat(branch_lines_list, ignore_index=True)) - if config.skeleton.file_path.skeleton_lines_output_path: - gdf_branch_lines.to_file(config.skeleton.file_path.skeleton_lines_output_path, driver='GeoJSON') + if config.io.skeleton.skeleton_lines_output_path: + gdf_branch_lines.to_file(config.io.skeleton.skeleton_lines_output_path, driver='GeoJSON') # saving all lines gdf_global_lines = gpd.GeoDataFrame(pd.concat([gdf_branch_lines, gdf_gap_lines], ignore_index=True)) gdf_global_lines = line_merge(gdf_global_lines, crs) # merge lines into polylines - gdf_global_lines.to_file(config.skeleton.file_path.global_lines_output_path, driver='GeoJSON') + gdf_global_lines.to_file(config.io.skeleton.global_lines_output_path, driver='GeoJSON') if __name__ == "__main__": diff --git a/scripts/example_create_skeleton_lines.sh b/scripts/example_create_skeleton_lines.sh index 10fc5848..e7205046 100644 --- a/scripts/example_create_skeleton_lines.sh +++ b/scripts/example_create_skeleton_lines.sh @@ -1,16 +1,16 @@ # For lauching create skeleton lines python lidro/main_create_skeleton_lines.py \ -skeleton.file_path.mask_input_path=[input_filepath] \ -skeleton.file_path.global_lines_output_path=[output_filepath] \ +io.skeleton.mask_input_path=[input_filepath] \ +io.skeleton.global_lines_output_path=[output_filepath] \ skeleton.db_uni.db_user=[user_name] \ "skeleton.db_uni.db_password=[password]" \ -skeleton.file_path.skeleton_lines_output_path=[out_filepath(optional)] \ -skeleton.file_path.gap_lines_output_path=[out_filepath(optional)] +io.skeleton.skeleton_lines_output_path=[out_filepath(optional)] \ +io.skeleton.gap_lines_output_path=[out_filepath(optional)] -# skeleton.file_path.mask_input_path : input path to the .geojson with the water masks -# skeleton.file_path.global_lines_output_path : output path for all the skeletons +# io.skeleton.mask_input_path : input path to the .geojson with the water masks +# io.skeleton.global_lines_output_path : output path for all the skeletons # skeleton.db_uni.db_user : username for "bduni_france_consultation" # skeleton.db_uni.db_password : password for "bduni_france_consultation". WARNING ! If there is a special character in the password, # the line must be written like this : "skeleton.db_uni.db_password='$tr@ng€_ch@r@ct€r$'" (note the " and the ') -# skeleton.file_path.skeleton_lines_output_path : (optional) output path for only the skeleton inside water beds defined by the input masks -# skeleton.file_path.gap_lines_output_path : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file +# io.skeleton.skeleton_lines_output_path : (optional) output path for only the skeleton inside water beds defined by the input masks +# io.skeleton.gap_lines_output_path : (optional) output path for only the lines between water beds defined by the input masks \ No newline at end of file From a0870d014a2e6c026cfb334dc5af977de57c3d3d Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 21 Aug 2024 14:41:52 +0200 Subject: [PATCH 66/68] Add the possibility to disable db connection --- README.md | 1 + configs/configs_lidro.yaml | 1 + lidro/skeleton/create_skeleton_lines.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b9818717..4be22bb8 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ skeleton.max_bridges : Le nombre maximal de ponts entre deux bras séparés de c skeleton.gap_width_check_db : La distance à partir de laquelle on vérifie via la base de données s'il y a bien un pont skeleton.ratio_gap : la proportion de la ligne franchissant un pont qui est comparé en base pour voir s'il y a bien un pont (trop grande et on pourrait trouver un pont qui ne correspond pas) +skeleton.db_uni.db_using_db : # Si à faux, la base de données ne sera pas utilisée skeleton.db_uni.db_name : Le nom de la base de données skeleton.db_uni.db_host : l'adresse de la base de données skeleton.db_uni.db_user : L'utilisateur de la base de données diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index e0d109c3..ad977256 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -62,6 +62,7 @@ skeleton: # so we make the line smaller db_uni: + db_using_db: True # if false, the DB won't be queried db_name: bduni_france_consultation db_host: bduni_consult.ign.fr db_user: null diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index 3e8276d8..c67cbe7c 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -123,7 +123,8 @@ def select_candidates( # if the gap is wide enough, we check with DB_Uni to see if there is a bridge # On the other hand, if it's small enough the candidate is automatically validated - if candidate.squared_distance > config.skeleton.gap_width_check_db * config.skeleton.gap_width_check_db: + if config.skeleton.db_uni.db_using_db and \ + candidate.squared_distance > config.skeleton.gap_width_check_db * config.skeleton.gap_width_check_db: is_bridge = query_db_for_bridge_across_gap(config, candidate) # if the line does not cross any bridge, we don't validate that candidate if not is_bridge: From 109f43f93f8441afbed6ecbfbbd3e5612b262a89 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Wed, 21 Aug 2024 15:26:54 +0200 Subject: [PATCH 67/68] Add another test to branch --- lidro/skeleton/create_skeleton_lines.py | 4 +- test/skeleton/test_branch.py | 75 ++++++++++++++++--------- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/lidro/skeleton/create_skeleton_lines.py b/lidro/skeleton/create_skeleton_lines.py index c67cbe7c..6b72d951 100644 --- a/lidro/skeleton/create_skeleton_lines.py +++ b/lidro/skeleton/create_skeleton_lines.py @@ -123,8 +123,10 @@ def select_candidates( # if the gap is wide enough, we check with DB_Uni to see if there is a bridge # On the other hand, if it's small enough the candidate is automatically validated + # All that, ONLy if we want to interrogate the DB if config.skeleton.db_uni.db_using_db and \ - candidate.squared_distance > config.skeleton.gap_width_check_db * config.skeleton.gap_width_check_db: + candidate.squared_distance \ + > config.skeleton.gap_width_check_db * config.skeleton.gap_width_check_db: is_bridge = query_db_for_bridge_across_gap(config, candidate) # if the line does not cross any bridge, we don't validate that candidate if not is_bridge: diff --git a/test/skeleton/test_branch.py b/test/skeleton/test_branch.py index 444cd611..967b8998 100644 --- a/test/skeleton/test_branch.py +++ b/test/skeleton/test_branch.py @@ -10,7 +10,7 @@ sys.path.append('lidro/skeleton') BRANCH_TEST_1_PATH = "data/skeleton_hydro/test_files/90.geojson" -CRS_FOR_TEST = 2145 +CRS_FOR_TEST = 2154 WATER_MIN_SIZE_TEST = 20 @@ -75,31 +75,6 @@ def test_line_merge(): assert point_0_4 in line.coords -def test_creation_skeleton_lines(): - """test creation/simplification of skeleton lines for a branch""" - with initialize(version_base="1.2", config_path="../../configs"): - config = compose( - config_name="configs_lidro.yaml", - overrides=[ - f"skeleton.branch.water_min_size={WATER_MIN_SIZE_TEST}", - ] - ) - branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") - - # create branch_1's skeleton - branch_1.create_skeleton() - branch_1.simplify() - - # number of extremity of branch_1's skeleton - vertices_dict = get_vertices_dict(branch_1.gdf_skeleton_lines) - extremities_cpt = 0 - for lines_list in vertices_dict.values(): - if len(lines_list) == 1: - extremities_cpt += 1 - - assert extremities_cpt == 3 # check that this branch's skeleton has exactly 3 extremities - - def test_get_vertices_dict(): point_0_0 = [0, 0] point_0_1 = [0, 1] @@ -136,3 +111,51 @@ def test_get_vertices_dict(): assert vertices_dict[Point(point_2_4)] == [line_6] assert len(vertices_dict[Point(point_0_2)]) == 2 assert len(vertices_dict[Point(point_0_4)]) == 3 + + +def test_create_voronoi_lines(): + with initialize(version_base="1.2", config_path="../../configs"): + config = compose( + config_name="configs_lidro.yaml", + overrides=[ + f"skeleton.branch.water_min_size={WATER_MIN_SIZE_TEST}", + ] + ) + branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") + voronoi_lines = branch_1.create_voronoi_lines() + assert len(voronoi_lines) == 1028 + + voronoi_merged = line_merge(voronoi_lines, CRS_FOR_TEST) + assert len(voronoi_merged) == 157 + vertices_dict = get_vertices_dict(voronoi_merged) + # count number of extremities : + nb_extremities = 0 + for lines_list in vertices_dict.values(): + if len(lines_list) == 1: + nb_extremities += 1 + assert nb_extremities == 80 + + +def test_creation_skeleton_lines(): + """test creation/simplification of skeleton lines for a branch""" + with initialize(version_base="1.2", config_path="../../configs"): + config = compose( + config_name="configs_lidro.yaml", + overrides=[ + f"skeleton.branch.water_min_size={WATER_MIN_SIZE_TEST}", + ] + ) + branch_1 = read_branch(config, BRANCH_TEST_1_PATH, "test_branch_1") + + # create branch_1's skeleton + branch_1.create_skeleton() + branch_1.simplify() + + # number of extremity of branch_1's skeleton + vertices_dict = get_vertices_dict(branch_1.gdf_skeleton_lines) + extremities_cpt = 0 + for lines_list in vertices_dict.values(): + if len(lines_list) == 1: + extremities_cpt += 1 + + assert extremities_cpt == 3 # check that this branch's skeleton has exactly 3 extremities From 1130aef980b8ad74703a37d59807dcb6f506ee50 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Thu, 22 Aug 2024 07:52:28 +0200 Subject: [PATCH 68/68] add small comment about db_using_db --- README.md | 2 +- configs/configs_lidro.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4be22bb8..7440d383 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ skeleton.max_bridges : Le nombre maximal de ponts entre deux bras séparés de c skeleton.gap_width_check_db : La distance à partir de laquelle on vérifie via la base de données s'il y a bien un pont skeleton.ratio_gap : la proportion de la ligne franchissant un pont qui est comparé en base pour voir s'il y a bien un pont (trop grande et on pourrait trouver un pont qui ne correspond pas) -skeleton.db_uni.db_using_db : # Si à faux, la base de données ne sera pas utilisée +skeleton.db_uni.db_using_db : # Si à faux, la base de données ne sera pas utilisée (prévu pour être utilisé que s'il n'y pas d'accès à la base de données) skeleton.db_uni.db_name : Le nom de la base de données skeleton.db_uni.db_host : l'adresse de la base de données skeleton.db_uni.db_user : L'utilisateur de la base de données diff --git a/configs/configs_lidro.yaml b/configs/configs_lidro.yaml index ad977256..4f98e4b3 100644 --- a/configs/configs_lidro.yaml +++ b/configs/configs_lidro.yaml @@ -62,7 +62,7 @@ skeleton: # so we make the line smaller db_uni: - db_using_db: True # if false, the DB won't be queried + db_using_db: True # if false, the DB won't be queried (expected to be used only in case you don't have access to the database) db_name: bduni_france_consultation db_host: bduni_consult.ign.fr db_user: null