Merge pull request #5 from swiss-territorial-data-lab/ch/dev_tm

Ch/dev tm
swiss-territorial-data-lab · Jan 24, 2024 · 7881d34 · 7881d34
2 parents acfda2c + e228a1b
commit 7881d34
Show file tree

Hide file tree

Showing 67 changed files with 1,414 additions and 145,263 deletions.
diff --git a/.github/workflows/analyze.yml b/.github/workflows/analyze.yml
@@ -2,8 +2,8 @@ name: Analyze
 
 on:
   push:
-    branches:
-      - '*'
+    # branches:
+    #   - '*'
   pull_request:
     types: [opened, synchronize, reopened]
 

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,5 @@
-input/input_det/*
-input/input_dm/*
-input/input-prd/logs/*
-input/input-prd/DEM/*
+input/*
+!intput/input_det/AoI
+!input/input_trne/
 output/*
-tools/*
+config/config_det_*
diff --git a/README.md b/README.md
diff --git a/config/config-prd.yaml b/config/config-prd.yaml
diff --git a/config/config-trne.yaml b/config/config-trne.yaml
diff --git a/config/config_det.template.yaml b/config/config_det.template.yaml
@@ -0,0 +1,67 @@
+###################################
+##### Detection by inference #####
+#######      template      ####### 
+# Automatic detection of Quarries and Mineral Extraction Sites (MES) in images
+
+# 1-Prepare the tiles geometry according to the AOI and zoom level
+prepare_data.py: 
+  srs: "EPSG:2056"  # Projection of the input file
+  datasets:
+    shapefile: ./input/input_det/AoI/swissimage_footprint_SWISSIMAGE_YEAR.shp
+  output_folder: ./output/output_det/SWISSIMAGE_YEAR
+  zoom_level: 16  # z, keep between 15 and 18  
+
+# 2-Request tiles according to the provided AOI and tiles parameters
+generate_tilesets.py:
+  debug_mode: False
+  datasets:
+    aoi_tiles_geojson: ./output/output_det/SWISSIMAGE_YEAR/tiles.geojson
+    orthophotos_web_service:
+      type: XYZ # supported values: 1. MIL = Map Image Layer 2. WMS 3. XYZ
+      url: https://wmts.geo.admin.ch/1.0.0/ch.swisstopo.swissimage-product/default/SWISSIMAGE_YEAR/3857/{z}/{x}/{y}.jpeg
+  output_folder: ./output/output_det/SWISSIMAGE_YEAR
+  tile_size: 256 # per side, in pixels
+  overwrite: True
+  n_jobs: 10
+  COCO_metadata:
+    year: 2021
+    version: 1.0
+    description: Swiss Image Hinterground w/ Quarries and Mineral Exploitation Sites detection
+    contributor: swisstopo
+    url: https://swisstopo.ch
+    license:
+      name: Unknown
+      url:
+    category:
+      name: "Quarry"
+      supercategory: "Land usage"
+
+# 3-Perform the object detection based on the optimized trained model
+make_detections.py:
+  working_folder: ./output/output_det/SWISSIMAGE_YEAR
+  log_subfolder: logs
+  sample_tagged_img_subfolder: sample_tagged_images
+  COCO_files: # relative paths, w/ respect to the working_folder
+    oth: COCO_oth.json
+  detectron2_config_file: '../../../config/detectron2_config_dqry.yaml' # path relative to the working_folder
+  model_weights:
+    pth_file: '../../../input/input_det/logs/model_0002999.pth' # trained model minimizing the validation loss curve, 
+                                                                # monitoring of the training process via tensorboard (tensorboard --logdir </logs>)
+                                                                # for the provided parameters model_0002999.pth is chosen 
+  image_metadata_json: './output/output_det/SWISSIMAGE_YEAR/img_metadata.json'
+  rdp_simplification: # rdp = Ramer-Douglas-Peucker
+    enabled: True
+    epsilon: 2.0 # cf. https://rdp.readthedocs.io/en/latest/
+  score_lower_threshold: 0.3
+
+# 4-Filtering and merging detection polygons to improve results
+filter_detections.py:
+  year: SWISSIMAGE_YEAR
+  detections: ./output/output_det/SWISSIMAGE_YEAR/oth_detections_at_0dot3_threshold.gpkg
+  shapefile: ./input/input_det/AoI/swissimage_footprint_SWISSIMAGE_YEAR.shp
+  dem: ./input/input_det/DEM/switzerland_dem_EPSG2056.tif
+  elevation: 1200.0 # m, altitude threshold
+  score: 0.95 # detection score (from 0 to 1) provided by detectron2
+  distance: 10 # m, distance use as a buffer to merge close polygons (likely to belong to the same object) together
+  area: 5000.0 # m2, area threshold under which polygons are discarded
+  output: ./output/output_det/SWISSIMAGE_YEAR/oth_detections_at_0dot3_threshold_year-{year}_score-{score}_area-{area}_elevation-{elevation}_distance-{distance}.geojson
diff --git a/config/config_det.yaml b/config/config_det.yaml
@@ -0,0 +1,66 @@
+###################################
+##### Detection by inference #####
+# Automatic detection of Quarries and Mineral Extraction Sites (MES) in images
+
+# 1-Produce tiles geometry according to the AoI extent and zoom level
+prepare_data.py: 
+  srs: "EPSG:2056"  # Projection of the input file
+  datasets:
+    shapefile: ./input/input_det/AoI/<AoI_shapefile>
+  output_folder: ./output/output_det
+  zoom_level: 16  # z, keep between 15 and 18  
+
+# 2-Request tiles according to the provided AOI and tiles parameters
+generate_tilesets.py:
+  debug_mode: False  # sample of tiles
+  datasets:
+    aoi_tiles_geojson: ./output/output_det/tiles.geojson
+    orthophotos_web_service:
+      type: XYZ # supported values: 1. MIL = Map Image Layer 2. WMS 3. XYZ
+      url: https://wmts.geo.admin.ch/1.0.0/ch.swisstopo.swissimage-product/default/<YEAR>/3857/{z}/{x}/{y}.jpeg # !!! Assign the desired year !!! 
+  output_folder: ./output/output_det
+  tile_size: 256 # per side, in pixels
+  overwrite: False
+  n_jobs: 10
+  COCO_metadata:
+    year: 2021
+    version: 1.0
+    description: Swiss Image Hinterground w/ Quarries and Mineral Exploitation Sites detection
+    contributor: swisstopo
+    url: https://swisstopo.ch
+    license:
+      name: Unknown
+      url:
+    category:
+      name: "Quarry"
+      supercategory: "Land usage"
+
+# 3-Perform the object detection based on the optimized trained model
+make_detections.py:
+  working_folder: ./output/output_det
+  log_subfolder: logs
+  sample_tagged_img_subfolder: sample_tagged_images
+  COCO_files: # relative paths, w/ respect to the working_folder
+    oth: COCO_oth.json
+  detectron2_config_file: '../../config/detectron2_config_dqry.yaml' # path relative to the working_folder
+  model_weights:
+    pth_file: '../../input/input_det/logs/model_0002999.pth' # trained model minimizing the validation loss curve, 
+                                                                # monitoring of the training process via tensorboard (tensorboard --logdir </logs>)
+                                                                # for the provided parameters model_0002999.pth is chosen 
+  image_metadata_json: './output/output_det/img_metadata.json'
+  rdp_simplification: # rdp = Ramer-Douglas-Peucker
+    enabled: True
+    epsilon: 2.0 # cf. https://rdp.readthedocs.io/en/latest/
+  score_lower_threshold: 0.3
+
+# 4-Filtering and merging detection polygons to improve results
+filter_detections.py:
+  year: <YEAR>
+  detections: ./output/output_det/oth_detections_at_0dot3_threshold.gpkg
+  shapefile: ./input/input_det/AoI/<AoI_shapefile>
+  dem: ./input/input_det/DEM/switzerland_dem_EPSG2056.tif
+  elevation: 1200.0 # m, altitude threshold
+  score: 0.95 # detection score (from 0 to 1) provided by detectron2
+  distance: 10 # m, distance use as a buffer to merge close polygons (likely to belong to the same object) together
+  area: 5000.0 # m2, area threshold under which polygons are discarded
+  output: ./output/output_det/oth_detections_at_0dot3_threshold_year-{year}_score-{score}_area-{area}_elevation-{elevation}_distance-{distance}.geojson
diff --git a/config/config_track.yaml b/config/config_track.yaml
@@ -0,0 +1,17 @@
+################################################
+###  Identifying and tracking MES over time  ### 
+
+# Track object in different datasets over years
+track_detections.py:  
+  years: <[YEAR1, YEAR2, YEAR3,...]>                          # Provide a list of years used for detection
+  datasets:
+    detection: ./input/input_track/oth_detections_at_0dot3_threshold_year-{year}_score-0dot95_area-5000_elevation-1200_distance-10.geojson  # Final detection file, produced by filter_detections.py
+  output_folder: ./output/output_track/oth_detections_at_0dot3_threshold_score-0dot95_area-5000_elevation-1200_distance-10
+
+# Plots
+plots.py:  
+  object_id: <[ID_OBJECT1, ID_OBJECT2, ID_OBJECT3,...]>       # Provide a list of id_object defined by track_detections.py
+  plots: ['area-year'] 
+  datasets:
+    detection: ./output/output_track/oth_detections_at_0dot3_threshold_score-0dot95_area-5000_elevation-1200_distance-10/detections_years.geojson  # Object tracking file, produced by track_detections.py
+  output_folder: ./output/output_track/oth_detections_at_0dot3_threshold_score-0dot95_area-5000_elevation-1200_distance-10/plots
diff --git a/config/config_trne.yaml b/config/config_trne.yaml
@@ -0,0 +1,83 @@
+#############################################
+####### Model training and evaluation ####### 
+# Training of automatic detection of Quarries and Mineral Extraction Sites (MES) in images with a provided ground truth
+
+# 1-Prepare the tiles geometry according to the AOI and zoom level
+prepare_data.py:  
+  srs: "EPSG:2056"
+  datasets:
+    shapefile: ./input/input_trne/tlm-hr-trn-topo.shp
+  output_folder: ./output/output_trne
+  zoom_level: 16  #z, keep between 15 and 18  
+
+# 2-Fetch of tiles (online server) and split into 3 datasets: train, test, validation
+generate_tilesets.py:
+  debug_mode: False # sample of tiles
+  datasets:
+    aoi_tiles_geojson: ./output/output_trne/tiles.geojson
+    ground_truth_labels_geojson: ./output/output_trne/labels.geojson
+    orthophotos_web_service:
+      type: XYZ # supported values: 1. MIL = Map Image Layer 2. WMS 3. XYZ
+      url: https://wmts.geo.admin.ch/1.0.0/ch.swisstopo.swissimage-product/default/2020/3857/{z}/{x}/{y}.jpeg # Models are trained on 2020 (default year) SWISSIMAGE mosaic 
+  output_folder: ./output/output_trne
+  tile_size: 256 # per side, in pixels
+  overwrite: False
+  n_jobs: 10
+  COCO_metadata:
+    year: 2021
+    version: 1.0
+    description: Swiss Image Hinterground w/ Quarries and Mineral Exploitation Sites detection
+    contributor: swisstopo
+    url: https://swisstopo.ch
+    license:
+      name: Unknown
+      url:
+    category:
+      name: "Quarry"
+      supercategory: "Land usage"
+
+# 3-Train the model with the detectron2 algorithm
+# Monitor the training process via tensorboard (tensorboard --logdir </logs>). Choice of the optimized model: minimisation of the validation loss curve
+train_model.py:
+  working_folder: ./output/output_trne
+  log_subfolder: logs
+  sample_tagged_img_subfolder: sample_tagged_images
+  COCO_files: # relative paths, w/ respect to the working_folder
+    trn: COCO_trn.json
+    val: COCO_val.json
+    tst: COCO_tst.json
+  detectron2_config_file: '../../config/detectron2_config_dqry.yaml' # path relative to the working_folder
+  model_weights:
+    model_zoo_checkpoint_url: "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml"
+
+# 4-Perform the object detection based on the optimized trained model
+make_detections.py:
+  working_folder: ./output/output_trne
+  log_subfolder: logs
+  sample_tagged_img_subfolder: sample_tagged_images
+  COCO_files: # relative paths, w/ respect to the working_folder
+    trn: COCO_trn.json
+    val: COCO_val.json
+    tst: COCO_tst.json
+  detectron2_config_file: '../../config/detectron2_config_dqry.yaml' # path relative to the working_folder
+  model_weights:
+    pth_file: './logs/model_0002999.pth'                        # trained model minimizing the validation loss curve, 
+                                                                # monitoring of the training process via tensorboard (tensorboard --logdir </logs>)
+                                                                # for the provided parameters model_0002999.pth is chosen 
+  image_metadata_json: './output/output_trne/img_metadata.json'
+  rdp_simplification: # rdp = Ramer-Douglas-Peucker
+    enabled: true
+    epsilon: 2.0 # cf. https://rdp.readthedocs.io/en/latest/
+  score_lower_threshold: 0.05
+
+# 5-Evaluate the quality of the detection for the different datasets with metrics calculation
+assess_detections.py:
+  datasets:
+    ground_truth_labels_geojson: ./output/output_trne/labels.geojson
+    image_metadata_json: ./output/output_trne/img_metadata.json
+    split_aoi_tiles_geojson: ./output/output_trne/split_aoi_tiles.geojson # aoi = Area of Interest
+    detections:
+      trn: ./output/output_trne/trn_detections_at_0dot05_threshold.gpkg
+      val: ./output/output_trne/val_detections_at_0dot05_threshold.gpkg
+      tst: ./output/output_trne/tst_detections_at_0dot05_threshold.gpkg
+  output_folder: ./output/output_trne