Feature/experiment

README.md

@@ -2,11 +2,11 @@
 [![tests](https://github.com/prime-slam/lidar-labelling/actions/workflows/ci.yml/badge.svg)](https://github.com/prime-slam/lidar-labelling/actions/workflows/ci.yml)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
-lidar-labelling is a tool for automatic segmentation of raw lidar clouds based on image segmentation.
+lidar-labelling is a tool for automatic segmentation of dense lidar clouds based on image segmentation.
 
 Our labelling algorithm consists of two stages.
 
-The first stage is a preliminary marking - a naive transfer of segmentation from images (performed by the [SAM](https://github.com/facebookresearch/segment-anything) algorithm) to the cloud. Then multi-stage processing of the cloud is performed, which allows us to make the cloud more compact before the final labelling without losing points that are significant for labelling. The preprocessing stages include removing points whose corresponding pixels were not marked on a sequence of images, selection of points close to the sensors, removal of noise, cloud voxelization.
+The first stage is a initial segmentation - a naive transfer of segmentation from images (performed by the [SAM](https://github.com/facebookresearch/segment-anything) algorithm) to the cloud. Then multi-stage processing of the cloud is performed, which allows us to make the cloud more compact before the final labelling without losing points that are significant for labelling. The preprocessing stages include removing points whose corresponding pixels were not marked on a sequence of images, selection of points close to the sensors, removal of noise, cloud voxelization.
 
 The next stage is segmentation itself. The segmentation criterion in this work is the distance between points, which is calculated through the physical distance and the degree of similarity of the labelling of points on several images. Based on the distance matrix, the prepared cloud is segmented using the [GraphCut](https://ieeexplore.ieee.org/abstract/document/937505) algorithm.
 
@@ -16,6 +16,11 @@ This tool currently supports processing of [KITTI](https://www.cvlibs.net/datase
 ## Usage
 Please check `example.ipynb` with a example of cloud segmentation from the [KITTI](https://www.cvlibs.net/datasets/kitti/eval_odometry.php) dataset.
 
+## Experiments
+Files for reproducing experiments are in folder `experiment`. You will need the [KITTI](https://www.cvlibs.net/datasets/kitti/eval_odometry.php) dataset with its folder structure and pre-performed image segmentation using the [SAM](https://github.com/facebookresearch/segment-anything) algorithm in npz format.
+
+First run `main_kitti_processing.py` to generate segmentation with our algorithm. Then run `main_kitti_processing_metrics.py` to calculate the segmentation metrics of each run and write them to a csv file. Then run `main_calc_metrics_by_csv.py` to calculate the average values ​​of the metrics in the csv file.
+
 ## License
 This project is licensed under the Apache License — 
 see the [LICENSE](https://github.com/prime-slam/lidar-labelling/blob/main/LICENSE) file for details.

example.ipynb

@@ -8,9 +8,9 @@
    "source": [
     "from src.datasets.kitti_dataset import KittiDataset\n",
     "\n",
-    "dataset_path = \"dataset/\"\n",
+    "dataset_path = \"dataset/\"  # kitti\n",
     "sequence = \"00\"\n",
-    "image_instances_path = \"pipeline/vfm-labels/sam/00/\"\n",
+    "image_instances_path = \"pipeline/vfm-labelss/sam/00/\"  # images processed by the SAM algorithm in npz format\n",
     "kitti = KittiDataset(dataset_path, sequence, image_instances_path)"
    ]
   },
@@ -20,16 +20,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# setting parameter values\n",
+    "\n",
     "from src.services.preprocessing.common.config import ConfigDTO\n",
     "\n",
     "config = ConfigDTO(\n",
     "    **{\n",
     "        \"dataset\": kitti,\n",
     "        \"start_index\": 20,\n",
     "        \"end_index\": 24,\n",
-    "        \"start_image_index_offset\": 3,\n",
+    "        \"start_image_index_offset\": 0,\n",
     "        \"alpha_physical_distance\": 5,\n",
-    "        \"beta_instance_distance\": 3,\n",
+    "        \"beta_instance_distance\": 5,\n",
     "        \"T_normalized_cut\": 0.02,\n",
     "        \"reduce_detail_int_to_union_threshold\": 0.5,\n",
     "        \"reduce_detail_int_to_mask_threshold\": 0.6,\n",
@@ -48,6 +50,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# pcd initialisation and initial segmentation based on images\n",
+    "\n",
     "from src.services.preprocessing.init.map import InitMapProcessor\n",
     "from src.services.preprocessing.init.instances_matrix import InitInstancesMatrixProcessor\n",
     "\n",
@@ -70,6 +74,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# visualisation of the initial pcd segmentation masks for a particular image\n",
+    "\n",
     "import copy\n",
     "\n",
     "from src.utils.pcd_utils import color_pcd_by_labels\n",
@@ -85,6 +91,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# pcd handler initialisation\n",
+    "\n",
     "from src.services.preprocessing.not_zero import SelectionNotZeroProcessor\n",
     "from src.services.preprocessing.in_cube import SelectionInCubeProcessor\n",
     "from src.services.preprocessing.statistical_outlier import StatisticalOutlierProcessor\n",
@@ -102,6 +110,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# pcd processing and saving the state\n",
+    "\n",
     "import copy\n",
     "\n",
     "pcd = copy.deepcopy(init_pcd)\n",
@@ -118,13 +128,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# visualisation of the processed pcd before voxelization\n",
+    "\n",
     "import copy\n",
     "\n",
     "from src.utils.pcd_utils import color_pcd_by_labels\n",
     "from src.utils.pcd_utils import visualize_pcd\n",
     "\n",
-    "colored_pcd_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering), points2instances_pcd_for_clustering[:, 0])\n",
-    "visualize_pcd(colored_pcd_pcd_for_clustering)"
+    "colored_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering), points2instances_pcd_for_clustering[:, 0])\n",
+    "visualize_pcd(colored_pcd_for_clustering)"
    ]
   },
   {
@@ -133,6 +145,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# final processing step - voxelisation of the pcd\n",
+    "\n",
     "from src.services.preprocessing.voxel_down import VoxelDownProcessor\n",
     "\n",
     "pcd, points2instances, trace = VoxelDownProcessor().process(config, pcd, points2instances)"
@@ -144,6 +158,25 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# visualisation of the voxelised pcd\n",
+    "\n",
+    "import copy\n",
+    "\n",
+    "from src.utils.pcd_utils import color_pcd_by_labels\n",
+    "from src.utils.pcd_utils import visualize_pcd\n",
+    "\n",
+    "colored_voxel_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd), points2instances[:, 0])\n",
+    "visualize_pcd(colored_voxel_pcd_for_clustering)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# calculation of distance matrix for voxelised pcd\n",
+    "\n",
     "import numpy as np\n",
     "\n",
     "from scipy.spatial.distance import cdist\n",
@@ -176,6 +209,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# distance matrix processing\n",
+    "\n",
     "from src.services.distance.isolated import RemovingIsolatedPointsProcessor\n",
     "from src.services.distance.connected_component import ExtractionLargestConnectedComponentProcessor\n",
     "\n",
@@ -205,10 +240,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# pcd clustering using GraphCut algorithm\n",
+    "\n",
     "from src.services.normalized_cut_service import normalized_cut\n",
     "\n",
     "eigenval = 2\n",
-    "\n",
     "clusters = normalized_cut(\n",
     "    dist,\n",
     "    np.array([i for i in range(len(points))], dtype=int),\n",
@@ -232,11 +268,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from src.utils.pcd_utils import color_pcd_by_clusters_and_voxels\n",
-    "from src.utils.pcd_utils import visualize_pcd\n",
-    "\n",
-    "pcd_colored = color_pcd_by_clusters_and_voxels(pcd_for_clustering, trace, clusters)\n",
-    "visualize_pcd(pcd_colored)"
+    "clusters[0]"
    ]
   },
   {
@@ -245,10 +277,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from src.utils.pcd_utils import color_pcd_by_labels\n",
+    "# visualisation of segmentation results. masks will be drawn on the processed pcd before voxelisation\n",
     "\n",
-    "pcd_src_colored = color_pcd_by_labels(pcd_for_clustering, points2instances_pcd_for_clustering[:, 2])\n",
-    "visualize_pcd(pcd_src_colored)"
+    "from src.utils.pcd_utils import color_pcd_by_clusters_and_voxels\n",
+    "from src.utils.pcd_utils import visualize_pcd\n",
+    "\n",
+    "pcd_colored = color_pcd_by_clusters_and_voxels(pcd_for_clustering, trace, clusters)\n",
+    "visualize_pcd(pcd_colored)"
    ]
   }
  ],