notebooks/Dataset_statistics.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Dataset statistics\n",
        "\n",
        "This notebook shows how to gather some statistics, such as the class distribution, from a labeled point cloud dataset.\n",
        "\n",
        "In this code we assume the point clouds have been labelled following the process in our [Urban PointCloud Processing](https://github.com/Amsterdam-AI-Team/Urban_PointCloud_Processing/tree/main/datasets) project. For more information on the specifics of the datasets used, see [the description there](https://github.com/Amsterdam-AI-Team/Urban_PointCloud_Processing/blob/main/datasets/README.md)."
      ],
      "metadata": {},
      "id": "9dde9cef-6aa2-4268-9c0c-f38272d3914c"
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import laspy\n",
        "import pathlib\n",
        "from tqdm import tqdm\n",
        "\n",
        "import set_path  # add project src to path\n",
        "from upcp.utils import las_utils\n",
        "\n",
        "import config as cf  # use config or config_azure"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1691583090202
        }
      },
      "id": "e22121c2-f260-4db3-8337-78de2efa846c"
    },
    {
      "cell_type": "code",
      "source": [
        "# Retrieve paths to point cloud demo data\n",
        "files = list(pathlib.Path(cf.dataset_folder).glob(f'{cf.prefix}*.laz'))\n",
        "\n",
        "# Set class labels\n",
        "CLS_LABELS = {\n",
        "        1: 'Road',\n",
        "        9: 'Other ground',\n",
        "        10: 'Building',\n",
        "        30: 'Tree',\n",
        "        40: 'Car',\n",
        "        60: 'Streetlight',\n",
        "        61: 'Traffic light',\n",
        "        62: 'Traffic sign',\n",
        "        80: 'City bench',\n",
        "        81: 'Rubbish bin'\n",
        "    }"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1691583091274
        }
      },
      "id": "273ef42d-7fe7-485e-9fc3-cd092dd34219"
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "## Statistics per point cloud tile\n",
        "\n",
        "We collect the total number of points, the number of classes, and the number of points per class for each labelled point cloud tile."
      ],
      "metadata": {},
      "id": "88665ef5-ad2c-46a5-9ddf-369f3a2f6ca7"
    },
    {
      "cell_type": "code",
      "source": [
        "# Create dataframe\n",
        "columns = ['tilecode', 'n_points', 'n_classes']\n",
        "columns.extend(CLS_LABELS.values())\n",
        "data_df = pd.DataFrame(columns=columns).set_index('tilecode')\n",
        "\n",
        "for f in tqdm(files):\n",
        "    # Load point cloud\n",
        "    pc = laspy.read(f.as_posix())\n",
        "    tilecode = las_utils.get_tilecode_from_filename(f.as_posix())\n",
        "    npz_file = np.load(cf.pred_folder + cf.prefix_pred + tilecode + '.npz')\n",
        "    labels = npz_file['label']\n",
        "\n",
        "    # Count points per class\n",
        "    classes, counts = np.unique(labels, return_counts=True)\n",
        "    data = {CLS_LABELS[c]: cnt for c, cnt in zip(classes, counts)}\n",
        "\n",
        "    # Get total point count\n",
        "    data['n_points'] = np.sum(counts)\n",
        "\n",
        "    # Get number of classes present in the point cloud (excluding 'noise' and 'unlabelled')\n",
        "    real_classes = [cnt for c, cnt in zip(classes, counts) if c not in (0, 99)]\n",
        "    data['n_classes'] = np.count_nonzero(real_classes)\n",
        "\n",
        "    data_df.loc[tilecode] = data\n",
        "\n",
        "data_df = data_df.fillna(0).astype('int64').sort_index()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1691583098195
        }
      },
      "id": "3ff60478-967c-4982-99fc-d160cf3700b8"
    },
    {
      "cell_type": "code",
      "source": [
        "data_df"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1691583099285
        }
      },
      "id": "f5815a99-81ad-495c-831f-03579e440ce8"
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "## Summary statistics for the entire dataset\n",
        "\n",
        "We sum over all point cloud tiles and compute summary statistics:\n",
        "* the total number of points for each class accross the dataset;\n",
        "* the percentage of points (of the total) belonging to each class;\n",
        "* the number of tiles that contain points of each class."
      ],
      "metadata": {},
      "id": "89cfcddb-6ea6-4994-943d-5e22a653dd7d"
    },
    {
      "cell_type": "code",
      "source": [
        "# Create dataframe\n",
        "columns = ['Total']\n",
        "columns.extend(CLS_LABELS.values())\n",
        "stats_df = pd.DataFrame(columns=columns)\n",
        "\n",
        "# Get total point counts\n",
        "counts = data_df.sum()\n",
        "counts['Total'] = counts['n_points']\n",
        "stats_df.loc['n_points'] = counts\n",
        "\n",
        "# Compute percentage\n",
        "stats_df.loc['percentage'] = (100 * stats_df.loc['n_points'] /\n",
        "                              stats_df.loc['n_points', 'Total']).astype(float).round(2)\n",
        "\n",
        "# Compute number of tiles where class is present\n",
        "occurs = [len(data_df)]\n",
        "occurs.extend(np.count_nonzero(data_df[CLS_LABELS.values()], axis=0))\n",
        "stats_df.loc['n_tiles'] = occurs"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1691583100511
        }
      },
      "id": "165a6f08-febb-47af-acf3-4ab8b806a1ab"
    },
    {
      "cell_type": "code",
      "source": [
        "stats_df"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1691583101559
        }
      },
      "id": "476d3a26-0cad-47e8-ae06-a44085265e57"
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3 (ipykernel)"
    },
    "language_info": {
      "name": "python",
      "version": "3.8.5",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "microsoft": {
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      },
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      }
    },
    "kernel_info": {
      "name": "python3"
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}