clustering_comparison_e175.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import scanpy as sc\n",
    "import numpy as np\n",
    "from sklearn.metrics.cluster import normalized_mutual_info_score, adjusted_rand_score\n",
    "from sklearn.metrics import homogeneity_score, completeness_score, v_measure_score, fowlkes_mallows_score, silhouette_score\n",
    "from utils import sankey_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR = './data/'\n",
    "DATASET_NAME = 'E175'\n",
    "TOOLS = ['monocle', 'seurat', 'scanpy', 'scvi-tools']\n",
    "PARAMS_TUNING = ['default', 'celltypist']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_scores(dir, dataset, labels_df, labels_matched, ground_truth_labels):\n",
    "    scores = {}\n",
    "    scores['NMI'] = {}\n",
    "    scores['ARI'] = {}\n",
    "    scores['homogeneity'] = {}\n",
    "    scores['completeness'] = {}\n",
    "    scores['v_measure'] = {}\n",
    "    scores['fowlkes_mallows'] = {}\n",
    "    for tool in TOOLS:\n",
    "        scores['NMI'][tool] = normalized_mutual_info_score(labels_pred=labels_df['cluster_'+tool], labels_true=labels_df[f'cluster_{ground_truth_labels}'], average_method='arithmetic')\n",
    "        scores['ARI'][tool] = adjusted_rand_score(labels_pred=labels_df['cluster_'+tool], labels_true=labels_df[f'cluster_{ground_truth_labels}'])\n",
    "        scores['homogeneity'][tool] = homogeneity_score(labels_pred=labels_df['cluster_'+tool], labels_true=labels_df[f'cluster_{ground_truth_labels}'])\n",
    "        scores['completeness'][tool] = completeness_score(labels_pred=labels_df['cluster_'+tool], labels_true=labels_df[f'cluster_{ground_truth_labels}'])\n",
    "        scores['v_measure'][tool] = v_measure_score(labels_pred=labels_df['cluster_'+tool], labels_true=labels_df[f'cluster_{ground_truth_labels}'])\n",
    "        scores['fowlkes_mallows'][tool] = fowlkes_mallows_score(labels_pred=labels_df['cluster_'+tool], labels_true=labels_df[f'cluster_{ground_truth_labels}'])\n",
    "    scores_df = pd.DataFrame(scores)\n",
    "    scores_df.to_csv(f'{dir}{dataset}/scores_{labels_matched}_{ground_truth_labels}.csv')\n",
    "    scores_df.to_latex(f'{dir}{dataset}/scores_{labels_matched}_{ground_truth_labels}.tex')\n",
    "    display(scores_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "E175 - default labels\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster_COTAN</th>\n",
       "      <th>cluster_monocle</th>\n",
       "      <th>cluster_seurat</th>\n",
       "      <th>cluster_scanpy</th>\n",
       "      <th>cluster_scvi-tools</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CGTTTAGTTTAC</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCTAGAACAACG</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACCTTTGTTCGT</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAAAATATCGCC</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTACCCTATTTC</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACTAAGCCGCTT</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CTTAGGGGGGCT</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GCTATCTGGTTG</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CACTATTGTCAA</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CGGACGTTCGTC</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>874 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              cluster_COTAN  cluster_monocle  cluster_seurat  cluster_scanpy  \\\n",
       "cell                                                                           \n",
       "CGTTTAGTTTAC              1                1               5               7   \n",
       "TCTAGAACAACG              3                1               4               4   \n",
       "ACCTTTGTTCGT             10                1               4               5   \n",
       "TAAAATATCGCC              5                1               4               8   \n",
       "GTACCCTATTTC              5                1               4               8   \n",
       "...                     ...              ...             ...             ...   \n",
       "ACTAAGCCGCTT              7                1               5               7   \n",
       "CTTAGGGGGGCT              2                1               3               9   \n",
       "GCTATCTGGTTG              8                1               2               6   \n",
       "CACTATTGTCAA              2                1               3               3   \n",
       "CGGACGTTCGTC              2                1               3               6   \n",
       "\n",
       "              cluster_scvi-tools  \n",
       "cell                              \n",
       "CGTTTAGTTTAC                   4  \n",
       "TCTAGAACAACG                   9  \n",
       "ACCTTTGTTCGT                   1  \n",
       "TAAAATATCGCC                   5  \n",
       "GTACCCTATTTC                   5  \n",
       "...                          ...  \n",
       "ACTAAGCCGCTT                   4  \n",
       "CTTAGGGGGGCT                  10  \n",
       "GCTATCTGGTTG                   1  \n",
       "CACTATTGTCAA                   5  \n",
       "CGGACGTTCGTC                  10  \n",
       "\n",
       "[874 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NMI</th>\n",
       "      <th>ARI</th>\n",
       "      <th>homogeneity</th>\n",
       "      <th>completeness</th>\n",
       "      <th>v_measure</th>\n",
       "      <th>fowlkes_mallows</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>monocle</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.811093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seurat</th>\n",
       "      <td>0.314587</td>\n",
       "      <td>0.102977</td>\n",
       "      <td>0.432556</td>\n",
       "      <td>0.247176</td>\n",
       "      <td>0.314587</td>\n",
       "      <td>0.448201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scanpy</th>\n",
       "      <td>0.287347</td>\n",
       "      <td>0.049438</td>\n",
       "      <td>0.497811</td>\n",
       "      <td>0.201962</td>\n",
       "      <td>0.287347</td>\n",
       "      <td>0.309711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scvi-tools</th>\n",
       "      <td>0.255927</td>\n",
       "      <td>0.032870</td>\n",
       "      <td>0.413885</td>\n",
       "      <td>0.185234</td>\n",
       "      <td>0.255927</td>\n",
       "      <td>0.319597</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 NMI       ARI  homogeneity  completeness  v_measure  \\\n",
       "monocle     0.000000  0.000000     0.000000      1.000000   0.000000   \n",
       "seurat      0.314587  0.102977     0.432556      0.247176   0.314587   \n",
       "scanpy      0.287347  0.049438     0.497811      0.201962   0.287347   \n",
       "scvi-tools  0.255927  0.032870     0.413885      0.185234   0.255927   \n",
       "\n",
       "            fowlkes_mallows  \n",
       "monocle            0.811093  \n",
       "seurat             0.448201  \n",
       "scanpy             0.309711  \n",
       "scvi-tools         0.319597  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "link": {
          "source": [
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           3,
           3,
           3,
           3,
           3,
           3,
           3,
           4,
           4,
           4,
           4,
           4,
           4,
           5,
           6,
           6,
           6,
           6,
           6,
           6,
           7,
           8,
           8,
           8,
           8,
           8,
           8,
           9,
           9,
           9,
           9,
           9,
           10,
           10,
           10,
           10,
           10,
           10,
           10,
           10,
           11,
           11,
           11,
           11,
           11,
           11,
           11,
           11,
           11,
           12,
           12,
           12,
           12,
           12,
           12,
           13,
           13,
           13,
           13,
           13,
           13,
           13,
           14,
           15,
           15,
           15,
           15,
           15,
           15,
           15,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           17,
           17,
           17,
           17,
           18,
           18,
           18,
           18,
           18,
           18,
           19,
           20,
           20,
           20,
           20,
           20,
           20,
           20,
           20,
           20,
           21,
           21,
           21,
           22,
           22,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           23,
           24,
           24,
           25,
           26,
           27,
           27,
           27,
           27,
           27,
           27,
           27,
           27,
           28,
           28,
           29,
           29
          ],
          "target": [
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           11,
           12,
           13,
           15,
           16,
           8,
           10,
           11,
           12,
           13,
           16,
           17,
           18,
           19,
           8,
           9,
           10,
           11,
           12,
           13,
           16,
           10,
           11,
           12,
           13,
           15,
           18,
           14,
           9,
           11,
           12,
           14,
           16,
           18,
           19,
           20,
           21,
           22,
           25,
           26,
           28,
           21,
           22,
           25,
           26,
           28,
           20,
           21,
           22,
           23,
           24,
           25,
           26,
           28,
           20,
           21,
           22,
           24,
           25,
           26,
           27,
           28,
           29,
           20,
           22,
           24,
           25,
           27,
           28,
           20,
           22,
           23,
           25,
           26,
           28,
           29,
           23,
           20,
           22,
           24,
           25,
           26,
           27,
           29,
           20,
           21,
           22,
           25,
           26,
           27,
           28,
           29,
           20,
           23,
           26,
           29,
           20,
           21,
           24,
           25,
           27,
           28,
           27,
           30,
           38,
           41,
           42,
           44,
           46,
           47,
           48,
           51,
           30,
           46,
           47,
           41,
           46,
           30,
           31,
           32,
           33,
           34,
           35,
           36,
           37,
           38,
           39,
           40,
           44,
           46,
           47,
           52,
           53,
           54,
           55,
           46,
           47,
           46,
           46,
           30,
           43,
           45,
           46,
           47,
           48,
           49,
           50,
           46,
           47,
           44,
           46
          ],
          "value": [
           218,
           215,
           212,
           84,
           75,
           56,
           14,
           74,
           25,
           61,
           36,
           5,
           2,
           5,
           10,
           1,
           8,
           7,
           64,
           69,
           14,
           49,
           2,
           1,
           42,
           76,
           10,
           31,
           5,
           5,
           43,
           10,
           1,
           3,
           2,
           66,
           2,
           75,
           2,
           6,
           2,
           1,
           1,
           44,
           14,
           4,
           59,
           20,
           13,
           8,
           13,
           43,
           15,
           13,
           26,
           6,
           6,
           8,
           44,
           1,
           8,
           11,
           1,
           10,
           6,
           15,
           14,
           3,
           16,
           12,
           4,
           9,
           2,
           63,
           7,
           1,
           3,
           1,
           4,
           52,
           2,
           1,
           3,
           3,
           4,
           13,
           76,
           1,
           4,
           58,
           3,
           1,
           1,
           3,
           9,
           4,
           11,
           13,
           16,
           3,
           10,
           2,
           38,
           8,
           1,
           2,
           1,
           1,
           9,
           1,
           35,
           1,
           15,
           14,
           1,
           16,
           1,
           7,
           133,
           6,
           1,
           1,
           2,
           127,
           1,
           1,
           116,
           5,
           1,
           1,
           28,
           5,
           1,
           3,
           9,
           2,
           2,
           2,
           10,
           3,
           1,
           1,
           7,
           3,
           2,
           78,
           1,
           76,
           68,
           12,
           1,
           11,
           30,
           2,
           1,
           1,
           1,
           55,
           2,
           1,
           21
          ]
         },
         "node": {
          "color": [
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b"
          ],
          "label": [
           1,
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           11,
           12,
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           "Blood: Erythrocyte",
           "Fibroblast: Pia 2",
           "Gastrulation: Mesoderm",
           "Glioblast: Forebrain",
           "Glioblast: Forebrain astrocyte",
           "Glioblast: Midbrain",
           "Glioblast: Mixed region",
           "Glioblast: Mixed region astrocytes",
           "Glioblast: PreOPC",
           "Immune: Non-cycling microglia",
           "Neuroblast: Forebrain GABAergic",
           "Neuroblast: Forebrain glutamatergic",
           "Neuroblast: Hindbrain glutamatergic",
           "Neuroblast: Mixed region glutamatergic",
           "Neuroblast: Neuronal intermediate progenitor",
           "Neuron: Cajal-Retzius",
           "Neuron: Cortical or hippocampal glutamatergic",
           "Neuron: Forebrain GABAergic",
           "Neuron: Forebrain glutamatergic",
           "Neuron: Midbrain GABAergic",
           "Neuron: Mixed region GABAergic",
           "Neuron: Spinal cord glutamatergic",
           "Olfactory ensheathing cell: Olfactory ensheathing cell",
           "Oligodendrocyte: Oligodendrocyte precursor cell",
           "Radial glia: Dorsal forebrain",
           "Radial glia: Forebrain"
          ],
          "line": {
           "color": "black",
           "width": 0.5
          },
          "pad": 15,
          "thickness": 20
         },
         "type": "sankey"
        }
       ],
       "layout": {
        "annotations": [
         {
          "showarrow": false,
          "text": "monocle",
          "x": 0,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "seurat",
          "x": 1,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "scanpy",
          "x": 2,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "scvi-tools",
          "x": 3,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "celltypist",
          "x": 4,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         }
        ],
        "font": {
         "size": 10
        },
        "height": 550,
        "plot_bgcolor": "rgba(0,0,0,0)",
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "E175 - default labels"
        },
        "width": 1400,
        "xaxis": {
         "showgrid": false,
         "visible": false,
         "zeroline": false
        },
        "yaxis": {
         "showgrid": false,
         "visible": false,
         "zeroline": false
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "E175 - matching celltypist labels\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster_COTAN</th>\n",
       "      <th>cluster_monocle</th>\n",
       "      <th>cluster_seurat</th>\n",
       "      <th>cluster_scanpy</th>\n",
       "      <th>cluster_scvi-tools</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CGTTTAGTTTAC</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCTAGAACAACG</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACCTTTGTTCGT</th>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAAAATATCGCC</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTACCCTATTTC</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ACTAAGCCGCTT</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CTTAGGGGGGCT</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GCTATCTGGTTG</th>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CACTATTGTCAA</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CGGACGTTCGTC</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>16</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>874 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              cluster_COTAN  cluster_monocle  cluster_seurat  cluster_scanpy  \\\n",
       "cell                                                                           \n",
       "CGTTTAGTTTAC              1                1               1              23   \n",
       "TCTAGAACAACG              3                2               7               3   \n",
       "ACCTTTGTTCGT             10                9              18               2   \n",
       "TAAAATATCGCC              5                2              10              10   \n",
       "GTACCCTATTTC              5                2              10              10   \n",
       "...                     ...              ...             ...             ...   \n",
       "ACTAAGCCGCTT              7                1               1               1   \n",
       "CTTAGGGGGGCT              2               11              12              18   \n",
       "GCTATCTGGTTG              8               12               5               7   \n",
       "CACTATTGTCAA              2                2              14              13   \n",
       "CGGACGTTCGTC              2               11              12              16   \n",
       "\n",
       "              cluster_scvi-tools  \n",
       "cell                              \n",
       "CGTTTAGTTTAC                   1  \n",
       "TCTAGAACAACG                  22  \n",
       "ACCTTTGTTCGT                  11  \n",
       "TAAAATATCGCC                  17  \n",
       "GTACCCTATTTC                  17  \n",
       "...                          ...  \n",
       "ACTAAGCCGCTT                   1  \n",
       "CTTAGGGGGGCT                  24  \n",
       "GCTATCTGGTTG                  25  \n",
       "CACTATTGTCAA                  17  \n",
       "CGGACGTTCGTC                  24  \n",
       "\n",
       "[874 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NMI</th>\n",
       "      <th>ARI</th>\n",
       "      <th>homogeneity</th>\n",
       "      <th>completeness</th>\n",
       "      <th>v_measure</th>\n",
       "      <th>fowlkes_mallows</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>monocle</th>\n",
       "      <td>0.233652</td>\n",
       "      <td>0.011822</td>\n",
       "      <td>0.469499</td>\n",
       "      <td>0.155526</td>\n",
       "      <td>0.233652</td>\n",
       "      <td>0.211310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seurat</th>\n",
       "      <td>0.247473</td>\n",
       "      <td>0.010727</td>\n",
       "      <td>0.513258</td>\n",
       "      <td>0.163043</td>\n",
       "      <td>0.247473</td>\n",
       "      <td>0.197565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scanpy</th>\n",
       "      <td>0.261658</td>\n",
       "      <td>0.014857</td>\n",
       "      <td>0.534624</td>\n",
       "      <td>0.173218</td>\n",
       "      <td>0.261658</td>\n",
       "      <td>0.209212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scvi-tools</th>\n",
       "      <td>0.231802</td>\n",
       "      <td>0.005674</td>\n",
       "      <td>0.497821</td>\n",
       "      <td>0.151073</td>\n",
       "      <td>0.231802</td>\n",
       "      <td>0.176757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 NMI       ARI  homogeneity  completeness  v_measure  \\\n",
       "monocle     0.233652  0.011822     0.469499      0.155526   0.233652   \n",
       "seurat      0.247473  0.010727     0.513258      0.163043   0.247473   \n",
       "scanpy      0.261658  0.014857     0.534624      0.173218   0.261658   \n",
       "scvi-tools  0.231802  0.005674     0.497821      0.151073   0.231802   \n",
       "\n",
       "            fowlkes_mallows  \n",
       "monocle            0.211310  \n",
       "seurat             0.197565  \n",
       "scanpy             0.209212  \n",
       "scvi-tools         0.176757  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "link": {
          "source": [
           0,
           0,
           0,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           2,
           3,
           3,
           3,
           3,
           3,
           3,
           3,
           4,
           4,
           4,
           4,
           4,
           4,
           4,
           4,
           4,
           4,
           5,
           5,
           5,
           5,
           5,
           5,
           5,
           6,
           6,
           6,
           6,
           6,
           6,
           6,
           6,
           6,
           6,
           6,
           6,
           7,
           7,
           7,
           7,
           7,
           7,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           8,
           9,
           9,
           9,
           9,
           9,
           9,
           9,
           9,
           9,
           10,
           10,
           10,
           10,
           10,
           10,
           10,
           11,
           11,
           11,
           11,
           11,
           11,
           11,
           12,
           12,
           12,
           12,
           12,
           12,
           12,
           13,
           13,
           13,
           13,
           13,
           13,
           13,
           13,
           13,
           13,
           14,
           14,
           14,
           14,
           14,
           14,
           14,
           14,
           14,
           14,
           15,
           15,
           15,
           15,
           15,
           15,
           15,
           15,
           15,
           15,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           16,
           17,
           17,
           17,
           17,
           17,
           17,
           17,
           17,
           17,
           17,
           18,
           18,
           18,
           18,
           18,
           18,
           18,
           19,
           19,
           19,
           19,
           19,
           20,
           20,
           20,
           20,
           20,
           21,
           21,
           21,
           21,
           21,
           22,
           22,
           23,
           24,
           24,
           25,
           25,
           25,
           25,
           25,
           25,
           25,
           25,
           25,
           26,
           26,
           26,
           26,
           26,
           26,
           27,
           27,
           27,
           27,
           27,
           27,
           27,
           27,
           28,
           28,
           28,
           28,
           28,
           28,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           29,
           30,
           30,
           30,
           30,
           30,
           30,
           30,
           30,
           31,
           31,
           31,
           31,
           31,
           31,
           31,
           32,
           32,
           32,
           32,
           32,
           32,
           32,
           32,
           32,
           33,
           33,
           34,
           34,
           34,
           34,
           34,
           34,
           35,
           35,
           35,
           35,
           35,
           35,
           35,
           35,
           35,
           35,
           36,
           36,
           36,
           36,
           36,
           36,
           36,
           37,
           37,
           37,
           37,
           37,
           37,
           37,
           37,
           37,
           37,
           37,
           38,
           38,
           38,
           38,
           38,
           38,
           38,
           38,
           38,
           39,
           39,
           39,
           39,
           39,
           39,
           39,
           39,
           39,
           39,
           39,
           40,
           40,
           40,
           40,
           40,
           40,
           40,
           40,
           40,
           41,
           41,
           41,
           41,
           41,
           41,
           41,
           41,
           42,
           42,
           42,
           42,
           42,
           42,
           42,
           42,
           43,
           43,
           43,
           43,
           43,
           44,
           44,
           44,
           44,
           44,
           44,
           44,
           44,
           45,
           45,
           45,
           45,
           45,
           45,
           45,
           46,
           46,
           46,
           46,
           46,
           46,
           47,
           48,
           48,
           49,
           49,
           49,
           49,
           49,
           49,
           49,
           49,
           49,
           49,
           49,
           49,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           50,
           51,
           51,
           51,
           51,
           51,
           51,
           51,
           51,
           51,
           51,
           52,
           52,
           52,
           52,
           52,
           52,
           52,
           52,
           52,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           53,
           54,
           54,
           54,
           54,
           54,
           54,
           54,
           54,
           54,
           54,
           54,
           54,
           55,
           55,
           55,
           55,
           55,
           55,
           55,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           56,
           57,
           57,
           57,
           57,
           57,
           57,
           57,
           57,
           57,
           58,
           58,
           58,
           58,
           58,
           58,
           58,
           58,
           58,
           58,
           58,
           59,
           59,
           59,
           59,
           59,
           59,
           59,
           59,
           59,
           60,
           60,
           60,
           60,
           60,
           60,
           60,
           60,
           60,
           60,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           61,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           62,
           63,
           63,
           63,
           63,
           63,
           63,
           63,
           63,
           63,
           63,
           64,
           64,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           65,
           66,
           66,
           66,
           66,
           66,
           66,
           66,
           66,
           66,
           67,
           68,
           68,
           68,
           68,
           69,
           69,
           69,
           69,
           69,
           69,
           70,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           71,
           72,
           72,
           72,
           72,
           72,
           73,
           73,
           74,
           75,
           76,
           76,
           76,
           76,
           76,
           76,
           76,
           77,
           78,
           79,
           79,
           80,
           81,
           81,
           81,
           81,
           81,
           82,
           83,
           83,
           83,
           83,
           84,
           85,
           85,
           85,
           86,
           86,
           87,
           88,
           89,
           89,
           90,
           90,
           90,
           90,
           91,
           91,
           92,
           93,
           93,
           94,
           94,
           95,
           95,
           95,
           95,
           96,
           97,
           97,
           97,
           97
          ],
          "target": [
           24,
           31,
           45,
           25,
           30,
           32,
           33,
           37,
           39,
           47,
           25,
           27,
           30,
           32,
           33,
           34,
           37,
           38,
           39,
           42,
           43,
           44,
           45,
           28,
           29,
           31,
           35,
           36,
           43,
           46,
           25,
           27,
           32,
           37,
           38,
           40,
           41,
           42,
           43,
           45,
           25,
           26,
           29,
           39,
           41,
           44,
           47,
           27,
           28,
           31,
           32,
           34,
           35,
           36,
           37,
           38,
           40,
           43,
           46,
           25,
           30,
           32,
           33,
           34,
           39,
           27,
           29,
           32,
           34,
           35,
           36,
           37,
           38,
           39,
           40,
           41,
           42,
           43,
           45,
           28,
           29,
           31,
           35,
           36,
           40,
           41,
           43,
           46,
           28,
           29,
           31,
           32,
           34,
           35,
           46,
           28,
           29,
           34,
           35,
           36,
           40,
           46,
           26,
           29,
           34,
           35,
           39,
           41,
           47,
           27,
           29,
           30,
           32,
           34,
           37,
           38,
           42,
           44,
           45,
           25,
           27,
           32,
           35,
           36,
           38,
           40,
           41,
           42,
           45,
           25,
           27,
           32,
           34,
           37,
           38,
           40,
           41,
           44,
           45,
           25,
           27,
           30,
           32,
           33,
           34,
           38,
           39,
           41,
           42,
           44,
           25,
           30,
           32,
           34,
           37,
           38,
           40,
           41,
           42,
           44,
           29,
           31,
           34,
           36,
           37,
           40,
           46,
           26,
           29,
           34,
           37,
           39,
           24,
           25,
           32,
           43,
           46,
           25,
           27,
           32,
           37,
           40,
           24,
           39,
           24,
           48,
           70,
           51,
           53,
           58,
           59,
           60,
           61,
           62,
           66,
           69,
           49,
           50,
           53,
           55,
           65,
           70,
           51,
           53,
           56,
           59,
           61,
           62,
           66,
           69,
           52,
           54,
           56,
           58,
           63,
           65,
           49,
           50,
           52,
           53,
           54,
           55,
           56,
           58,
           62,
           63,
           65,
           67,
           68,
           49,
           50,
           55,
           57,
           58,
           60,
           63,
           64,
           48,
           49,
           52,
           54,
           56,
           63,
           68,
           49,
           50,
           56,
           58,
           60,
           61,
           62,
           66,
           69,
           57,
           64,
           49,
           50,
           54,
           55,
           58,
           68,
           49,
           50,
           51,
           53,
           54,
           56,
           62,
           63,
           65,
           66,
           50,
           51,
           53,
           56,
           63,
           65,
           69,
           49,
           50,
           51,
           53,
           58,
           59,
           60,
           61,
           62,
           66,
           69,
           50,
           51,
           53,
           58,
           59,
           61,
           62,
           66,
           69,
           50,
           53,
           55,
           56,
           57,
           58,
           60,
           61,
           62,
           64,
           66,
           50,
           51,
           53,
           56,
           59,
           61,
           62,
           65,
           66,
           49,
           50,
           51,
           54,
           56,
           57,
           58,
           63,
           50,
           53,
           56,
           57,
           58,
           61,
           62,
           65,
           50,
           53,
           54,
           56,
           63,
           50,
           51,
           53,
           59,
           60,
           61,
           62,
           66,
           50,
           51,
           53,
           61,
           62,
           65,
           69,
           49,
           50,
           52,
           54,
           63,
           68,
           67,
           71,
           76,
           72,
           74,
           76,
           77,
           81,
           82,
           83,
           85,
           86,
           88,
           90,
           95,
           72,
           74,
           75,
           76,
           77,
           78,
           79,
           81,
           82,
           86,
           87,
           88,
           89,
           90,
           92,
           93,
           94,
           95,
           96,
           74,
           75,
           78,
           79,
           80,
           82,
           84,
           89,
           90,
           93,
           71,
           72,
           81,
           83,
           85,
           90,
           93,
           94,
           95,
           73,
           74,
           75,
           76,
           78,
           79,
           80,
           81,
           82,
           84,
           86,
           89,
           91,
           93,
           96,
           72,
           74,
           81,
           82,
           83,
           85,
           86,
           88,
           90,
           93,
           94,
           95,
           73,
           76,
           81,
           87,
           90,
           91,
           92,
           72,
           74,
           75,
           78,
           79,
           82,
           85,
           86,
           89,
           90,
           93,
           95,
           96,
           73,
           74,
           76,
           77,
           81,
           82,
           87,
           94,
           96,
           74,
           77,
           79,
           81,
           82,
           85,
           87,
           88,
           90,
           92,
           96,
           74,
           75,
           77,
           78,
           79,
           80,
           84,
           89,
           91,
           74,
           75,
           77,
           78,
           79,
           82,
           87,
           88,
           91,
           96,
           74,
           75,
           77,
           78,
           79,
           82,
           84,
           89,
           90,
           91,
           92,
           93,
           96,
           74,
           75,
           77,
           78,
           79,
           80,
           82,
           83,
           85,
           86,
           88,
           89,
           92,
           96,
           72,
           74,
           78,
           79,
           83,
           86,
           90,
           93,
           94,
           95,
           73,
           77,
           74,
           76,
           78,
           79,
           80,
           82,
           83,
           86,
           88,
           90,
           91,
           93,
           94,
           95,
           74,
           75,
           77,
           79,
           80,
           82,
           84,
           89,
           90,
           97,
           72,
           74,
           83,
           85,
           74,
           78,
           80,
           89,
           92,
           93,
           71,
           98,
           99,
           100,
           101,
           102,
           103,
           104,
           105,
           106,
           107,
           108,
           112,
           114,
           115,
           120,
           121,
           122,
           123,
           98,
           109,
           110,
           112,
           114,
           114,
           115,
           114,
           114,
           98,
           107,
           114,
           115,
           116,
           117,
           118,
           114,
           114,
           98,
           114,
           114,
           98,
           109,
           114,
           115,
           116,
           114,
           98,
           106,
           109,
           114,
           114,
           98,
           114,
           115,
           109,
           114,
           114,
           114,
           114,
           115,
           98,
           114,
           115,
           119,
           98,
           114,
           114,
           109,
           114,
           112,
           114,
           109,
           112,
           114,
           115,
           114,
           98,
           111,
           113,
           114
          ],
          "value": [
           71,
           2,
           1,
           5,
           14,
           2,
           39,
           1,
           4,
           8,
           19,
           5,
           5,
           10,
           1,
           1,
           3,
           2,
           6,
           3,
           1,
           7,
           1,
           22,
           4,
           30,
           1,
           1,
           1,
           3,
           1,
           12,
           8,
           4,
           16,
           4,
           2,
           8,
           1,
           2,
           1,
           41,
           1,
           4,
           2,
           1,
           5,
           1,
           1,
           1,
           2,
           1,
           8,
           12,
           8,
           4,
           8,
           2,
           1,
           14,
           22,
           2,
           3,
           1,
           7,
           3,
           1,
           1,
           11,
           8,
           6,
           3,
           3,
           1,
           2,
           1,
           1,
           2,
           4,
           11,
           1,
           10,
           3,
           6,
           1,
           2,
           9,
           4,
           6,
           17,
           2,
           1,
           5,
           8,
           2,
           10,
           10,
           4,
           6,
           6,
           1,
           2,
           12,
           7,
           7,
           4,
           3,
           1,
           1,
           6,
           2,
           1,
           9,
           1,
           5,
           3,
           2,
           3,
           2,
           1,
           14,
           3,
           1,
           2,
           3,
           3,
           3,
           1,
           3,
           1,
           9,
           1,
           2,
           4,
           1,
           2,
           7,
           2,
           1,
           5,
           1,
           1,
           2,
           1,
           3,
           1,
           4,
           2,
           3,
           2,
           6,
           3,
           1,
           1,
           1,
           1,
           1,
           3,
           3,
           1,
           4,
           1,
           4,
           2,
           2,
           2,
           1,
           1,
           1,
           2,
           2,
           1,
           1,
           1,
           2,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           1,
           67,
           7,
           2,
           1,
           1,
           18,
           16,
           10,
           4,
           2,
           1,
           2,
           1,
           6,
           43,
           1,
           1,
           11,
           4,
           9,
           11,
           11,
           3,
           1,
           2,
           12,
           21,
           2,
           1,
           13,
           1,
           15,
           1,
           4,
           1,
           10,
           1,
           2,
           2,
           1,
           2,
           4,
           1,
           4,
           2,
           1,
           1,
           25,
           7,
           6,
           2,
           2,
           1,
           1,
           34,
           7,
           1,
           1,
           1,
           1,
           7,
           1,
           15,
           8,
           2,
           7,
           2,
           2,
           15,
           29,
           33,
           1,
           1,
           1,
           5,
           2,
           3,
           3,
           2,
           7,
           1,
           8,
           5,
           2,
           7,
           1,
           3,
           7,
           5,
           11,
           5,
           2,
           2,
           1,
           4,
           2,
           7,
           4,
           1,
           2,
           4,
           6,
           1,
           2,
           3,
           15,
           2,
           1,
           6,
           2,
           1,
           3,
           1,
           4,
           5,
           1,
           1,
           2,
           3,
           10,
           1,
           1,
           1,
           2,
           2,
           6,
           5,
           6,
           1,
           1,
           1,
           2,
           1,
           4,
           11,
           1,
           1,
           1,
           1,
           2,
           2,
           5,
           2,
           2,
           1,
           2,
           5,
           3,
           1,
           3,
           1,
           5,
           2,
           6,
           1,
           3,
           1,
           6,
           1,
           1,
           1,
           2,
           2,
           3,
           2,
           2,
           3,
           1,
           1,
           3,
           2,
           1,
           3,
           1,
           4,
           14,
           67,
           1,
           2,
           1,
           1,
           1,
           18,
           1,
           7,
           16,
           2,
           9,
           6,
           1,
           1,
           7,
           4,
           1,
           1,
           4,
           3,
           2,
           6,
           3,
           2,
           3,
           2,
           3,
           4,
           3,
           1,
           1,
           3,
           3,
           13,
           6,
           2,
           5,
           2,
           12,
           6,
           1,
           2,
           1,
           32,
           1,
           6,
           1,
           1,
           1,
           2,
           6,
           1,
           2,
           5,
           1,
           7,
           6,
           12,
           1,
           1,
           1,
           1,
           2,
           5,
           2,
           2,
           11,
           1,
           5,
           1,
           10,
           2,
           7,
           1,
           2,
           2,
           1,
           6,
           8,
           32,
           1,
           1,
           3,
           1,
           1,
           3,
           5,
           3,
           8,
           4,
           4,
           1,
           5,
           4,
           1,
           6,
           1,
           1,
           10,
           3,
           1,
           3,
           1,
           2,
           20,
           3,
           1,
           2,
           10,
           2,
           5,
           4,
           4,
           1,
           5,
           1,
           7,
           2,
           3,
           2,
           2,
           3,
           6,
           10,
           8,
           2,
           7,
           1,
           1,
           15,
           1,
           2,
           3,
           3,
           4,
           8,
           5,
           2,
           8,
           1,
           2,
           5,
           1,
           5,
           6,
           1,
           2,
           4,
           1,
           1,
           5,
           3,
           1,
           2,
           3,
           3,
           4,
           2,
           1,
           2,
           3,
           1,
           5,
           1,
           2,
           1,
           2,
           1,
           1,
           5,
           3,
           3,
           14,
           2,
           31,
           1,
           1,
           2,
           2,
           1,
           1,
           2,
           1,
           2,
           1,
           1,
           1,
           1,
           1,
           2,
           1,
           1,
           4,
           2,
           1,
           1,
           3,
           1,
           1,
           15,
           5,
           1,
           3,
           2,
           2,
           2,
           2,
           1,
           2,
           2,
           8,
           5,
           1,
           1,
           28,
           5,
           1,
           3,
           9,
           2,
           1,
           2,
           3,
           1,
           1,
           1,
           7,
           3,
           2,
           4,
           9,
           1,
           12,
           30,
           49,
           1,
           41,
           40,
           9,
           1,
           25,
           1,
           1,
           1,
           1,
           39,
           39,
           1,
           36,
           34,
           4,
           2,
           25,
           2,
           1,
           32,
           2,
           1,
           3,
           24,
           29,
           2,
           23,
           2,
           1,
           26,
           27,
           26,
           24,
           1,
           3,
           16,
           4,
           1,
           1,
           23,
           23,
           1,
           22,
           1,
           21,
           1,
           2,
           15,
           1,
           16,
           2,
           1,
           11,
           1
          ]
         },
         "node": {
          "color": [
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728",
           "#9467bd",
           "#8c564b",
           "#e377c2",
           "#7f7f7f",
           "#bcbd22",
           "#17becf",
           "#1f77b4",
           "#ff7f0e",
           "#2ca02c",
           "#d62728"
          ],
          "label": [
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           11,
           12,
           13,
           14,
           15,
           16,
           17,
           18,
           19,
           20,
           21,
           22,
           23,
           24,
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           11,
           12,
           13,
           14,
           15,
           16,
           17,
           18,
           19,
           20,
           21,
           22,
           23,
           24,
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           11,
           12,
           13,
           14,
           15,
           16,
           17,
           18,
           19,
           20,
           21,
           22,
           23,
           1,
           2,
           3,
           4,
           5,
           6,
           7,
           8,
           9,
           10,
           11,
           12,
           13,
           14,
           15,
           16,
           17,
           18,
           19,
           20,
           21,
           22,
           23,
           24,
           25,
           26,
           27,
           "Blood: Erythrocyte",
           "Fibroblast: Pia 2",
           "Gastrulation: Mesoderm",
           "Glioblast: Forebrain",
           "Glioblast: Forebrain astrocyte",
           "Glioblast: Midbrain",
           "Glioblast: Mixed region",
           "Glioblast: Mixed region astrocytes",
           "Glioblast: PreOPC",
           "Immune: Non-cycling microglia",
           "Neuroblast: Forebrain GABAergic",
           "Neuroblast: Forebrain glutamatergic",
           "Neuroblast: Hindbrain glutamatergic",
           "Neuroblast: Mixed region glutamatergic",
           "Neuroblast: Neuronal intermediate progenitor",
           "Neuron: Cajal-Retzius",
           "Neuron: Cortical or hippocampal glutamatergic",
           "Neuron: Forebrain GABAergic",
           "Neuron: Forebrain glutamatergic",
           "Neuron: Midbrain GABAergic",
           "Neuron: Mixed region GABAergic",
           "Neuron: Spinal cord glutamatergic",
           "Olfactory ensheathing cell: Olfactory ensheathing cell",
           "Oligodendrocyte: Oligodendrocyte precursor cell",
           "Radial glia: Dorsal forebrain",
           "Radial glia: Forebrain"
          ],
          "line": {
           "color": "black",
           "width": 0.5
          },
          "pad": 15,
          "thickness": 20
         },
         "type": "sankey"
        }
       ],
       "layout": {
        "annotations": [
         {
          "showarrow": false,
          "text": "monocle",
          "x": 0,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "seurat",
          "x": 1,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "scanpy",
          "x": 2,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "scvi-tools",
          "x": 3,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         },
         {
          "showarrow": false,
          "text": "celltypist",
          "x": 4,
          "xref": "x",
          "y": 1.05,
          "yref": "paper"
         }
        ],
        "font": {
         "size": 10
        },
        "height": 550,
        "plot_bgcolor": "rgba(0,0,0,0)",
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "E175 - matching celltypist labels"
        },
        "width": 1400,
        "xaxis": {
         "showgrid": false,
         "visible": false,
         "zeroline": false
        },
        "yaxis": {
         "showgrid": false,
         "visible": false,
         "zeroline": false
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for tuning in PARAMS_TUNING:\n",
    "    print('------------------------------')\n",
    "    print(f'{DATASET_NAME} - matching {tuning} labels' if tuning != 'default' else f'{DATASET_NAME} - default labels')\n",
    "    \n",
    "    # concat tools labels\n",
    "    labels_df = pd.read_csv(f'{DIR}{DATASET_NAME}/COTAN/{tuning}/clustering_labels.csv', index_col=0)\n",
    "    labels_df.rename(columns={\"cluster\": \"cluster_COTAN\"}, inplace=True)\n",
    "    for tool in [t for t in TOOLS if t != 'COTAN']:\n",
    "        tool_labels_df = pd.read_csv(f'{DIR}{DATASET_NAME}/{tool}/{tuning}/clustering_labels.csv', index_col=0)\n",
    "        labels_df = labels_df.merge(tool_labels_df, how='inner', on='cell')\n",
    "        labels_df.rename(columns={\"cluster\": f\"cluster_{tool}\"}, inplace=True)\n",
    "    \n",
    "    # load and concat celltypist labels\n",
    "    celltypist_df = pd.read_csv(f'{DIR}{DATASET_NAME}/E17_5_Devel_Mouse_Brain_predicted_labels.csv', index_col=0)\n",
    "    celltypist_df = labels_df.merge(celltypist_df, how='inner', on='cell')\n",
    "    celltypist_df.rename(columns={\"predicted_labels\": f\"cluster_celltypist\"}, inplace=True)\n",
    "    \n",
    "    display(labels_df)\n",
    "\n",
    "    # read dataset\n",
    "    adata = sc.read_csv(\n",
    "        f'./data/{DATASET_NAME}/CorticalCells_GSM2861514_E175_cleaned.csv',\n",
    "        first_column_names=None\n",
    "    )\n",
    "    adata = adata.T\n",
    "    # keep only labelled cells\n",
    "    adata.var_names_make_unique()\n",
    "    subset_cells = adata.obs_names.isin(labels_df.index)\n",
    "    adata = adata[subset_cells, :]\n",
    "\n",
    "    # compute silhouette score\n",
    "    silhouette = {}\n",
    "    for tool in TOOLS:\n",
    "        try:\n",
    "            silhouette[tool] = silhouette_score(adata.X, labels_df[f'cluster_{tool}'])\n",
    "        except:\n",
    "            silhouette[tool] = np.nan\n",
    "    if tuning=='celltypist':\n",
    "        silhouette['celltypist'] = silhouette_score(adata.X, celltypist_df[f'cluster_celltypist'])\n",
    "    silhouette_df = pd.DataFrame(silhouette, index=[0])\n",
    "    silhouette_df.to_csv(f'{DIR}{DATASET_NAME}/{tuning}_silhouette.csv')\n",
    "    silhouette_df.to_latex(f'{DIR}{DATASET_NAME}/{tuning}_silhouette.tex')\n",
    "\n",
    "    # compute scores comparing each tool labels with celltypist labels\n",
    "    if tuning == 'celltypist' or tuning == 'default':\n",
    "        compute_scores(DIR, DATASET_NAME, celltypist_df, tuning, 'celltypist')\n",
    "        labels = []\n",
    "        labels_titles = []\n",
    "        for tool in TOOLS:\n",
    "            labels.append(celltypist_df[f'cluster_{tool}'].to_list())\n",
    "            labels_titles.append(tool)\n",
    "        labels.append(celltypist_df[f'cluster_celltypist'].to_list())#.map(celltypist_mapping_df['go'].to_dict()).to_list())\n",
    "        labels_titles.append('celltypist')\n",
    "        title = f'{DATASET_NAME} - matching {tuning} labels' if tuning != 'default' else f'{DATASET_NAME} - default labels'\n",
    "        sankey_plot(\n",
    "            labels=labels,\n",
    "            labels_titles=labels_titles,\n",
    "            title=title, path=f'{DIR}{DATASET_NAME}/{tuning}_celltypist.html',\n",
    "            height=550,\n",
    "            width=1400\n",
    "        )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}