preprocessing/S3a_Pan_neuro_integrated_pySCENIC_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "69fcd067",
   "metadata": {},
   "source": [
    "# Emma Pan Neuro (Control + ND75KD) - pySCENIC pipeline (Embedded version)\n",
    "\n",
    "**Author:** Vincent Gardeux\n",
    "\n",
    "**Date Created:** 03/06/2024\n",
    "\n",
    "**Date Modified:** 25/10/2024"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cc57e0c",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b9c1fa0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fix OPENBLAS Warnings\n",
    "import os\n",
    "default_n_threads = 12\n",
    "os.environ['OPENBLAS_NUM_THREADS'] = f\"{default_n_threads}\"\n",
    "os.environ['MKL_NUM_THREADS'] = f\"{default_n_threads}\"\n",
    "os.environ['OMP_NUM_THREADS'] = f\"{default_n_threads}\"\n",
    "\n",
    "# import dependencies\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import h5py\n",
    "import pickle\n",
    "import pytz\n",
    "\n",
    "from datetime import datetime\n",
    "from arboreto.utils import load_tf_names\n",
    "from arboreto.algo import grnboost2\n",
    "from distributed import Client, LocalCluster\n",
    "\n",
    "from ctxcore.rnkdb import FeatherRankingDatabase as RankingDatabase\n",
    "from pyscenic.binarization import binarize\n",
    "from pyscenic.utils import modules_from_adjacencies\n",
    "from pyscenic.prune import prune2df, df2regulons\n",
    "from pyscenic.aucell import aucell"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cd961d0",
   "metadata": {},
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "beaef2b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AAACCCAAGGTGATAT-1_ctrl</th>\n",
       "      <th>AAACCCACAAATAGCA-1_ctrl</th>\n",
       "      <th>AAACCCACAACAAAGT-1_ctrl</th>\n",
       "      <th>AAACCCACACTCATAG-1_ctrl</th>\n",
       "      <th>AAACCCACAGAGAGGG-1_ctrl</th>\n",
       "      <th>AAACCCACAGCCTATA-1_ctrl</th>\n",
       "      <th>AAACCCAGTACCTTCC-1_ctrl</th>\n",
       "      <th>AAACCCAGTACTGCCG-1_ctrl</th>\n",
       "      <th>AAACCCAGTGTTCGTA-1_ctrl</th>\n",
       "      <th>AAACCCAGTTCAGGTT-1_ctrl</th>\n",
       "      <th>...</th>\n",
       "      <th>TTTGTTGCAAGCACCC-1_ndkd</th>\n",
       "      <th>TTTGTTGCAGTTACCA-1_ndkd</th>\n",
       "      <th>TTTGTTGGTGAGATAT-1_ndkd</th>\n",
       "      <th>TTTGTTGGTGTCACAT-1_ndkd</th>\n",
       "      <th>TTTGTTGGTTAAGAAC-1_ndkd</th>\n",
       "      <th>TTTGTTGGTTTGGAGG-1_ndkd</th>\n",
       "      <th>TTTGTTGTCCGTTTCG-1_ndkd</th>\n",
       "      <th>TTTGTTGTCCTTCAGC-1_ndkd</th>\n",
       "      <th>TTTGTTGTCGAACGCC-1_ndkd</th>\n",
       "      <th>TTTGTTGTCGTGTTCC-1_ndkd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gfzf</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Osi24</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>msps</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CG6013</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CR44091</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mitf</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pur-alpha</th>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gw</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CR44027</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sox102F</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16818 rows × 23179 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           AAACCCAAGGTGATAT-1_ctrl  AAACCCACAAATAGCA-1_ctrl  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      0.0   \n",
       "CG6013                         0.0                      0.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           0.0                      0.0   \n",
       "Pur-alpha                      3.0                      6.0   \n",
       "gw                             2.0                      1.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                      0.0   \n",
       "\n",
       "           AAACCCACAACAAAGT-1_ctrl  AAACCCACACTCATAG-1_ctrl  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      0.0   \n",
       "CG6013                         0.0                      0.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           0.0                      0.0   \n",
       "Pur-alpha                      3.0                      1.0   \n",
       "gw                             6.0                      0.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                      0.0   \n",
       "\n",
       "           AAACCCACAGAGAGGG-1_ctrl  AAACCCACAGCCTATA-1_ctrl  \\\n",
       "gfzf                           0.0                      1.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      0.0   \n",
       "CG6013                         0.0                      0.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           0.0                      2.0   \n",
       "Pur-alpha                      0.0                     15.0   \n",
       "gw                             0.0                      6.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                     15.0   \n",
       "\n",
       "           AAACCCAGTACCTTCC-1_ctrl  AAACCCAGTACTGCCG-1_ctrl  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      0.0   \n",
       "CG6013                         1.0                      0.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           4.0                      0.0   \n",
       "Pur-alpha                     11.0                      3.0   \n",
       "gw                             2.0                      3.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                      0.0   \n",
       "\n",
       "           AAACCCAGTGTTCGTA-1_ctrl  AAACCCAGTTCAGGTT-1_ctrl  ...  \\\n",
       "gfzf                           0.0                      0.0  ...   \n",
       "Osi24                          0.0                      0.0  ...   \n",
       "msps                           0.0                      0.0  ...   \n",
       "CG6013                         0.0                      0.0  ...   \n",
       "CR44091                        0.0                      0.0  ...   \n",
       "...                            ...                      ...  ...   \n",
       "Mitf                           0.0                      0.0  ...   \n",
       "Pur-alpha                      0.0                      0.0  ...   \n",
       "gw                             1.0                      1.0  ...   \n",
       "CR44027                        0.0                      0.0  ...   \n",
       "Sox102F                        0.0                      0.0  ...   \n",
       "\n",
       "           TTTGTTGCAAGCACCC-1_ndkd  TTTGTTGCAGTTACCA-1_ndkd  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      0.0   \n",
       "CG6013                         0.0                      0.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           5.0                      1.0   \n",
       "Pur-alpha                      2.0                      3.0   \n",
       "gw                             3.0                      4.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        3.0                      0.0   \n",
       "\n",
       "           TTTGTTGGTGAGATAT-1_ndkd  TTTGTTGGTGTCACAT-1_ndkd  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           1.0                      0.0   \n",
       "CG6013                         0.0                      0.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           0.0                      2.0   \n",
       "Pur-alpha                      5.0                      1.0   \n",
       "gw                             6.0                      0.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                      0.0   \n",
       "\n",
       "           TTTGTTGGTTAAGAAC-1_ndkd  TTTGTTGGTTTGGAGG-1_ndkd  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      1.0   \n",
       "CG6013                         0.0                      1.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           0.0                      0.0   \n",
       "Pur-alpha                      1.0                      2.0   \n",
       "gw                             4.0                      2.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                      0.0   \n",
       "\n",
       "           TTTGTTGTCCGTTTCG-1_ndkd  TTTGTTGTCCTTCAGC-1_ndkd  \\\n",
       "gfzf                           0.0                      0.0   \n",
       "Osi24                          0.0                      0.0   \n",
       "msps                           0.0                      0.0   \n",
       "CG6013                         0.0                      1.0   \n",
       "CR44091                        0.0                      0.0   \n",
       "...                            ...                      ...   \n",
       "Mitf                           1.0                      0.0   \n",
       "Pur-alpha                      1.0                      6.0   \n",
       "gw                             2.0                      6.0   \n",
       "CR44027                        0.0                      0.0   \n",
       "Sox102F                        0.0                      2.0   \n",
       "\n",
       "           TTTGTTGTCGAACGCC-1_ndkd  TTTGTTGTCGTGTTCC-1_ndkd  \n",
       "gfzf                           0.0                      0.0  \n",
       "Osi24                          0.0                      0.0  \n",
       "msps                           0.0                      0.0  \n",
       "CG6013                         0.0                      0.0  \n",
       "CR44091                        0.0                      0.0  \n",
       "...                            ...                      ...  \n",
       "Mitf                           0.0                      0.0  \n",
       "Pur-alpha                      1.0                      0.0  \n",
       "gw                             8.0                      0.0  \n",
       "CR44027                        0.0                      0.0  \n",
       "Sox102F                        2.0                      0.0  \n",
       "\n",
       "[16818 rows x 23179 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# [Input] Fixed gene annotation (for SCENIC feather file compatibility)\n",
    "genome_data = pd.read_csv(\"./data/fixed_gene_annotation.tsv\", sep = \"\\t\", na_filter=False)\n",
    "\n",
    "# [Input] Loom file to use\n",
    "f_loom_path_scenic = \"./data/Pan_neuro_integrated.loom\"\n",
    "# Open Loom file in reading mode\n",
    "f = h5py.File(f_loom_path_scenic, 'r')\n",
    "f_m = f[\"/matrix\"][:,:]\n",
    "f_gene_names = f[\"/row_attrs/Gene\"].asstr()[:]\n",
    "f_ensembl_names = f[\"/row_attrs/Accession\"].asstr()[:]\n",
    "f_cell_names = f[\"/col_attrs/CellID\"].asstr()[:]\n",
    "f.close()\n",
    "\n",
    "# Update gene names using fixing table\n",
    "ensembl_to_gene_dict = genome_data.set_index('Ensembl')['Name'].to_dict()\n",
    "f_gene_names_updated = np.array([\n",
    "    ensembl_to_gene_dict[ensembl] if ensembl in ensembl_to_gene_dict else gene\n",
    "    for gene, ensembl in zip(f_gene_names, f_ensembl_names)\n",
    "])\n",
    "\n",
    "# Create expression matrix\n",
    "ex_matrix = pd.DataFrame(f_m, columns = f_cell_names, index = f_gene_names_updated)\n",
    "ex_matrix = ex_matrix.loc[f_gene_names_updated, :]\n",
    "\n",
    "# [Input] Transcription factors list (SCENIC step 1: GRNBoost2)\n",
    "#f_tfs = \"./data/allTFs_dmel.txt\" # From pySCENIC github [Discarded because contains genes that are NOT TFs]\n",
    "f_tfs = \"./data/FBgg0000745_TF_Flybase.filtered.txt\" # From Flybase\n",
    "# Derive list of Transcription Factors(TF)\n",
    "tf_names = load_tf_names(f_tfs)\n",
    "\n",
    "# [Output] Adjacency matrix (SCENIC step 1: GRNBoost2)\n",
    "adj_matrix = \"./data/Pan_neuro_integrated_adj.csv\"\n",
    "\n",
    "# [Input] Ranking databases (SCENIC step 2-3: cisTarget)\n",
    "f_db_names = [\"./data/dm6_v10_clust.genes_vs_motifs.rankings.feather\"] # From pySCENIC db: cistopic_flybase_r6.02/mc_v10_clust/\n",
    "dbs = [RankingDatabase(fname=f_name, name=os.path.basename(f_name)) for f_name in f_db_names]\n",
    "\n",
    "# [Input] Motif databases (SCENIC step 2-3: cisTarget)\n",
    "f_motif_path = \"./data/motifs-v10-nr.flybase-m0.00001-o0.0.tbl\" # From pySCENIC db: cistopic_flybase_r6.02/mc_v10_clust/\n",
    "\n",
    "# [Output] Regulons (SCENIC step 2-3: cisTarget)\n",
    "f_motifs_path = \"./data/Pan_neuro_integrated_motifs.tsv\"\n",
    "f_modules_path = \"./data/Pan_neuro_integrated_modules.tsv\"\n",
    "f_regulons_path = \"./data/Pan_neuro_integrated_regulons.tsv\"\n",
    "f_regulons_aucell_path = \"./data/Pan_neuro_integrated_regulons_aucell.tsv\"\n",
    "f_regulons_binarized_aucell_path = \"./data/Pan_neuro_integrated_regulons_aucell_binarized.tsv\"\n",
    "f_regulons_binarization_thresholds_aucell_path = \"./data/Pan_neuro_integrated_regulons_aucell_binarization_thresholds.tsv\"\n",
    "\n",
    "# Restrict matrix to feather genes\n",
    "ranking_feather = pd.read_feather(f_db_names[0])\n",
    "overlap_values = ex_matrix.index[pd.Series(ex_matrix.index).isin(ranking_feather.columns)].unique()\n",
    "ex_matrix = ex_matrix.loc[overlap_values, :]\n",
    "\n",
    "ex_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bd70d34",
   "metadata": {},
   "source": [
    "# SCENIC steps\n",
    "\n",
    "## STEP 1: Gene regulatory network inference, and generation of co-expression modules\n",
    "\n",
    "### 1.a. GRN inference using the GRNBoost2 algorithm\n",
    "\n",
    "In the initial phase of the pySCENIC pipeline the single cell expression profiles are used to infer co-expression modules from.\n",
    "\n",
    "Run GRNboost from arboreto to infer co-expression modules\n",
    "\n",
    "The arboreto package is used for this phase of the pipeline.\n",
    "\n",
    "*Output:* List of adjacencies between a TF and its targets."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4418d077",
   "metadata": {},
   "source": [
    "Run GRNBoost2 algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a355d9ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start time: 14:29:31\n",
      "preparing dask client\n",
      "parsing input\n",
      "creating dask graph\n",
      "12 partitions\n",
      "computing dask graph\n",
      "not shutting down client, client was created externally\n",
      "finished\n",
      "End time: 15:15:18\n",
      "Running time: 0:45:46.592788\n"
     ]
    }
   ],
   "source": [
    "start = datetime.now(pytz.timezone('Europe/Paris'))\n",
    "print(\"Start time:\", start.strftime(\"%H:%M:%S\"))\n",
    "\n",
    "# Prepare the multithreading\n",
    "cluster = LocalCluster(name='grn_call', dashboard_address=\":12345\", n_workers=default_n_threads, threads_per_worker=8)\n",
    "client = Client(cluster)\n",
    "\n",
    "# Here I run the function within the package (no CLI)\n",
    "adjacencies = grnboost2(expression_data=ex_matrix.transpose(), tf_names=tf_names, seed=42, verbose=True, client_or_address=client)\n",
    "            \n",
    "# Shutting down cluster\n",
    "client.close()\n",
    "cluster.close()\n",
    "    \n",
    "end = datetime.now(pytz.timezone('Europe/Paris'))\n",
    "print(\"End time:\", end.strftime(\"%H:%M:%S\"))\n",
    "print(\"Running time:\", (end - start))\n",
    "\n",
    "# Note: Takes ~50mn with n_workers=12, threads_per_worker=8 => With reduced matrix to overlapping genes between .feather file and our matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52d76b44",
   "metadata": {},
   "source": [
    "Read in the adjacencies matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7da47991",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TF</th>\n",
       "      <th>target</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>bi</td>\n",
       "      <td>CR32773</td>\n",
       "      <td>8.560182e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>CG9650</td>\n",
       "      <td>CR44357</td>\n",
       "      <td>4.424130e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>salm</td>\n",
       "      <td>salr</td>\n",
       "      <td>4.024105e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>br</td>\n",
       "      <td>Mur2B</td>\n",
       "      <td>3.896239e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>474</th>\n",
       "      <td>Vsx2</td>\n",
       "      <td>Vsx1</td>\n",
       "      <td>3.817732e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>rgr</td>\n",
       "      <td>Sox21a</td>\n",
       "      <td>2.229785e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>REPTOR</td>\n",
       "      <td>CG9416</td>\n",
       "      <td>1.459898e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>kni</td>\n",
       "      <td>CG30271</td>\n",
       "      <td>1.115953e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>mamo</td>\n",
       "      <td>CCAP</td>\n",
       "      <td>5.907672e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>sd</td>\n",
       "      <td>Orcokinin</td>\n",
       "      <td>1.184474e-20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3118632 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         TF     target    importance\n",
       "395      bi    CR32773  8.560182e+02\n",
       "478  CG9650    CR44357  4.424130e+02\n",
       "573    salm       salr  4.024105e+02\n",
       "491      br      Mur2B  3.896239e+02\n",
       "474    Vsx2       Vsx1  3.817732e+02\n",
       "..      ...        ...           ...\n",
       "300     rgr     Sox21a  2.229785e-17\n",
       "125  REPTOR     CG9416  1.459898e-17\n",
       "239     kni    CG30271  1.115953e-17\n",
       "498    mamo       CCAP  5.907672e-20\n",
       "451      sd  Orcokinin  1.184474e-20\n",
       "\n",
       "[3118632 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adjacencies.to_csv(adj_matrix, index=False, sep=',')\n",
    "#adjacencies = pd.read_csv(adj_matrix, sep=',', na_filter=False) # If na_filter=True, the nan gene is detected as NaN\n",
    "adjacencies"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a6d51d7",
   "metadata": {},
   "source": [
    "## STEP 2-3: Regulon prediction aka cisTarget\n",
    "\n",
    "*Output:* List of adjacencies between a TF and its targets.\n",
    "\n",
    "### 2.a. Running regulon prediction using cisTarget\n",
    "\n",
    "Here, we use the --mask_dropouts option, which affects how the correlation between TF and target genes is calculated during module creation. It is important to note that prior to pySCENIC v0.9.18, the default behavior was to mask dropouts, while in v0.9.18 and later, the correlation is performed using the entire set of cells (including those with zero expression). When using the modules_from_adjacencies function directly in python instead of via the command line, the rho_mask_dropouts option can be used to control this."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12ea2e9f",
   "metadata": {},
   "source": [
    "**Note:** Here I tried with and without the `rho_mask_dropouts=True` option. Without it, it generates 46 regulons. With it, 81 regulons. So I keep it."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9fb8875",
   "metadata": {},
   "source": [
    "**Update:** Now 99 regulons with fixed gene names, and `rho_mask_dropouts=True` option"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3b08b97",
   "metadata": {},
   "source": [
    "**Update 2:** Now 108 regulons with fixed gene names, restricted to feather genes, and `rho_mask_dropouts=True` option. It contains dati and crc, but not repo anymore :("
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f740570",
   "metadata": {},
   "source": [
    "**Update 3:** Now 764 regulons with filter_for_annotation=False ( no filtering on pruning )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2814aff5",
   "metadata": {},
   "source": [
    "**Update 4:** Now 579 regulons when using TF list from Flybase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "178d47d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "2024-05-15 13:15:29,670 - pyscenic.utils - INFO - Calculating Pearson correlations.\n",
      "\n",
      "2024-05-15 13:15:30,372 - pyscenic.utils - WARNING - Note on correlation calculation: the default behaviour for calculating the correlations has changed after pySCENIC verion 0.9.16. Previously, the default was to calculate the correlation between a TF and target gene using only cells with non-zero expression values (mask_dropouts=True). The current default is now to use all cells to match the behavior of the R verision of SCENIC. The original settings can be retained by setting 'rho_mask_dropouts=True' in the modules_from_adjacencies function, or '--mask_dropouts' from the CLI.\n",
      "\tDropout masking is currently set to [True].\n",
      "\n",
      "2024-05-15 13:15:52,271 - pyscenic.utils - INFO - Creating modules.\n"
     ]
    }
   ],
   "source": [
    "modules = list(modules_from_adjacencies(adjacencies, ex_matrix.transpose(), rho_mask_dropouts=True, keep_only_activating=True)) # rho_mask_dropouts=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3a51da7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Regulon</th>\n",
       "      <th>TF</th>\n",
       "      <th>TFTargetGenesCorrelation</th>\n",
       "      <th>NbMarkers</th>\n",
       "      <th>Context</th>\n",
       "      <th>NES</th>\n",
       "      <th>Markers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3104</th>\n",
       "      <td>lola</td>\n",
       "      <td>lola</td>\n",
       "      <td>activating</td>\n",
       "      <td>7496</td>\n",
       "      <td>top50perTarget</td>\n",
       "      <td>0.0</td>\n",
       "      <td>128up,14-3-3epsilon,14-3-3zeta,140up,18w,26-29...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3003</th>\n",
       "      <td>bun</td>\n",
       "      <td>bun</td>\n",
       "      <td>activating</td>\n",
       "      <td>7319</td>\n",
       "      <td>top50perTarget</td>\n",
       "      <td>0.0</td>\n",
       "      <td>128up,14-3-3epsilon,14-3-3zeta,18w,26-29-p,2mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>Regulon for lola</td>\n",
       "      <td>lola</td>\n",
       "      <td>activating</td>\n",
       "      <td>7142</td>\n",
       "      <td>weight&gt;75.0%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>roX1,Cam,ps,14-3-3zeta,Atpalpha,sbb,Vha16-1,Rp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>Regulon for bun</td>\n",
       "      <td>bun</td>\n",
       "      <td>activating</td>\n",
       "      <td>7041</td>\n",
       "      <td>weight&gt;75.0%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CG42613,zfh2,Rbfox1,ct,lola,RapGAP1,sbb,NFAT,S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939</th>\n",
       "      <td>Pdp1</td>\n",
       "      <td>Pdp1</td>\n",
       "      <td>activating</td>\n",
       "      <td>6842</td>\n",
       "      <td>top50perTarget</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14-3-3epsilon,14-3-3zeta,140up,18w,26-29-p,2mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1773</th>\n",
       "      <td>CG2889</td>\n",
       "      <td>CG2889</td>\n",
       "      <td>top5perTarget</td>\n",
       "      <td>20</td>\n",
       "      <td>activating</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CG10026,CG13164,CG14069,CG14327,CG17217,CG3106...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1761</th>\n",
       "      <td>CG17612</td>\n",
       "      <td>CG17612</td>\n",
       "      <td>top5perTarget</td>\n",
       "      <td>20</td>\n",
       "      <td>activating</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CG15725,CG16716,CG16979,CG17068,CG3740,CG4159,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1730</th>\n",
       "      <td>CG10366</td>\n",
       "      <td>CG10366</td>\n",
       "      <td>top5perTarget</td>\n",
       "      <td>20</td>\n",
       "      <td>activating</td>\n",
       "      <td>0.0</td>\n",
       "      <td>BCAS2,CG10347,CG11975,CG13151,CG14882,CG17118,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1789</th>\n",
       "      <td>CG4328</td>\n",
       "      <td>CG4328</td>\n",
       "      <td>top5perTarget</td>\n",
       "      <td>20</td>\n",
       "      <td>activating</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CG11997,CG13807,CG32150,CG33253,CG42564,CG4415...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2338</th>\n",
       "      <td>Hey</td>\n",
       "      <td>Hey</td>\n",
       "      <td>activating</td>\n",
       "      <td>20</td>\n",
       "      <td>top10perTarget</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Atx-1,CG10163,CG1227,CG13018,CG14907,CG17186,C...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3212 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Regulon       TF TFTargetGenesCorrelation NbMarkers  \\\n",
       "3104              lola     lola               activating      7496   \n",
       "3003               bun      bun               activating      7319   \n",
       "463   Regulon for lola     lola               activating      7142   \n",
       "362    Regulon for bun      bun               activating      7041   \n",
       "2939              Pdp1     Pdp1               activating      6842   \n",
       "...                ...      ...                      ...       ...   \n",
       "1773            CG2889   CG2889            top5perTarget        20   \n",
       "1761           CG17612  CG17612            top5perTarget        20   \n",
       "1730           CG10366  CG10366            top5perTarget        20   \n",
       "1789            CG4328   CG4328            top5perTarget        20   \n",
       "2338               Hey      Hey               activating        20   \n",
       "\n",
       "             Context  NES                                            Markers  \n",
       "3104  top50perTarget  0.0  128up,14-3-3epsilon,14-3-3zeta,140up,18w,26-29...  \n",
       "3003  top50perTarget  0.0  128up,14-3-3epsilon,14-3-3zeta,18w,26-29-p,2mi...  \n",
       "463     weight>75.0%  0.0  roX1,Cam,ps,14-3-3zeta,Atpalpha,sbb,Vha16-1,Rp...  \n",
       "362     weight>75.0%  0.0  CG42613,zfh2,Rbfox1,ct,lola,RapGAP1,sbb,NFAT,S...  \n",
       "2939  top50perTarget  0.0  14-3-3epsilon,14-3-3zeta,140up,18w,26-29-p,2mi...  \n",
       "...              ...  ...                                                ...  \n",
       "1773      activating  0.0  CG10026,CG13164,CG14069,CG14327,CG17217,CG3106...  \n",
       "1761      activating  0.0  CG15725,CG16716,CG16979,CG17068,CG3740,CG4159,...  \n",
       "1730      activating  0.0  BCAS2,CG10347,CG11975,CG13151,CG14882,CG17118,...  \n",
       "1789      activating  0.0  CG11997,CG13807,CG32150,CG33253,CG42564,CG4415...  \n",
       "2338  top10perTarget  0.0  Atx-1,CG10163,CG1227,CG13018,CG14907,CG17186,C...  \n",
       "\n",
       "[3212 rows x 7 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modules_df = pd.DataFrame(index = range(0, len(modules)), columns = (\"Regulon\", \"TF\", \"TFTargetGenesCorrelation\", \"NbMarkers\", \"Context\", \"NES\", \"Markers\"))\n",
    "for j in range(0, len(modules)):\n",
    "    # Setting values\n",
    "    context = list(modules[j].context)\n",
    "    modules_df[\"Regulon\"].iloc[j] = modules[j].name\n",
    "    modules_df[\"TF\"].iloc[j] = modules[j].transcription_factor\n",
    "    modules_df[\"TFTargetGenesCorrelation\"].iloc[j] = context[0]\n",
    "    modules_df[\"NbMarkers\"].iloc[j] = len(set(modules[j].gene2weight))\n",
    "    modules_df[\"Context\"].iloc[j] = context[1]\n",
    "    modules_df[\"NES\"].iloc[j] = modules[j].score\n",
    "    modules_df[\"Markers\"].iloc[j] = ','.join(list(modules[j].gene2weight))\n",
    "\n",
    "modules_df = modules_df.sort_values(by='NbMarkers', ascending=False)\n",
    "modules_df.to_csv(f_modules_path, index=False, sep = \"\\t\")\n",
    "modules_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6a6c41a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579 unique TF-modules were found ( out of 631 ). Modules with less than 20 markers were filtered out.\n"
     ]
    }
   ],
   "source": [
    "print(modules_df.TF.nunique(), \"unique TF-modules were found ( out of\",len(tf_names),\"). Modules with less than 20 markers were filtered out.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e8cd87fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start time: 15:17:15\n",
      "End time: 15:28:45\n",
      "Running time: 0:11:30.094269\n"
     ]
    }
   ],
   "source": [
    "start = datetime.now(pytz.timezone('Europe/Paris'))\n",
    "print(\"Start time:\", start.strftime(\"%H:%M:%S\"))\n",
    "\n",
    "df = prune2df(dbs, modules, f_motif_path, num_workers=default_n_threads, weighted_recovery=False, rank_threshold = 1500, nes_threshold=3, motif_similarity_fdr=0.001, auc_threshold=0.05, filter_for_annotation=False)\n",
    "    \n",
    "end = datetime.now(pytz.timezone('Europe/Paris'))\n",
    "print(\"End time:\", end.strftime(\"%H:%M:%S\"))\n",
    "print(\"Running time:\", (end - start))\n",
    "# Note: 5mn37 with num_workers=48\n",
    "# Note: 5mn29 with num_workers=12\n",
    "# Note: 5mn33 with num_workers=10\n",
    "# Note: 5mn37 with num_workers=8 ** optimal? 12mn with filter_for_annotation=False)\n",
    "# Note: 8mn40 with num_workers=6\n",
    "# Note: 9mn53 with num_workers=4\n",
    "# Note: 30mn24 with num_workers=1\n",
    "\n",
    "df.to_csv(f_motifs_path,sep = \"\\t\")\n",
    "#df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "58bac17d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579 regulons were kept, after pruning\n"
     ]
    }
   ],
   "source": [
    "print(len(set(df.index.get_level_values('TF').values)), \"regulons were kept, after pruning\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dcbce244",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pros\tTrue\n",
      "dati\tTrue\n",
      "scro\tTrue\n",
      "crc\tTrue\n",
      "repo\tTrue\n",
      "sima\tTrue\n"
     ]
    }
   ],
   "source": [
    "# Look for main regulons\n",
    "print(\"pros\", \"pros\" in df.index.get_level_values('TF').values, sep=\"\\t\")\n",
    "print(\"dati\", \"dati\" in df.index.get_level_values('TF').values, sep=\"\\t\")\n",
    "print(\"scro\", \"scro\" in df.index.get_level_values('TF').values, sep=\"\\t\")\n",
    "print(\"crc\", \"crc\" in df.index.get_level_values('TF').values, sep=\"\\t\")\n",
    "print(\"repo\", \"repo\" in df.index.get_level_values('TF').values, sep=\"\\t\")\n",
    "print(\"sima\", \"sima\" in df.index.get_level_values('TF').values, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "531b3e86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of Dataframe: 26009\n",
      "Size of Dataframe: 26009\n"
     ]
    }
   ],
   "source": [
    "print(\"Size of Dataframe:\", len(df))\n",
    "drop_indexes = []\n",
    "for j in range(0, len(df)):\n",
    "    # Setting values\n",
    "    if(len(df[\"Enrichment\"][\"TargetGenes\"][j]) == 0): drop_indexes.append(df.index[j])\n",
    "df_filtered = df[\"Enrichment\"].drop(index=drop_indexes)\n",
    "print(\"Size of Dataframe:\", len(df_filtered))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e1dbd42",
   "metadata": {},
   "source": [
    "These \"modules\" are then combined into regulons, by taking the top NES for each TF (for main Motif, and final score of regulon). All genes are bundled together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "92aab324",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create regulons from a dataframe of enriched features.\n",
      "Additional columns saved: []\n"
     ]
    }
   ],
   "source": [
    "# This dataframe can then be converted to regulons.\n",
    "regulons = df2regulons(df_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "44659b3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Regulon</th>\n",
       "      <th>TF</th>\n",
       "      <th>TFTargetGenesCorrelation</th>\n",
       "      <th>NbMarkers</th>\n",
       "      <th>Motif</th>\n",
       "      <th>NES</th>\n",
       "      <th>Markers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>471</th>\n",
       "      <td>lola(+)</td>\n",
       "      <td>lola</td>\n",
       "      <td>activating</td>\n",
       "      <td>4254</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>5.945286</td>\n",
       "      <td>CG17698,CR45736,p120ctn,jim,AGO3,Myo81F,CG4578...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>Pdp1(+)</td>\n",
       "      <td>Pdp1</td>\n",
       "      <td>activating</td>\n",
       "      <td>3819</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>5.439625</td>\n",
       "      <td>CG17698,CR45736,p120ctn,jim,AGO3,Myo81F,CG1320...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430</th>\n",
       "      <td>fs(1)h(+)</td>\n",
       "      <td>fs(1)h</td>\n",
       "      <td>activating</td>\n",
       "      <td>3814</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>6.649265</td>\n",
       "      <td>CG17698,p120ctn,jim,Myo81F,CG45781,kl-3,CG1320...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>crol(+)</td>\n",
       "      <td>crol</td>\n",
       "      <td>activating</td>\n",
       "      <td>3688</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>9.530519</td>\n",
       "      <td>CG17698,CR45736,p120ctn,jim,CG13204,CG41378,kl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>bun(+)</td>\n",
       "      <td>bun</td>\n",
       "      <td>activating</td>\n",
       "      <td>3633</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>4.974753</td>\n",
       "      <td>CG17698,CR45736,p120ctn,jim,AGO3,Myo81F,CG4578...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>amos(+)</td>\n",
       "      <td>amos</td>\n",
       "      <td>activating</td>\n",
       "      <td>20</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>4.08343</td>\n",
       "      <td>Nlg3,Pdp1,CG30463,CadN,LPCAT,Gyc88E,shakB,Frq1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>CG15696(+)</td>\n",
       "      <td>CG15696</td>\n",
       "      <td>activating</td>\n",
       "      <td>17</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>3.91921</td>\n",
       "      <td>Syt12,CG1275,lt,flw,alpha-Man-Ia,CG42747,Mid1,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>ase(+)</td>\n",
       "      <td>ase</td>\n",
       "      <td>activating</td>\n",
       "      <td>14</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>3.430874</td>\n",
       "      <td>jim,zld,CG34354,CG13293,CG30116,CG34357,CG1576...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>Doc3(+)</td>\n",
       "      <td>Doc3</td>\n",
       "      <td>activating</td>\n",
       "      <td>13</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>5.210102</td>\n",
       "      <td>Syx7,Axud1,CG44247,CG14015,esn,HmgD,Rpn13,CG29...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>E(spl)m8-HLH(+)</td>\n",
       "      <td>E(spl)m8-HLH</td>\n",
       "      <td>activating</td>\n",
       "      <td>10</td>\n",
       "      <td>https://resources.aertslab.org/cistarget/motif...</td>\n",
       "      <td>3.051988</td>\n",
       "      <td>ftz-f1,CG42339,lilli,CG7956,rdgA,grp,GC,CG2017...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>579 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Regulon            TF TFTargetGenesCorrelation NbMarkers  \\\n",
       "471          lola(+)          lola               activating      4254   \n",
       "301          Pdp1(+)          Pdp1               activating      3819   \n",
       "430        fs(1)h(+)        fs(1)h               activating      3814   \n",
       "382          crol(+)          crol               activating      3688   \n",
       "368           bun(+)           bun               activating      3633   \n",
       "..               ...           ...                      ...       ...   \n",
       "348          amos(+)          amos               activating        20   \n",
       "76        CG15696(+)       CG15696               activating        17   \n",
       "353           ase(+)           ase               activating        14   \n",
       "203          Doc3(+)          Doc3               activating        13   \n",
       "212  E(spl)m8-HLH(+)  E(spl)m8-HLH               activating        10   \n",
       "\n",
       "                                                 Motif       NES  \\\n",
       "471  https://resources.aertslab.org/cistarget/motif...  5.945286   \n",
       "301  https://resources.aertslab.org/cistarget/motif...  5.439625   \n",
       "430  https://resources.aertslab.org/cistarget/motif...  6.649265   \n",
       "382  https://resources.aertslab.org/cistarget/motif...  9.530519   \n",
       "368  https://resources.aertslab.org/cistarget/motif...  4.974753   \n",
       "..                                                 ...       ...   \n",
       "348  https://resources.aertslab.org/cistarget/motif...   4.08343   \n",
       "76   https://resources.aertslab.org/cistarget/motif...   3.91921   \n",
       "353  https://resources.aertslab.org/cistarget/motif...  3.430874   \n",
       "203  https://resources.aertslab.org/cistarget/motif...  5.210102   \n",
       "212  https://resources.aertslab.org/cistarget/motif...  3.051988   \n",
       "\n",
       "                                               Markers  \n",
       "471  CG17698,CR45736,p120ctn,jim,AGO3,Myo81F,CG4578...  \n",
       "301  CG17698,CR45736,p120ctn,jim,AGO3,Myo81F,CG1320...  \n",
       "430  CG17698,p120ctn,jim,Myo81F,CG45781,kl-3,CG1320...  \n",
       "382  CG17698,CR45736,p120ctn,jim,CG13204,CG41378,kl...  \n",
       "368  CG17698,CR45736,p120ctn,jim,AGO3,Myo81F,CG4578...  \n",
       "..                                                 ...  \n",
       "348  Nlg3,Pdp1,CG30463,CadN,LPCAT,Gyc88E,shakB,Frq1...  \n",
       "76   Syt12,CG1275,lt,flw,alpha-Man-Ia,CG42747,Mid1,...  \n",
       "353  jim,zld,CG34354,CG13293,CG30116,CG34357,CG1576...  \n",
       "203  Syx7,Axud1,CG44247,CG14015,esn,HmgD,Rpn13,CG29...  \n",
       "212  ftz-f1,CG42339,lilli,CG7956,rdgA,grp,GC,CG2017...  \n",
       "\n",
       "[579 rows x 7 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regulon_df = pd.DataFrame(index = range(0, len(regulons)), columns = (\"Regulon\", \"TF\", \"TFTargetGenesCorrelation\", \"NbMarkers\", \"Motif\", \"NES\", \"Markers\"))\n",
    "for j in range(0, len(regulons)):\n",
    "    # Fixing order of set\n",
    "    context = list(regulons[j].context)\n",
    "    if(context[0].endswith(\".png\")):\n",
    "        tmp = context[0]\n",
    "        context[0] = context[1]\n",
    "        context[1] = tmp\n",
    "    # Setting values\n",
    "    regulon_df[\"Regulon\"].iloc[j] = regulons[j].name\n",
    "    regulon_df[\"TF\"].iloc[j] = regulons[j].transcription_factor\n",
    "    regulon_df[\"TFTargetGenesCorrelation\"].iloc[j] = context[0]\n",
    "    regulon_df[\"NbMarkers\"].iloc[j] = len(set(regulons[j].gene2weight))\n",
    "    regulon_df[\"Motif\"].iloc[j] = \"https://resources.aertslab.org/cistarget/motif_collections/v10nr_clust_public/logos/\" + context[1]\n",
    "    regulon_df[\"NES\"].iloc[j] = regulons[j].score\n",
    "    regulon_df[\"Markers\"].iloc[j] = ','.join(list(regulons[j].gene2weight))\n",
    "\n",
    "regulon_df = regulon_df.sort_values(by='NbMarkers', ascending=False)\n",
    "regulon_df.to_csv(f_regulons_path, index=False, sep = \"\\t\")\n",
    "regulon_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "43779865",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pros\tTrue\n",
      "dati\tTrue\n",
      "scro\tTrue\n",
      "crc\tTrue\n",
      "repo\tTrue\n",
      "sima\tTrue\n"
     ]
    }
   ],
   "source": [
    "# Look for main regulons\n",
    "print(\"pros\", \"pros\" in regulon_df[\"TF\"].values, sep=\"\\t\")\n",
    "print(\"dati\", \"dati\" in regulon_df[\"TF\"].values, sep=\"\\t\")\n",
    "print(\"scro\", \"scro\" in regulon_df[\"TF\"].values, sep=\"\\t\")\n",
    "print(\"crc\", \"crc\" in regulon_df[\"TF\"].values, sep=\"\\t\")\n",
    "print(\"repo\", \"repo\" in regulon_df[\"TF\"].values, sep=\"\\t\")\n",
    "print(\"sima\", \"sima\" in regulon_df[\"TF\"].values, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f1d7455",
   "metadata": {},
   "source": [
    "## Phase III: Cellular regulon enrichment matrix (aka AUCell)\n",
    "\n",
    "Characterize the different cells in a single-cell transcriptomics experiment by the enrichment of the regulons. Enrichment of a regulon is measures as AUC of the recovery curve of the genes that define this regulon."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d3579d01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Regulon</th>\n",
       "      <th>ATbp(+)</th>\n",
       "      <th>Abd-B(+)</th>\n",
       "      <th>Adf1(+)</th>\n",
       "      <th>Aef1(+)</th>\n",
       "      <th>Antp(+)</th>\n",
       "      <th>Asciz(+)</th>\n",
       "      <th>Atf-2(+)</th>\n",
       "      <th>Atf3(+)</th>\n",
       "      <th>Atf6(+)</th>\n",
       "      <th>Awh(+)</th>\n",
       "      <th>...</th>\n",
       "      <th>vri(+)</th>\n",
       "      <th>vvl(+)</th>\n",
       "      <th>wdn(+)</th>\n",
       "      <th>wek(+)</th>\n",
       "      <th>woc(+)</th>\n",
       "      <th>z(+)</th>\n",
       "      <th>zf30C(+)</th>\n",
       "      <th>zfh1(+)</th>\n",
       "      <th>zfh2(+)</th>\n",
       "      <th>zld(+)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAACCCAAGGTGATAT-1_ctrl</th>\n",
       "      <td>0.025016</td>\n",
       "      <td>0.051157</td>\n",
       "      <td>0.009165</td>\n",
       "      <td>0.217876</td>\n",
       "      <td>0.005565</td>\n",
       "      <td>0.043619</td>\n",
       "      <td>0.004420</td>\n",
       "      <td>0.058608</td>\n",
       "      <td>0.065610</td>\n",
       "      <td>0.060508</td>\n",
       "      <td>...</td>\n",
       "      <td>0.112104</td>\n",
       "      <td>0.048170</td>\n",
       "      <td>0.045674</td>\n",
       "      <td>0.008630</td>\n",
       "      <td>0.035798</td>\n",
       "      <td>0.020901</td>\n",
       "      <td>0.040082</td>\n",
       "      <td>0.051026</td>\n",
       "      <td>0.073097</td>\n",
       "      <td>0.054074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACAAATAGCA-1_ctrl</th>\n",
       "      <td>0.048197</td>\n",
       "      <td>0.072628</td>\n",
       "      <td>0.004738</td>\n",
       "      <td>0.093205</td>\n",
       "      <td>0.024699</td>\n",
       "      <td>0.043178</td>\n",
       "      <td>0.050244</td>\n",
       "      <td>0.027497</td>\n",
       "      <td>0.183466</td>\n",
       "      <td>0.106904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.081442</td>\n",
       "      <td>0.063286</td>\n",
       "      <td>0.016767</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.007462</td>\n",
       "      <td>0.057980</td>\n",
       "      <td>0.024195</td>\n",
       "      <td>0.050381</td>\n",
       "      <td>0.187626</td>\n",
       "      <td>0.134823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACAACAAAGT-1_ctrl</th>\n",
       "      <td>0.048374</td>\n",
       "      <td>0.055401</td>\n",
       "      <td>0.010618</td>\n",
       "      <td>0.095788</td>\n",
       "      <td>0.007051</td>\n",
       "      <td>0.140935</td>\n",
       "      <td>0.012873</td>\n",
       "      <td>0.032550</td>\n",
       "      <td>0.233749</td>\n",
       "      <td>0.132549</td>\n",
       "      <td>...</td>\n",
       "      <td>0.077538</td>\n",
       "      <td>0.050187</td>\n",
       "      <td>0.012419</td>\n",
       "      <td>0.014976</td>\n",
       "      <td>0.016292</td>\n",
       "      <td>0.060347</td>\n",
       "      <td>0.019741</td>\n",
       "      <td>0.055351</td>\n",
       "      <td>0.156163</td>\n",
       "      <td>0.068243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACACTCATAG-1_ctrl</th>\n",
       "      <td>0.056667</td>\n",
       "      <td>0.080954</td>\n",
       "      <td>0.005031</td>\n",
       "      <td>0.128824</td>\n",
       "      <td>0.007040</td>\n",
       "      <td>0.029652</td>\n",
       "      <td>0.015074</td>\n",
       "      <td>0.039802</td>\n",
       "      <td>0.155645</td>\n",
       "      <td>0.171546</td>\n",
       "      <td>...</td>\n",
       "      <td>0.091319</td>\n",
       "      <td>0.063301</td>\n",
       "      <td>0.019340</td>\n",
       "      <td>0.006113</td>\n",
       "      <td>0.026780</td>\n",
       "      <td>0.052161</td>\n",
       "      <td>0.024468</td>\n",
       "      <td>0.041457</td>\n",
       "      <td>0.196490</td>\n",
       "      <td>0.076487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACAGAGAGGG-1_ctrl</th>\n",
       "      <td>0.013838</td>\n",
       "      <td>0.008583</td>\n",
       "      <td>0.014718</td>\n",
       "      <td>0.283570</td>\n",
       "      <td>0.012601</td>\n",
       "      <td>0.032273</td>\n",
       "      <td>0.034665</td>\n",
       "      <td>0.067232</td>\n",
       "      <td>0.045371</td>\n",
       "      <td>0.070239</td>\n",
       "      <td>...</td>\n",
       "      <td>0.164186</td>\n",
       "      <td>0.060007</td>\n",
       "      <td>0.047588</td>\n",
       "      <td>0.019446</td>\n",
       "      <td>0.062699</td>\n",
       "      <td>0.024874</td>\n",
       "      <td>0.031490</td>\n",
       "      <td>0.047543</td>\n",
       "      <td>0.079467</td>\n",
       "      <td>0.016080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGGTTTGGAGG-1_ndkd</th>\n",
       "      <td>0.015443</td>\n",
       "      <td>0.029266</td>\n",
       "      <td>0.004485</td>\n",
       "      <td>0.228821</td>\n",
       "      <td>0.018392</td>\n",
       "      <td>0.022867</td>\n",
       "      <td>0.077920</td>\n",
       "      <td>0.163073</td>\n",
       "      <td>0.080727</td>\n",
       "      <td>0.160639</td>\n",
       "      <td>...</td>\n",
       "      <td>0.194089</td>\n",
       "      <td>0.102957</td>\n",
       "      <td>0.042985</td>\n",
       "      <td>0.067163</td>\n",
       "      <td>0.096124</td>\n",
       "      <td>0.055005</td>\n",
       "      <td>0.036921</td>\n",
       "      <td>0.118856</td>\n",
       "      <td>0.202013</td>\n",
       "      <td>0.022660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCCGTTTCG-1_ndkd</th>\n",
       "      <td>0.044308</td>\n",
       "      <td>0.037830</td>\n",
       "      <td>0.006690</td>\n",
       "      <td>0.127690</td>\n",
       "      <td>0.016011</td>\n",
       "      <td>0.031436</td>\n",
       "      <td>0.044347</td>\n",
       "      <td>0.040877</td>\n",
       "      <td>0.179631</td>\n",
       "      <td>0.126737</td>\n",
       "      <td>...</td>\n",
       "      <td>0.097696</td>\n",
       "      <td>0.066936</td>\n",
       "      <td>0.019406</td>\n",
       "      <td>0.011160</td>\n",
       "      <td>0.027647</td>\n",
       "      <td>0.043161</td>\n",
       "      <td>0.025456</td>\n",
       "      <td>0.066068</td>\n",
       "      <td>0.169896</td>\n",
       "      <td>0.093657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCCTTCAGC-1_ndkd</th>\n",
       "      <td>0.043543</td>\n",
       "      <td>0.066490</td>\n",
       "      <td>0.014648</td>\n",
       "      <td>0.079371</td>\n",
       "      <td>0.010937</td>\n",
       "      <td>0.016735</td>\n",
       "      <td>0.007212</td>\n",
       "      <td>0.043082</td>\n",
       "      <td>0.173221</td>\n",
       "      <td>0.687419</td>\n",
       "      <td>...</td>\n",
       "      <td>0.079111</td>\n",
       "      <td>0.055287</td>\n",
       "      <td>0.017202</td>\n",
       "      <td>0.002248</td>\n",
       "      <td>0.021235</td>\n",
       "      <td>0.074826</td>\n",
       "      <td>0.017163</td>\n",
       "      <td>0.057514</td>\n",
       "      <td>0.150466</td>\n",
       "      <td>0.080344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCGAACGCC-1_ndkd</th>\n",
       "      <td>0.045354</td>\n",
       "      <td>0.045479</td>\n",
       "      <td>0.011098</td>\n",
       "      <td>0.149059</td>\n",
       "      <td>0.011069</td>\n",
       "      <td>0.018916</td>\n",
       "      <td>0.018311</td>\n",
       "      <td>0.060819</td>\n",
       "      <td>0.169757</td>\n",
       "      <td>0.185152</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103534</td>\n",
       "      <td>0.069396</td>\n",
       "      <td>0.015553</td>\n",
       "      <td>0.010884</td>\n",
       "      <td>0.051322</td>\n",
       "      <td>0.053105</td>\n",
       "      <td>0.017973</td>\n",
       "      <td>0.093731</td>\n",
       "      <td>0.251925</td>\n",
       "      <td>0.080254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCGTGTTCC-1_ndkd</th>\n",
       "      <td>0.005439</td>\n",
       "      <td>0.003235</td>\n",
       "      <td>0.010625</td>\n",
       "      <td>0.288568</td>\n",
       "      <td>0.005627</td>\n",
       "      <td>0.044556</td>\n",
       "      <td>0.014916</td>\n",
       "      <td>0.102904</td>\n",
       "      <td>0.038323</td>\n",
       "      <td>0.038682</td>\n",
       "      <td>...</td>\n",
       "      <td>0.148546</td>\n",
       "      <td>0.050959</td>\n",
       "      <td>0.054571</td>\n",
       "      <td>0.077759</td>\n",
       "      <td>0.091673</td>\n",
       "      <td>0.015425</td>\n",
       "      <td>0.031259</td>\n",
       "      <td>0.034612</td>\n",
       "      <td>0.025065</td>\n",
       "      <td>0.039047</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23179 rows × 579 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Regulon                   ATbp(+)  Abd-B(+)   Adf1(+)   Aef1(+)   Antp(+)  \\\n",
       "Cell                                                                        \n",
       "AAACCCAAGGTGATAT-1_ctrl  0.025016  0.051157  0.009165  0.217876  0.005565   \n",
       "AAACCCACAAATAGCA-1_ctrl  0.048197  0.072628  0.004738  0.093205  0.024699   \n",
       "AAACCCACAACAAAGT-1_ctrl  0.048374  0.055401  0.010618  0.095788  0.007051   \n",
       "AAACCCACACTCATAG-1_ctrl  0.056667  0.080954  0.005031  0.128824  0.007040   \n",
       "AAACCCACAGAGAGGG-1_ctrl  0.013838  0.008583  0.014718  0.283570  0.012601   \n",
       "...                           ...       ...       ...       ...       ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd  0.015443  0.029266  0.004485  0.228821  0.018392   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd  0.044308  0.037830  0.006690  0.127690  0.016011   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd  0.043543  0.066490  0.014648  0.079371  0.010937   \n",
       "TTTGTTGTCGAACGCC-1_ndkd  0.045354  0.045479  0.011098  0.149059  0.011069   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd  0.005439  0.003235  0.010625  0.288568  0.005627   \n",
       "\n",
       "Regulon                  Asciz(+)  Atf-2(+)   Atf3(+)   Atf6(+)    Awh(+)  \\\n",
       "Cell                                                                        \n",
       "AAACCCAAGGTGATAT-1_ctrl  0.043619  0.004420  0.058608  0.065610  0.060508   \n",
       "AAACCCACAAATAGCA-1_ctrl  0.043178  0.050244  0.027497  0.183466  0.106904   \n",
       "AAACCCACAACAAAGT-1_ctrl  0.140935  0.012873  0.032550  0.233749  0.132549   \n",
       "AAACCCACACTCATAG-1_ctrl  0.029652  0.015074  0.039802  0.155645  0.171546   \n",
       "AAACCCACAGAGAGGG-1_ctrl  0.032273  0.034665  0.067232  0.045371  0.070239   \n",
       "...                           ...       ...       ...       ...       ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd  0.022867  0.077920  0.163073  0.080727  0.160639   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd  0.031436  0.044347  0.040877  0.179631  0.126737   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd  0.016735  0.007212  0.043082  0.173221  0.687419   \n",
       "TTTGTTGTCGAACGCC-1_ndkd  0.018916  0.018311  0.060819  0.169757  0.185152   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd  0.044556  0.014916  0.102904  0.038323  0.038682   \n",
       "\n",
       "Regulon                  ...    vri(+)    vvl(+)    wdn(+)    wek(+)  \\\n",
       "Cell                     ...                                           \n",
       "AAACCCAAGGTGATAT-1_ctrl  ...  0.112104  0.048170  0.045674  0.008630   \n",
       "AAACCCACAAATAGCA-1_ctrl  ...  0.081442  0.063286  0.016767  0.001272   \n",
       "AAACCCACAACAAAGT-1_ctrl  ...  0.077538  0.050187  0.012419  0.014976   \n",
       "AAACCCACACTCATAG-1_ctrl  ...  0.091319  0.063301  0.019340  0.006113   \n",
       "AAACCCACAGAGAGGG-1_ctrl  ...  0.164186  0.060007  0.047588  0.019446   \n",
       "...                      ...       ...       ...       ...       ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd  ...  0.194089  0.102957  0.042985  0.067163   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd  ...  0.097696  0.066936  0.019406  0.011160   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd  ...  0.079111  0.055287  0.017202  0.002248   \n",
       "TTTGTTGTCGAACGCC-1_ndkd  ...  0.103534  0.069396  0.015553  0.010884   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd  ...  0.148546  0.050959  0.054571  0.077759   \n",
       "\n",
       "Regulon                    woc(+)      z(+)  zf30C(+)   zfh1(+)   zfh2(+)  \\\n",
       "Cell                                                                        \n",
       "AAACCCAAGGTGATAT-1_ctrl  0.035798  0.020901  0.040082  0.051026  0.073097   \n",
       "AAACCCACAAATAGCA-1_ctrl  0.007462  0.057980  0.024195  0.050381  0.187626   \n",
       "AAACCCACAACAAAGT-1_ctrl  0.016292  0.060347  0.019741  0.055351  0.156163   \n",
       "AAACCCACACTCATAG-1_ctrl  0.026780  0.052161  0.024468  0.041457  0.196490   \n",
       "AAACCCACAGAGAGGG-1_ctrl  0.062699  0.024874  0.031490  0.047543  0.079467   \n",
       "...                           ...       ...       ...       ...       ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd  0.096124  0.055005  0.036921  0.118856  0.202013   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd  0.027647  0.043161  0.025456  0.066068  0.169896   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd  0.021235  0.074826  0.017163  0.057514  0.150466   \n",
       "TTTGTTGTCGAACGCC-1_ndkd  0.051322  0.053105  0.017973  0.093731  0.251925   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd  0.091673  0.015425  0.031259  0.034612  0.025065   \n",
       "\n",
       "Regulon                    zld(+)  \n",
       "Cell                               \n",
       "AAACCCAAGGTGATAT-1_ctrl  0.054074  \n",
       "AAACCCACAAATAGCA-1_ctrl  0.134823  \n",
       "AAACCCACAACAAAGT-1_ctrl  0.068243  \n",
       "AAACCCACACTCATAG-1_ctrl  0.076487  \n",
       "AAACCCACAGAGAGGG-1_ctrl  0.016080  \n",
       "...                           ...  \n",
       "TTTGTTGGTTTGGAGG-1_ndkd  0.022660  \n",
       "TTTGTTGTCCGTTTCG-1_ndkd  0.093657  \n",
       "TTTGTTGTCCTTCAGC-1_ndkd  0.080344  \n",
       "TTTGTTGTCGAACGCC-1_ndkd  0.080254  \n",
       "TTTGTTGTCGTGTTCC-1_ndkd  0.039047  \n",
       "\n",
       "[23179 rows x 579 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx = aucell(ex_matrix.transpose(), regulons, num_workers=default_n_threads)\n",
    "auc_mtx.to_csv(f_regulons_aucell_path, sep = \"\\t\")\n",
    "auc_mtx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32c8362c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint to regenerate the object from the file\n",
    "#auc_mtx = pd.read_csv(\"/home/gardeux/SVRAW1/gardeux/2023-04-19_Emma_snRNAseq/analysis/old/Pan_neuro_both_reannotated_GFP_curated_reintegrated_regulons_aucell.tsv\", sep = \"\\t\", index_col = \"Cell\")\n",
    "#auc_mtx = pd.read_csv(f_regulons_aucell_path, sep = \"\\t\", index_col = \"Cell\")\n",
    "#auc_mtx.columns.name = \"Regulon\"\n",
    "#auc_mtx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "d6ab567b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start time: 11:01:44\n",
      "End time: 11:19:28\n",
      "Running time: 0:17:43.246747\n"
     ]
    }
   ],
   "source": [
    "start = datetime.now(pytz.timezone('Europe/Paris'))\n",
    "print(\"Start time:\", start.strftime(\"%H:%M:%S\"))\n",
    "\n",
    "auc_mtx_bin = binarize(auc_mtx, seed = 42, num_workers=default_n_threads)\n",
    "\n",
    "end = datetime.now(pytz.timezone('Europe/Paris'))\n",
    "print(\"End time:\", end.strftime(\"%H:%M:%S\"))\n",
    "print(\"Running time:\", (end - start))\n",
    "\n",
    "# Note: 23mn12 with num_workers=12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "b942bfe5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Regulon\n",
       "ATbp(+)     0.070662\n",
       "Abd-B(+)    0.137902\n",
       "Adf1(+)     0.024422\n",
       "Aef1(+)     0.256438\n",
       "Antp(+)     0.046179\n",
       "              ...   \n",
       "z(+)        0.083907\n",
       "zf30C(+)    0.041259\n",
       "zfh1(+)     0.097270\n",
       "zfh2(+)     0.242251\n",
       "zld(+)      0.134415\n",
       "Length: 579, dtype: float64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds = auc_mtx_bin[1]\n",
    "binarization_thresholds.to_csv(f_regulons_binarization_thresholds_aucell_path, sep = \"\\t\")\n",
    "binarization_thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "4aaf52f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Regulon</th>\n",
       "      <th>ATbp(+)</th>\n",
       "      <th>Abd-B(+)</th>\n",
       "      <th>Adf1(+)</th>\n",
       "      <th>Aef1(+)</th>\n",
       "      <th>Antp(+)</th>\n",
       "      <th>Asciz(+)</th>\n",
       "      <th>Atf-2(+)</th>\n",
       "      <th>Atf3(+)</th>\n",
       "      <th>Atf6(+)</th>\n",
       "      <th>Awh(+)</th>\n",
       "      <th>...</th>\n",
       "      <th>vri(+)</th>\n",
       "      <th>vvl(+)</th>\n",
       "      <th>wdn(+)</th>\n",
       "      <th>wek(+)</th>\n",
       "      <th>woc(+)</th>\n",
       "      <th>z(+)</th>\n",
       "      <th>zf30C(+)</th>\n",
       "      <th>zfh1(+)</th>\n",
       "      <th>zfh2(+)</th>\n",
       "      <th>zld(+)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cell</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAACCCAAGGTGATAT-1_ctrl</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACAAATAGCA-1_ctrl</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACAACAAAGT-1_ctrl</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACACTCATAG-1_ctrl</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAACCCACAGAGAGGG-1_ctrl</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGGTTTGGAGG-1_ndkd</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCCGTTTCG-1_ndkd</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCCTTCAGC-1_ndkd</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCGAACGCC-1_ndkd</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TTTGTTGTCGTGTTCC-1_ndkd</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23179 rows × 579 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Regulon                  ATbp(+)  Abd-B(+)  Adf1(+)  Aef1(+)  Antp(+)  \\\n",
       "Cell                                                                    \n",
       "AAACCCAAGGTGATAT-1_ctrl        0         0        0        0        0   \n",
       "AAACCCACAAATAGCA-1_ctrl        0         0        0        0        0   \n",
       "AAACCCACAACAAAGT-1_ctrl        0         0        0        0        0   \n",
       "AAACCCACACTCATAG-1_ctrl        0         0        0        0        0   \n",
       "AAACCCACAGAGAGGG-1_ctrl        0         0        0        1        0   \n",
       "...                          ...       ...      ...      ...      ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd        0         0        0        0        0   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd        0         0        0        0        0   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd        0         0        0        0        0   \n",
       "TTTGTTGTCGAACGCC-1_ndkd        0         0        0        0        0   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd        0         0        0        1        0   \n",
       "\n",
       "Regulon                  Asciz(+)  Atf-2(+)  Atf3(+)  Atf6(+)  Awh(+)  ...  \\\n",
       "Cell                                                                   ...   \n",
       "AAACCCAAGGTGATAT-1_ctrl         0         0        0        0       0  ...   \n",
       "AAACCCACAAATAGCA-1_ctrl         0         1        0        1       0  ...   \n",
       "AAACCCACAACAAAGT-1_ctrl         1         0        0        1       0  ...   \n",
       "AAACCCACACTCATAG-1_ctrl         0         0        0        1       0  ...   \n",
       "AAACCCACAGAGAGGG-1_ctrl         0         0        0        0       0  ...   \n",
       "...                           ...       ...      ...      ...     ...  ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd         0         1        1        0       0  ...   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd         0         0        0        1       0  ...   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd         0         0        0        1       1  ...   \n",
       "TTTGTTGTCGAACGCC-1_ndkd         0         0        0        1       0  ...   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd         0         0        0        0       0  ...   \n",
       "\n",
       "Regulon                  vri(+)  vvl(+)  wdn(+)  wek(+)  woc(+)  z(+)  \\\n",
       "Cell                                                                    \n",
       "AAACCCAAGGTGATAT-1_ctrl       0       0       0       0       0     0   \n",
       "AAACCCACAAATAGCA-1_ctrl       0       0       0       0       0     0   \n",
       "AAACCCACAACAAAGT-1_ctrl       0       0       0       0       0     0   \n",
       "AAACCCACACTCATAG-1_ctrl       0       0       0       0       0     0   \n",
       "AAACCCACAGAGAGGG-1_ctrl       0       0       0       0       0     0   \n",
       "...                         ...     ...     ...     ...     ...   ...   \n",
       "TTTGTTGGTTTGGAGG-1_ndkd       1       0       0       0       1     0   \n",
       "TTTGTTGTCCGTTTCG-1_ndkd       0       0       0       0       0     0   \n",
       "TTTGTTGTCCTTCAGC-1_ndkd       0       0       0       0       0     0   \n",
       "TTTGTTGTCGAACGCC-1_ndkd       0       0       0       0       0     0   \n",
       "TTTGTTGTCGTGTTCC-1_ndkd       0       0       1       1       1     0   \n",
       "\n",
       "Regulon                  zf30C(+)  zfh1(+)  zfh2(+)  zld(+)  \n",
       "Cell                                                         \n",
       "AAACCCAAGGTGATAT-1_ctrl         0        0        0       0  \n",
       "AAACCCACAAATAGCA-1_ctrl         0        0        0       1  \n",
       "AAACCCACAACAAAGT-1_ctrl         0        0        0       0  \n",
       "AAACCCACACTCATAG-1_ctrl         0        0        0       0  \n",
       "AAACCCACAGAGAGGG-1_ctrl         0        0        0       0  \n",
       "...                           ...      ...      ...     ...  \n",
       "TTTGTTGGTTTGGAGG-1_ndkd         0        1        0       0  \n",
       "TTTGTTGTCCGTTTCG-1_ndkd         0        0        0       0  \n",
       "TTTGTTGTCCTTCAGC-1_ndkd         0        0        0       0  \n",
       "TTTGTTGTCGAACGCC-1_ndkd         0        0        1       0  \n",
       "TTTGTTGTCGTGTTCC-1_ndkd         0        0        0       0  \n",
       "\n",
       "[23179 rows x 579 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin = auc_mtx_bin[0]\n",
    "auc_mtx_bin.to_csv(f_regulons_binarized_aucell_path, sep = \"\\t\")\n",
    "auc_mtx_bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "d9a4a4e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "716"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin.sum(numeric_only=True, axis=0).loc['scro(+)'] # 733 original (764 regulons, Aerts). 716 ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "630c7a4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2968568380139414"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds.loc['scro(+)'] # 0.3359530388214861 original (764 regulons, Aerts). 0.2968568380139414"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "418c7f4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1212"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin.sum(numeric_only=True, axis=0).loc['sima(+)'] # 1067 original (764 regulons, Aerts). 1212 ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "dff63c03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18676990882955136"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds.loc['sima(+)'] # 0.21179422980092633 original (764 regulons, Aerts). 0.18676990882955136"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "c063f21c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1036"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin.sum(numeric_only=True, axis=0).loc['crc(+)'] # 1136 original (764 regulons, Aerts). 1036 a bit too many?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e65f0095",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23696722508584317"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds.loc['crc(+)'] # 0.21042246303356155 original (764 regulons, Aerts). 0.23696722508584317"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "0493e2c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11631"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin.sum(numeric_only=True, axis=0).loc['dati(+)'] # 11733 original (764 regulons, Aerts). 11631 ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "7cc117da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2806370838941694"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds.loc['dati(+)'] # 0.25299783006969817 original (764 regulons, Aerts). 0.2806370838941694"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "e96925f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18548"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin.sum(numeric_only=True, axis=0).loc['pros(+)'] # 18482 original (764 regulons, Aerts). 18548 ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "99f17245",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13361937192963938"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds.loc['pros(+)'] # 0.11976533052607628 original (764 regulons, Aerts). 0.13361937192963938"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "3d9c6121",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "920"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_mtx_bin.sum(numeric_only=True, axis=0).loc['repo(+)'] # 906 original (764 regulons, Aerts). 920 ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "8b236548",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11348899982841607"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "binarization_thresholds.loc['repo(+)'] # 0.10907608108082668 original (764 regulons, Aerts). 0.11348899982841607"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "0e90cd43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "975"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(auc_mtx[\"crc(+)\"] > 0.238)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a2fa4f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(auc_mtx[\"crc(+)\"] > binarization_thresholds.loc['crc(+)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "0014fe92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1036"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(auc_mtx_bin[\"crc(+)\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pySCENIC (Python v3.10)",
   "language": "python",
   "name": "pyscenic"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}