2.train-VAE/5.apply_ica.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "fdc3df1b-9f13-464f-94b3-b088c804965d",
   "metadata": {},
   "source": [
    "## Apply ICA to gene dependencies\n",
    "\n",
    "Independent Component Analysis (ICA) is a dimensionality reduction method that separates a multivariate signal into additive, independent components.\n",
    "\n",
    "We apply it to GeneEffect scores here and save latent representations (ICA components) for downstream comparative analyses (to compare with BetaVAE).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e9edbfb3-e84e-4435-9900-4def2649cc6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import pathlib\n",
    "import pandas as pd\n",
    "import plotnine as gg\n",
    "from sklearn.decomposition import FastICA\n",
    "\n",
    "sys.path.insert(0, \"../utils/\")\n",
    "from data_loader import load_model_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9259847e-982d-4486-ab72-de50888aae36",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_directory = pathlib.Path(\"../0.data-download/data\").resolve()\n",
    "dependency_file = pathlib.Path(f\"{data_directory}/CRISPRGeneEffect.parquet\").resolve()\n",
    "gene_dict_file = pathlib.Path(f\"{data_directory}/CRISPR_gene_dictionary.parquet\").resolve()\n",
    "\n",
    "output_dir = pathlib.Path(\"results\").resolve()\n",
    "ica_output_file = pathlib.Path(f\"{output_dir}/ica_latent.parquet.gz\").resolve()\n",
    "output_ica_weights_file = pathlib.Path(f\"{output_dir}/ICA_weight_matrix_gsea.parquet\").resolve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4b736280-f8f3-46c6-a229-7f2d757f6f54",
   "metadata": {},
   "outputs": [],
   "source": [
    "ica_components = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "09c9c59c-5be6-4fca-96b3-c7f303a20b34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1150, 18444)\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "dependency_df, gene_dict_df = load_model_data(dependency_file, gene_dict_file)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "fe75f876-9b6b-4444-bc0b-0c67135f955e",
   "metadata": {},
   "source": [
    "# Perform ICA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f7d590b4-23c9-4f92-820a-8fbe1fe3c7c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-2 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-2 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-2 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-2 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-2 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-2 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>FastICA(n_components=50, random_state=0)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;FastICA<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.decomposition.FastICA.html\">?<span>Documentation for FastICA</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>FastICA(n_components=50, random_state=0)</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "FastICA(n_components=50, random_state=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ica = FastICA(n_components=ica_components, random_state=0)\n",
    "ica.fit(dependency_df.drop(columns=[\"ModelID\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "65756252",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transform models into ICA space\n",
    "dependency_df_transformed = pd.DataFrame(\n",
    "   ica.transform(dependency_df.drop(columns=[\"ModelID\"]))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "12ee4081-4d2a-4266-96a4-334b985871b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1150, 51)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ModelID</th>\n",
       "      <th>ICA_0</th>\n",
       "      <th>ICA_1</th>\n",
       "      <th>ICA_2</th>\n",
       "      <th>ICA_3</th>\n",
       "      <th>ICA_4</th>\n",
       "      <th>ICA_5</th>\n",
       "      <th>ICA_6</th>\n",
       "      <th>ICA_7</th>\n",
       "      <th>ICA_8</th>\n",
       "      <th>...</th>\n",
       "      <th>ICA_40</th>\n",
       "      <th>ICA_41</th>\n",
       "      <th>ICA_42</th>\n",
       "      <th>ICA_43</th>\n",
       "      <th>ICA_44</th>\n",
       "      <th>ICA_45</th>\n",
       "      <th>ICA_46</th>\n",
       "      <th>ICA_47</th>\n",
       "      <th>ICA_48</th>\n",
       "      <th>ICA_49</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ACH-000001</td>\n",
       "      <td>-1.894820</td>\n",
       "      <td>-0.420639</td>\n",
       "      <td>0.152816</td>\n",
       "      <td>0.683013</td>\n",
       "      <td>-0.576386</td>\n",
       "      <td>-1.254303</td>\n",
       "      <td>0.875927</td>\n",
       "      <td>-1.027659</td>\n",
       "      <td>0.147783</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.392907</td>\n",
       "      <td>-0.136282</td>\n",
       "      <td>-3.179023</td>\n",
       "      <td>0.041179</td>\n",
       "      <td>0.131887</td>\n",
       "      <td>0.356428</td>\n",
       "      <td>-0.911628</td>\n",
       "      <td>-0.139935</td>\n",
       "      <td>0.427778</td>\n",
       "      <td>-1.135748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ACH-000004</td>\n",
       "      <td>0.199190</td>\n",
       "      <td>0.878038</td>\n",
       "      <td>0.234728</td>\n",
       "      <td>-2.292786</td>\n",
       "      <td>0.262948</td>\n",
       "      <td>-0.014035</td>\n",
       "      <td>-0.237673</td>\n",
       "      <td>-1.491049</td>\n",
       "      <td>-0.696543</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.667632</td>\n",
       "      <td>0.400782</td>\n",
       "      <td>0.015032</td>\n",
       "      <td>0.764805</td>\n",
       "      <td>1.560523</td>\n",
       "      <td>0.082876</td>\n",
       "      <td>0.558015</td>\n",
       "      <td>0.268848</td>\n",
       "      <td>-0.079606</td>\n",
       "      <td>0.221561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ACH-000005</td>\n",
       "      <td>0.102404</td>\n",
       "      <td>0.134753</td>\n",
       "      <td>-0.001711</td>\n",
       "      <td>-0.578072</td>\n",
       "      <td>-0.197917</td>\n",
       "      <td>-1.190910</td>\n",
       "      <td>-0.065030</td>\n",
       "      <td>-0.879704</td>\n",
       "      <td>-1.128394</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.439670</td>\n",
       "      <td>0.167856</td>\n",
       "      <td>-0.193180</td>\n",
       "      <td>3.324927</td>\n",
       "      <td>0.527463</td>\n",
       "      <td>0.296716</td>\n",
       "      <td>0.149641</td>\n",
       "      <td>-0.156201</td>\n",
       "      <td>-0.212288</td>\n",
       "      <td>0.140653</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ModelID     ICA_0     ICA_1     ICA_2     ICA_3     ICA_4     ICA_5  \\\n",
       "0  ACH-000001 -1.894820 -0.420639  0.152816  0.683013 -0.576386 -1.254303   \n",
       "1  ACH-000004  0.199190  0.878038  0.234728 -2.292786  0.262948 -0.014035   \n",
       "2  ACH-000005  0.102404  0.134753 -0.001711 -0.578072 -0.197917 -1.190910   \n",
       "\n",
       "      ICA_6     ICA_7     ICA_8  ...    ICA_40    ICA_41    ICA_42    ICA_43  \\\n",
       "0  0.875927 -1.027659  0.147783  ... -0.392907 -0.136282 -3.179023  0.041179   \n",
       "1 -0.237673 -1.491049 -0.696543  ... -3.667632  0.400782  0.015032  0.764805   \n",
       "2 -0.065030 -0.879704 -1.128394  ... -3.439670  0.167856 -0.193180  3.324927   \n",
       "\n",
       "     ICA_44    ICA_45    ICA_46    ICA_47    ICA_48    ICA_49  \n",
       "0  0.131887  0.356428 -0.911628 -0.139935  0.427778 -1.135748  \n",
       "1  1.560523  0.082876  0.558015  0.268848 -0.079606  0.221561  \n",
       "2  0.527463  0.296716  0.149641 -0.156201 -0.212288  0.140653  \n",
       "\n",
       "[3 rows x 51 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recode column space and add back model IDs\n",
    "dependency_df_transformed.columns = [f\"ICA_{x}\" for x in range(0, dependency_df_transformed.shape[1])]\n",
    "dependency_df_transformed = pd.concat([dependency_df.loc[:, \"ModelID\"], dependency_df_transformed], axis=\"columns\")\n",
    "\n",
    "\n",
    "dependency_df_transformed.to_parquet(ica_output_file, index=False)\n",
    "\n",
    "\n",
    "print(dependency_df_transformed.shape)\n",
    "dependency_df_transformed.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "26ab257b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2718, 51)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genes</th>\n",
       "      <th>ICA_0</th>\n",
       "      <th>ICA_1</th>\n",
       "      <th>ICA_2</th>\n",
       "      <th>ICA_3</th>\n",
       "      <th>ICA_4</th>\n",
       "      <th>ICA_5</th>\n",
       "      <th>ICA_6</th>\n",
       "      <th>ICA_7</th>\n",
       "      <th>ICA_8</th>\n",
       "      <th>...</th>\n",
       "      <th>ICA_40</th>\n",
       "      <th>ICA_41</th>\n",
       "      <th>ICA_42</th>\n",
       "      <th>ICA_43</th>\n",
       "      <th>ICA_44</th>\n",
       "      <th>ICA_45</th>\n",
       "      <th>ICA_46</th>\n",
       "      <th>ICA_47</th>\n",
       "      <th>ICA_48</th>\n",
       "      <th>ICA_49</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CYB5R4</td>\n",
       "      <td>0.002743</td>\n",
       "      <td>0.004387</td>\n",
       "      <td>-0.002686</td>\n",
       "      <td>0.013452</td>\n",
       "      <td>-0.024221</td>\n",
       "      <td>-0.012599</td>\n",
       "      <td>-0.004682</td>\n",
       "      <td>0.001535</td>\n",
       "      <td>-0.008149</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.025868</td>\n",
       "      <td>0.024477</td>\n",
       "      <td>0.018240</td>\n",
       "      <td>0.006149</td>\n",
       "      <td>-0.009292</td>\n",
       "      <td>-0.000889</td>\n",
       "      <td>0.037068</td>\n",
       "      <td>0.019506</td>\n",
       "      <td>0.003236</td>\n",
       "      <td>-0.018785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CNEP1R1</td>\n",
       "      <td>0.000538</td>\n",
       "      <td>0.000177</td>\n",
       "      <td>-0.007442</td>\n",
       "      <td>0.011053</td>\n",
       "      <td>-0.015358</td>\n",
       "      <td>-0.004890</td>\n",
       "      <td>-0.008380</td>\n",
       "      <td>0.005434</td>\n",
       "      <td>-0.021387</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014664</td>\n",
       "      <td>0.012351</td>\n",
       "      <td>0.040456</td>\n",
       "      <td>-0.010147</td>\n",
       "      <td>0.022489</td>\n",
       "      <td>-0.005652</td>\n",
       "      <td>0.010396</td>\n",
       "      <td>0.026636</td>\n",
       "      <td>-0.021068</td>\n",
       "      <td>-0.012807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ARHGEF12</td>\n",
       "      <td>0.014816</td>\n",
       "      <td>-0.023335</td>\n",
       "      <td>-0.010923</td>\n",
       "      <td>0.002226</td>\n",
       "      <td>-0.002431</td>\n",
       "      <td>0.004405</td>\n",
       "      <td>0.001888</td>\n",
       "      <td>0.002457</td>\n",
       "      <td>0.007297</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.007788</td>\n",
       "      <td>-0.004481</td>\n",
       "      <td>0.003721</td>\n",
       "      <td>-0.024953</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>0.014994</td>\n",
       "      <td>0.003732</td>\n",
       "      <td>-0.022538</td>\n",
       "      <td>-0.007353</td>\n",
       "      <td>-0.000842</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      genes     ICA_0     ICA_1     ICA_2     ICA_3     ICA_4     ICA_5  \\\n",
       "0    CYB5R4  0.002743  0.004387 -0.002686  0.013452 -0.024221 -0.012599   \n",
       "1   CNEP1R1  0.000538  0.000177 -0.007442  0.011053 -0.015358 -0.004890   \n",
       "2  ARHGEF12  0.014816 -0.023335 -0.010923  0.002226 -0.002431  0.004405   \n",
       "\n",
       "      ICA_6     ICA_7     ICA_8  ...    ICA_40    ICA_41    ICA_42    ICA_43  \\\n",
       "0 -0.004682  0.001535 -0.008149  ... -0.025868  0.024477  0.018240  0.006149   \n",
       "1 -0.008380  0.005434 -0.021387  ...  0.014664  0.012351  0.040456 -0.010147   \n",
       "2  0.001888  0.002457  0.007297  ... -0.007788 -0.004481  0.003721 -0.024953   \n",
       "\n",
       "     ICA_44    ICA_45    ICA_46    ICA_47    ICA_48    ICA_49  \n",
       "0 -0.009292 -0.000889  0.037068  0.019506  0.003236 -0.018785  \n",
       "1  0.022489 -0.005652  0.010396  0.026636 -0.021068 -0.012807  \n",
       "2  0.000335  0.014994  0.003732 -0.022538 -0.007353 -0.000842  \n",
       "\n",
       "[3 rows x 51 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Obtain weights, which can be used in GSEA\n",
    "ica_weights = pd.DataFrame(ica.mixing_, index=dependency_df.drop(columns=[\"ModelID\"]).columns.tolist())\n",
    "ica_weights.columns = [f\"ICA_{x}\" for x in range(ica_weights.shape[1])]\n",
    "\n",
    "\n",
    "ica_weights = ica_weights.reset_index().rename(columns={\"index\": \"genes\"})\n",
    "\n",
    "\n",
    "ica_weights.to_parquet(output_ica_weights_file, index=False)\n",
    "\n",
    "\n",
    "print(ica_weights.shape)\n",
    "ica_weights.head(3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}