main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "69ebd984-68a9-4728-817d-714e53cb84b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Author: Rute Ayalew\n",
    "# Date: 11/5/23\n",
    "# Collaborators: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63a0a9f8-d145-4fe8-a585-62f832df0d35",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Package Installations \n",
    "!pip install pandas\n",
    "!pip install numpy\n",
    "!pip install scikit-learn\n",
    "!pip install rds2py\n",
    "!pip install kneed\n",
    "!pip install yellowbrick\n",
    "!pip install imblearn\n",
    "!pip install statsmodels\n",
    "!pip install dask\n",
    "!pip install polars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a888d7ab-b6b4-4b19-9031-9308ccce6dae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Package Import\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.cluster import KMeans\n",
    "import rds2py\n",
    "import yellowbrick\n",
    "import kneed\n",
    "import imblearn\n",
    "import os\n",
    "import statsmodels\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor \n",
    "from dask.dataframe import from_pandas\n",
    "import polars as pl\n",
    "\n",
    "# Self-Defined Module Import\n",
    "import Rdata_understanding as du\n",
    "import Rdata_prep as dp\n",
    "import Rdata_model as dm\n",
    "import Rmy_utils as mu\n",
    "\n",
    "\n",
    "# temp imports\n",
    "import requests\n",
    "from io import BytesIO\n",
    "import gzip\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3e9d02c3-d8ba-464a-8157-c1a84c5eca9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# User-Initialized Variables to guide Data Mining Process\n",
    "target_attribute = 'disease' \n",
    "task_type = 'classification' \n",
    "random_seed = 45\n",
    "test_size = 0.3 \n",
    "\n",
    "# User-Initialized Variables to guide file uploads and downloads\n",
    "input_dir = '../data' # path for user's pre-existing data files to read\n",
    "output_dir = '../output' # direct csv downloads; to be updated with new subdirectories for each task \n",
    "read_type = 'url' # or 'disk'\n",
    "\n",
    "url = \"https://ftp.ncbi.nlm.nih.gov/geo/series/GSE185nnn/GSE185948/suppl/GSE185948_metadata_RNA.csv.gz\"\n",
    "file_name1 = 'demographic_clinical_data.csv' # user chosen file name for upload from url\n",
    "file_name2 = 'RDS_data.csv'\n",
    "file_path = \"GSE185948_count_RNA.rds\" # file name in user's disk\n",
    "\n",
    "# For constructing RDS data into readable arrays/dataframes\n",
    "dim_0 = 'gene_name' # rows\n",
    "dim_1 = 'cell_name' # cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3d37fcf0-d788-400d-95bf-10695ec2808f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<<<<<<<<<< URL File Read Successfully >>>>>>>>>>\n",
      "<<<<<<<<<< Processed data saved to ../data/demographic_clinical_data.csv >>>>>>>>>>\n"
     ]
    }
   ],
   "source": [
    "# Read demographic csv file from url\n",
    "df1 = mu.get_data(input_dir, url = url, file_name = file_name1,read_type='url')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5637b725-53a2-4e43-becd-67703ba6e841",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<<<<<<<<<< RDS File Read Successfully >>>>>>>>>>\n"
     ]
    }
   ],
   "source": [
    "# Read rds file from local disk\n",
    "df2 = mu.get_data(input_dir, file_name = file_name2, file_path= file_path, read_type='disk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "22da37d4-01a8-4bcb-a7f8-568272873904",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (102_710, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>name</th><th>barcode</th><th>patient</th><th>gender</th><th>disease</th><th>celltype</th><th>nCount_RNA</th><th>nFeature_RNA</th><th>UMAP_1</th><th>UMAP_2</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>i64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;PKD_ACACGCGGTA…</td><td>&quot;ACACGCGGTATCGG…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;TAL1&quot;</td><td>1234.684629</td><td>1250</td><td>-7.41602</td><td>-6.008874</td></tr><tr><td>&quot;PKD_ACACGCGGTT…</td><td>&quot;ACACGCGGTTTGGC…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;PT2&quot;</td><td>1865.542588</td><td>1650</td><td>2.499409</td><td>-6.587287</td></tr><tr><td>&quot;PKD_ACACGCGTCA…</td><td>&quot;ACACGCGTCATGTC…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;CNT_PC&quot;</td><td>1812.700523</td><td>1419</td><td>-2.254505</td><td>8.526364</td></tr><tr><td>&quot;PKD_ACACTGAAGA…</td><td>&quot;ACACTGAAGACCCT…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;FIB&quot;</td><td>978.772591</td><td>1089</td><td>9.949437</td><td>2.086737</td></tr><tr><td>&quot;PKD_ACACTGAAGC…</td><td>&quot;ACACTGAAGCGACA…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;PT1&quot;</td><td>2361.558871</td><td>2013</td><td>7.886921</td><td>-8.587954</td></tr><tr><td>&quot;PKD_ACACTGAAGT…</td><td>&quot;ACACTGAAGTTCGG…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;PT1&quot;</td><td>1022.824714</td><td>1016</td><td>7.670309</td><td>-7.718168</td></tr><tr><td>&quot;PKD_ACACTGACAA…</td><td>&quot;ACACTGACAAGGTA…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;DCT&quot;</td><td>2482.698271</td><td>1877</td><td>-8.110249</td><td>5.433267</td></tr><tr><td>&quot;PKD_ACACTGACAC…</td><td>&quot;ACACTGACACCGTC…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;FIB&quot;</td><td>1088.176821</td><td>1232</td><td>8.026856</td><td>1.465231</td></tr><tr><td>&quot;PKD_ACACTGACAC…</td><td>&quot;ACACTGACACCTTC…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;DCT&quot;</td><td>2553.667284</td><td>2006</td><td>-9.598119</td><td>6.063181</td></tr><tr><td>&quot;PKD_ACACTGACAG…</td><td>&quot;ACACTGACAGAGCC…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;TAL1&quot;</td><td>1515.820655</td><td>1499</td><td>-7.198015</td><td>-4.930681</td></tr><tr><td>&quot;PKD_ACACTGAGTA…</td><td>&quot;ACACTGAGTACCTA…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;DCT&quot;</td><td>2752.752142</td><td>2046</td><td>-8.675308</td><td>5.65736</td></tr><tr><td>&quot;PKD_ACACTGATCA…</td><td>&quot;ACACTGATCACAGA…</td><td>&quot;PKD1&quot;</td><td>&quot;female&quot;</td><td>&quot;PKD&quot;</td><td>&quot;CNT_PC&quot;</td><td>2674.427313</td><td>1788</td><td>-0.308515</td><td>8.673335</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Cont_TTTGGTTGT…</td><td>&quot;TTTGGTTGTCCACA…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;TAL1&quot;</td><td>3837.066091</td><td>2418</td><td>-5.329015</td><td>-2.819633</td></tr><tr><td>&quot;Cont_TTTGGTTGT…</td><td>&quot;TTTGGTTGTTTCAC…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;DCT&quot;</td><td>2423.301009</td><td>1749</td><td>-10.787855</td><td>6.041763</td></tr><tr><td>&quot;Cont_TTTGGTTTC…</td><td>&quot;TTTGGTTTCCGCAA…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;ENDO&quot;</td><td>797.469596</td><td>768</td><td>2.654418</td><td>3.949362</td></tr><tr><td>&quot;Cont_TTTGGTTTC…</td><td>&quot;TTTGGTTTCTAGTC…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;ICA&quot;</td><td>1759.332678</td><td>1212</td><td>2.766891</td><td>16.446197</td></tr><tr><td>&quot;Cont_TTTGTTGAG…</td><td>&quot;TTTGTTGAGTCGTC…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;TAL1&quot;</td><td>1364.414801</td><td>1083</td><td>-7.282649</td><td>-2.538467</td></tr><tr><td>&quot;Cont_TTTGTTGCA…</td><td>&quot;TTTGTTGCACAAGC…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;PT1&quot;</td><td>2480.306112</td><td>1612</td><td>7.831077</td><td>-8.54625</td></tr><tr><td>&quot;Cont_TTTGTTGCA…</td><td>&quot;TTTGTTGCACGTTC…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;TAL1&quot;</td><td>2263.964405</td><td>1623</td><td>-7.192175</td><td>-6.920322</td></tr><tr><td>&quot;Cont_TTTGTTGCA…</td><td>&quot;TTTGTTGCAGTTAA…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;DCT&quot;</td><td>2177.45694</td><td>1537</td><td>-8.038441</td><td>6.70523</td></tr><tr><td>&quot;Cont_TTTGTTGCA…</td><td>&quot;TTTGTTGCATATTC…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;PT1&quot;</td><td>1116.704812</td><td>984</td><td>5.658295</td><td>-6.785034</td></tr><tr><td>&quot;Cont_TTTGTTGGT…</td><td>&quot;TTTGTTGGTACGAT…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;DCT&quot;</td><td>2156.953541</td><td>1421</td><td>-8.645567</td><td>6.688504</td></tr><tr><td>&quot;Cont_TTTGTTGGT…</td><td>&quot;TTTGTTGGTGCGTT…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;CNT_PC&quot;</td><td>1553.371592</td><td>1209</td><td>-0.795698</td><td>7.423819</td></tr><tr><td>&quot;Cont_TTTGTTGTC…</td><td>&quot;TTTGTTGTCTGCGG…</td><td>&quot;control5&quot;</td><td>&quot;female&quot;</td><td>&quot;control&quot;</td><td>&quot;ENDO&quot;</td><td>1106.715246</td><td>917</td><td>2.259402</td><td>3.275315</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (102_710, 10)\n",
       "┌────────────┬────────────┬──────────┬────────┬───┬────────────┬───────────┬───────────┬───────────┐\n",
       "│ name       ┆ barcode    ┆ patient  ┆ gender ┆ … ┆ nCount_RNA ┆ nFeature_ ┆ UMAP_1    ┆ UMAP_2    │\n",
       "│ ---        ┆ ---        ┆ ---      ┆ ---    ┆   ┆ ---        ┆ RNA       ┆ ---       ┆ ---       │\n",
       "│ str        ┆ str        ┆ str      ┆ str    ┆   ┆ f64        ┆ ---       ┆ f64       ┆ f64       │\n",
       "│            ┆            ┆          ┆        ┆   ┆            ┆ i64       ┆           ┆           │\n",
       "╞════════════╪════════════╪══════════╪════════╪═══╪════════════╪═══════════╪═══════════╪═══════════╡\n",
       "│ PKD_ACACGC ┆ ACACGCGGTA ┆ PKD1     ┆ female ┆ … ┆ 1234.68462 ┆ 1250      ┆ -7.41602  ┆ -6.008874 │\n",
       "│ GGTATCGGTT ┆ TCGGTT-1   ┆          ┆        ┆   ┆ 9          ┆           ┆           ┆           │\n",
       "│ -1_1       ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ PKD_ACACGC ┆ ACACGCGGTT ┆ PKD1     ┆ female ┆ … ┆ 1865.54258 ┆ 1650      ┆ 2.499409  ┆ -6.587287 │\n",
       "│ GGTTTGGCTA ┆ TGGCTA-1   ┆          ┆        ┆   ┆ 8          ┆           ┆           ┆           │\n",
       "│ -1_1       ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ PKD_ACACGC ┆ ACACGCGTCA ┆ PKD1     ┆ female ┆ … ┆ 1812.70052 ┆ 1419      ┆ -2.254505 ┆ 8.526364  │\n",
       "│ GTCATGTCTT ┆ TGTCTT-1   ┆          ┆        ┆   ┆ 3          ┆           ┆           ┆           │\n",
       "│ -1_1       ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ PKD_ACACTG ┆ ACACTGAAGA ┆ PKD1     ┆ female ┆ … ┆ 978.772591 ┆ 1089      ┆ 9.949437  ┆ 2.086737  │\n",
       "│ AAGACCCTTA ┆ CCCTTA-1   ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ -1_1       ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ …          ┆ …          ┆ …        ┆ …      ┆ … ┆ …          ┆ …         ┆ …         ┆ …         │\n",
       "│ Cont_TTTGT ┆ TTTGTTGCAT ┆ control5 ┆ female ┆ … ┆ 1116.70481 ┆ 984       ┆ 5.658295  ┆ -6.785034 │\n",
       "│ TGCATATTCG ┆ ATTCGG-1   ┆          ┆        ┆   ┆ 2          ┆           ┆           ┆           │\n",
       "│ G-1_5      ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ Cont_TTTGT ┆ TTTGTTGGTA ┆ control5 ┆ female ┆ … ┆ 2156.95354 ┆ 1421      ┆ -8.645567 ┆ 6.688504  │\n",
       "│ TGGTACGATT ┆ CGATTC-1   ┆          ┆        ┆   ┆ 1          ┆           ┆           ┆           │\n",
       "│ C-1_5      ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ Cont_TTTGT ┆ TTTGTTGGTG ┆ control5 ┆ female ┆ … ┆ 1553.37159 ┆ 1209      ┆ -0.795698 ┆ 7.423819  │\n",
       "│ TGGTGCGTTT ┆ CGTTTA-1   ┆          ┆        ┆   ┆ 2          ┆           ┆           ┆           │\n",
       "│ A-1_5      ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "│ Cont_TTTGT ┆ TTTGTTGTCT ┆ control5 ┆ female ┆ … ┆ 1106.71524 ┆ 917       ┆ 2.259402  ┆ 3.275315  │\n",
       "│ TGTCTGCGGA ┆ GCGGAC-1   ┆          ┆        ┆   ┆ 6          ┆           ┆           ┆           │\n",
       "│ C-1_5      ┆            ┆          ┆        ┆   ┆            ┆           ┆           ┆           │\n",
       "└────────────┴────────────┴──────────┴────────┴───┴────────────┴───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "114be754-5541-46c5-94e5-5d194fa097ce",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (27_970, 102_710)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>PKD_ACACGCGGTATCGGTT-1_1</th><th>PKD_ACACGCGGTTTGGCTA-1_1</th><th>PKD_ACACGCGTCATGTCTT-1_1</th><th>PKD_ACACTGAAGACCCTTA-1_1</th><th>PKD_ACACTGAAGCGACAGT-1_1</th><th>PKD_ACACTGAAGTTCGGTT-1_1</th><th>PKD_ACACTGACAAGGTACG-1_1</th><th>PKD_ACACTGACACCGTCTT-1_1</th><th>PKD_ACACTGACACCTTCGT-1_1</th><th>PKD_ACACTGACAGAGCCCT-1_1</th><th>PKD_ACACTGAGTACCTAAC-1_1</th><th>PKD_ACACTGATCACAGAGG-1_1</th><th>PKD_ACACTGATCACTTGGA-1_1</th><th>PKD_ACACTGATCGCCAGAC-1_1</th><th>PKD_ACAGAAAAGATCACTC-1_1</th><th>PKD_ACAGAAAAGGTTCTAC-1_1</th><th>PKD_ACAGAAAAGTACCATC-1_1</th><th>PKD_ACAGAAACAATCAAGA-1_1</th><th>PKD_ACAGAAACACAAACGG-1_1</th><th>PKD_ACAGAAACAGCTCCTT-1_1</th><th>PKD_ACAGAAAGTAAGCAAT-1_1</th><th>PKD_ACAGAAAGTACCGTCG-1_1</th><th>PKD_ACAGAAAGTCCGGTGT-1_1</th><th>PKD_ACAGAAAGTCTTTCAT-1_1</th><th>PKD_ACAGAAATCCGGACTG-1_1</th><th>PKD_ACAGAAATCTCGCAGG-1_1</th><th>PKD_ACAGCCGAGATAACGT-1_1</th><th>PKD_ACAGCCGAGCGACTAG-1_1</th><th>PKD_ACAGCCGAGCGCGTTC-1_1</th><th>PKD_ACAGCCGAGCTCCATA-1_1</th><th>PKD_ACAGCCGAGGGCAGTT-1_1</th><th>PKD_ACAGCCGCACACCGCA-1_1</th><th>PKD_ACAGCCGCACGGTGTC-1_1</th><th>PKD_ACAGCCGCAGATCATC-1_1</th><th>PKD_ACAGCCGGTACTGCGC-1_1</th><th>PKD_ACAGCCGGTTAGTCGT-1_1</th><th>PKD_ACAGCCGTCACCATGA-1_1</th><th>&hellip;</th><th>Cont_TTTGATCAGCTGGCCT-1_5</th><th>Cont_TTTGATCCAAGTCCAT-1_5</th><th>Cont_TTTGATCCAGCTGTTA-1_5</th><th>Cont_TTTGATCGTACTTGTG-1_5</th><th>Cont_TTTGATCGTATGCTAC-1_5</th><th>Cont_TTTGATCGTCTTCGAA-1_5</th><th>Cont_TTTGATCGTTGCGGAA-1_5</th><th>Cont_TTTGATCTCGCGTGCA-1_5</th><th>Cont_TTTGATCTCGGTTCAA-1_5</th><th>Cont_TTTGGAGAGCATAGGC-1_5</th><th>Cont_TTTGGAGAGTAACCTC-1_5</th><th>Cont_TTTGGAGAGTGTTGTC-1_5</th><th>Cont_TTTGGAGCAAATCGGG-1_5</th><th>Cont_TTTGGAGCACAGTCCG-1_5</th><th>Cont_TTTGGAGCACTCTAGA-1_5</th><th>Cont_TTTGGAGCAGAACTCT-1_5</th><th>Cont_TTTGGAGCATCGCTGG-1_5</th><th>Cont_TTTGGAGGTACCCAGC-1_5</th><th>Cont_TTTGGAGGTATCGTTG-1_5</th><th>Cont_TTTGGAGTCGAATCCA-1_5</th><th>Cont_TTTGGAGTCTCTATGT-1_5</th><th>Cont_TTTGGTTAGGTCCCGT-1_5</th><th>Cont_TTTGGTTAGTAAAGCT-1_5</th><th>Cont_TTTGGTTCAACCGCCA-1_5</th><th>Cont_TTTGGTTCAATCCTAG-1_5</th><th>Cont_TTTGGTTGTCCACACG-1_5</th><th>Cont_TTTGGTTGTTTCACAG-1_5</th><th>Cont_TTTGGTTTCCGCAACG-1_5</th><th>Cont_TTTGGTTTCTAGTCAG-1_5</th><th>Cont_TTTGTTGAGTCGTCTA-1_5</th><th>Cont_TTTGTTGCACAAGCCC-1_5</th><th>Cont_TTTGTTGCACGTTCGG-1_5</th><th>Cont_TTTGTTGCAGTTAAAG-1_5</th><th>Cont_TTTGTTGCATATTCGG-1_5</th><th>Cont_TTTGTTGGTACGATTC-1_5</th><th>Cont_TTTGTTGGTGCGTTTA-1_5</th><th>Cont_TTTGTTGTCTGCGGAC-1_5</th></tr><tr><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>&hellip;</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.680108</td><td>0.0</td><td>0.0</td><td>0.770796</td><td>0.0</td><td>0.0</td><td>0.482712</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.61749</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.795865</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.771729</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.719765</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.875772</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.346982</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.126597</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.252593</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1.495558</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.553609</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (27_970, 102_710)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ PKD_ACACG ┆ PKD_ACACG ┆ PKD_ACACG ┆ PKD_ACACT ┆ … ┆ Cont_TTTG ┆ Cont_TTTG ┆ Cont_TTTG ┆ Cont_TTT │\n",
       "│ CGGTATCGG ┆ CGGTTTGGC ┆ CGTCATGTC ┆ GAAGACCCT ┆   ┆ TTGCATATT ┆ TTGGTACGA ┆ TTGGTGCGT ┆ GTTGTCTG │\n",
       "│ TT-1_1    ┆ TA-1_1    ┆ TT-1_1    ┆ TA-1_1    ┆   ┆ CGG-1_5   ┆ TTC-1_5   ┆ TTA-1_5   ┆ CGGAC-1_ │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ 5        │\n",
       "│ f64       ┆ f64       ┆ f64       ┆ f64       ┆   ┆ f64       ┆ f64       ┆ f64       ┆ ---      │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ f64      │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "187c64a9-a764-4256-a5fa-afe5f9bb2291",
   "metadata": {},
   "source": [
    "# **TASK 1: BASELINE**\n",
    ">### **Phase 2**\n",
    ">Cleaning the RDS file and reformatting with mapping and aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0dc86796-8daf-4ffe-acb7-ca61faa1c124",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''# mapping new attributes from other dataframes function    \n",
    "def df_mapping(df1, df2, mapping_attribute = None, key_attribute = None):\n",
    "    new_df = df2\n",
    "    new_vals = {}\n",
    "    count = 1\n",
    "    unique_instances = df1[mapping_attribute].unique()\n",
    "    mapping = dict(zip(df1[key_attribute], df1[mapping_attribute]))\n",
    "    for i in df2.columns:\n",
    "        new_vals[i] = str(mapping[i]+','+ str(count))\n",
    "        count = count+1\n",
    "    print(new_vals)\n",
    "    new_df = new_df.rename(new_vals)\n",
    "    #new_df.with_columns(pl.Series(name=mapping_attribute, values=map_values)) \n",
    "    #new_df[mapping_attribute] = map_values\n",
    "    #if new_index == True:\n",
    "        #new_index = [\"{}_{}\".format(key_attribute, mapping_attribute) for key_attribute, mapping_attribute in zip(transpose_df.index, map_values)]\n",
    "        #new_df.index = new_index\n",
    "    return new_df\n",
    "\n",
    "def unique_group(df1, grouping_attribute, df2=None):\n",
    "\n",
    "    # create dictionary of unique instances of passed attribute in first dataframe\n",
    "    unique_instances = df1[grouping_attribute].unique() \n",
    "    grouped_columns = {grouping_attribute: [] for grouping_attribute in unique_instances}\n",
    "    \n",
    "    # fill dictionary with second dataframe's data\n",
    "    if df2 is not None: \n",
    "        for col in df2.columns:\n",
    "            for grouping_attribute in unique_instances:\n",
    "                if grouping_attribute in col:\n",
    "                    grouped_columns[grouping_attribute].append(col)\n",
    "                    break\n",
    "    # fill dictionary with first dataframe's data\n",
    "    else:\n",
    "        for col in df1.columns:\n",
    "            for grouping_attribute in unique_instances:\n",
    "                if grouping_attribute in col:\n",
    "                    grouped_columns[grouping_attribute].append(col)\n",
    "                    break\n",
    "    return grouped_columns\n",
    "    \n",
    "\n",
    "def aggregate_by_groups(df, grouped_columns):\n",
    "    # Create an empty Polars DataFrame to store the aggregated results\n",
    "    aggregated_df = pd.DataFrame()\n",
    "    \n",
    "    for group_name, cols in grouped_columns.items():\n",
    "        if cols:  # Check if any columns exist for that group\n",
    "            # Calculate the mean for the selected columns\n",
    "            #mean_values = df.select(*cols).mean(axis=1)\n",
    "            mean_values = (df.select(cols).mean(axis=1)).to_pandas()\n",
    "            aggregated_df[group_name] = mean_values\n",
    "\n",
    "    return aggregated_df'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31375073-82b1-4ae3-8adf-cb34b0a6210a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# update output path to subdirectory of 'data' for Task 1\n",
    "output_path = os.path.join(output_dir, 'task1')\n",
    "\n",
    "#transpose raw molecular dataframe to map 'patient' attributes from df1\n",
    "#transpose_df = df2.T\n",
    "mapped_df2 = dp.df_mapping(df1,df2, mapping_attribute = 'patient', key_attribute = 'name')\n",
    "mapped_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2d9019c9-68a2-4cf2-b4ed-030f47e552d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27970, 102710)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapped_df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "124b9dfe-5bf8-47b2-b36b-232fb981cf9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = dp.unique_group(df1, 'patient', df2=mapped_df2)\n",
    "grouped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f51c9cd4-45ed-4163-a130-e42af0c180b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyarrow in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (14.0.1)\n",
      "Requirement already satisfied: numpy>=1.16.6 in /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages (from pyarrow) (1.26.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e6e2a2a8-1ae9-49e2-895f-5c99536d8bda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PKD4</th>\n",
       "      <th>PKD8</th>\n",
       "      <th>control2</th>\n",
       "      <th>control5</th>\n",
       "      <th>control3</th>\n",
       "      <th>PKD6</th>\n",
       "      <th>control4</th>\n",
       "      <th>PKD3</th>\n",
       "      <th>PKD1</th>\n",
       "      <th>PKD7</th>\n",
       "      <th>control1</th>\n",
       "      <th>PKD2</th>\n",
       "      <th>PKD5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.002954</td>\n",
       "      <td>0.004001</td>\n",
       "      <td>0.005165</td>\n",
       "      <td>0.002579</td>\n",
       "      <td>0.005716</td>\n",
       "      <td>0.002991</td>\n",
       "      <td>0.005334</td>\n",
       "      <td>0.001086</td>\n",
       "      <td>0.000753</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003540</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.001941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.007549</td>\n",
       "      <td>0.032600</td>\n",
       "      <td>0.011323</td>\n",
       "      <td>0.006374</td>\n",
       "      <td>0.012446</td>\n",
       "      <td>0.007005</td>\n",
       "      <td>0.015720</td>\n",
       "      <td>0.001668</td>\n",
       "      <td>0.004445</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.014732</td>\n",
       "      <td>0.007223</td>\n",
       "      <td>0.010352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.022174</td>\n",
       "      <td>0.112791</td>\n",
       "      <td>0.079155</td>\n",
       "      <td>0.032347</td>\n",
       "      <td>0.084424</td>\n",
       "      <td>0.068845</td>\n",
       "      <td>0.117683</td>\n",
       "      <td>0.044862</td>\n",
       "      <td>0.042604</td>\n",
       "      <td>0.068695</td>\n",
       "      <td>0.066192</td>\n",
       "      <td>0.056001</td>\n",
       "      <td>0.041855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.018220</td>\n",
       "      <td>0.046928</td>\n",
       "      <td>0.023825</td>\n",
       "      <td>0.013867</td>\n",
       "      <td>0.020626</td>\n",
       "      <td>0.019986</td>\n",
       "      <td>0.067427</td>\n",
       "      <td>0.018115</td>\n",
       "      <td>0.006373</td>\n",
       "      <td>0.026209</td>\n",
       "      <td>0.031483</td>\n",
       "      <td>0.026931</td>\n",
       "      <td>0.021822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000547</td>\n",
       "      <td>0.001977</td>\n",
       "      <td>0.009599</td>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.006503</td>\n",
       "      <td>0.003146</td>\n",
       "      <td>0.007133</td>\n",
       "      <td>0.000683</td>\n",
       "      <td>0.002645</td>\n",
       "      <td>0.001423</td>\n",
       "      <td>0.005242</td>\n",
       "      <td>0.001561</td>\n",
       "      <td>0.002424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27965</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003724</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27966</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000559</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27967</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001627</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27968</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000901</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27969</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27970 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PKD4      PKD8  control2  control5  control3      PKD6  control4  \\\n",
       "0      0.002954  0.004001  0.005165  0.002579  0.005716  0.002991  0.005334   \n",
       "1      0.007549  0.032600  0.011323  0.006374  0.012446  0.007005  0.015720   \n",
       "2      0.022174  0.112791  0.079155  0.032347  0.084424  0.068845  0.117683   \n",
       "3      0.018220  0.046928  0.023825  0.013867  0.020626  0.019986  0.067427   \n",
       "4      0.000547  0.001977  0.009599  0.003132  0.006503  0.003146  0.007133   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "27965  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.003724   \n",
       "27966  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000559   \n",
       "27967  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.001627   \n",
       "27968  0.000000  0.000000  0.000000  0.000901  0.000000  0.000000  0.000000   \n",
       "27969  0.000000  0.000000  0.000000  0.000603  0.000000  0.000000  0.000000   \n",
       "\n",
       "           PKD3      PKD1      PKD7  control1      PKD2      PKD5  \n",
       "0      0.001086  0.000753  0.000000  0.003540  0.001885  0.001941  \n",
       "1      0.001668  0.004445  0.008997  0.014732  0.007223  0.010352  \n",
       "2      0.044862  0.042604  0.068695  0.066192  0.056001  0.041855  \n",
       "3      0.018115  0.006373  0.026209  0.031483  0.026931  0.021822  \n",
       "4      0.000683  0.002645  0.001423  0.005242  0.001561  0.002424  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "27965  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "27966  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "27967  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "27968  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "27969  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "\n",
       "[27970 rows x 13 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregated_df2 = pd.DataFrame(dp.aggregate_by_groups(mapped_df2, grouped))\n",
    "aggregated_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dce6d27e-23d8-4cae-9fdf-d30eb2c683cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PKD1</th>\n",
       "      <th>PKD2</th>\n",
       "      <th>PKD3</th>\n",
       "      <th>PKD4</th>\n",
       "      <th>PKD5</th>\n",
       "      <th>PKD6</th>\n",
       "      <th>PKD7</th>\n",
       "      <th>PKD8</th>\n",
       "      <th>control1</th>\n",
       "      <th>control2</th>\n",
       "      <th>control3</th>\n",
       "      <th>control4</th>\n",
       "      <th>control5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000753</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.001086</td>\n",
       "      <td>0.002954</td>\n",
       "      <td>0.001941</td>\n",
       "      <td>0.002991</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004001</td>\n",
       "      <td>0.003540</td>\n",
       "      <td>0.005165</td>\n",
       "      <td>0.005716</td>\n",
       "      <td>0.005334</td>\n",
       "      <td>0.002579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.004445</td>\n",
       "      <td>0.007223</td>\n",
       "      <td>0.001668</td>\n",
       "      <td>0.007549</td>\n",
       "      <td>0.010352</td>\n",
       "      <td>0.007005</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.032600</td>\n",
       "      <td>0.014732</td>\n",
       "      <td>0.011323</td>\n",
       "      <td>0.012446</td>\n",
       "      <td>0.015720</td>\n",
       "      <td>0.006374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.042604</td>\n",
       "      <td>0.056001</td>\n",
       "      <td>0.044862</td>\n",
       "      <td>0.022174</td>\n",
       "      <td>0.041855</td>\n",
       "      <td>0.068845</td>\n",
       "      <td>0.068695</td>\n",
       "      <td>0.112791</td>\n",
       "      <td>0.066192</td>\n",
       "      <td>0.079155</td>\n",
       "      <td>0.084424</td>\n",
       "      <td>0.117683</td>\n",
       "      <td>0.032347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.006373</td>\n",
       "      <td>0.026931</td>\n",
       "      <td>0.018115</td>\n",
       "      <td>0.018220</td>\n",
       "      <td>0.021822</td>\n",
       "      <td>0.019986</td>\n",
       "      <td>0.026209</td>\n",
       "      <td>0.046928</td>\n",
       "      <td>0.031483</td>\n",
       "      <td>0.023825</td>\n",
       "      <td>0.020626</td>\n",
       "      <td>0.067427</td>\n",
       "      <td>0.013867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.002645</td>\n",
       "      <td>0.001561</td>\n",
       "      <td>0.000683</td>\n",
       "      <td>0.000547</td>\n",
       "      <td>0.002424</td>\n",
       "      <td>0.003146</td>\n",
       "      <td>0.001423</td>\n",
       "      <td>0.001977</td>\n",
       "      <td>0.005242</td>\n",
       "      <td>0.009599</td>\n",
       "      <td>0.006503</td>\n",
       "      <td>0.007133</td>\n",
       "      <td>0.003132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27965</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003724</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27966</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000559</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27967</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001627</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27968</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27969</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000603</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27970 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PKD1      PKD2      PKD3      PKD4      PKD5      PKD6      PKD7  \\\n",
       "0      0.000753  0.001885  0.001086  0.002954  0.001941  0.002991  0.000000   \n",
       "1      0.004445  0.007223  0.001668  0.007549  0.010352  0.007005  0.008997   \n",
       "2      0.042604  0.056001  0.044862  0.022174  0.041855  0.068845  0.068695   \n",
       "3      0.006373  0.026931  0.018115  0.018220  0.021822  0.019986  0.026209   \n",
       "4      0.002645  0.001561  0.000683  0.000547  0.002424  0.003146  0.001423   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "27965  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "27966  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "27967  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "27968  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "27969  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "           PKD8  control1  control2  control3  control4  control5  \n",
       "0      0.004001  0.003540  0.005165  0.005716  0.005334  0.002579  \n",
       "1      0.032600  0.014732  0.011323  0.012446  0.015720  0.006374  \n",
       "2      0.112791  0.066192  0.079155  0.084424  0.117683  0.032347  \n",
       "3      0.046928  0.031483  0.023825  0.020626  0.067427  0.013867  \n",
       "4      0.001977  0.005242  0.009599  0.006503  0.007133  0.003132  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "27965  0.000000  0.000000  0.000000  0.000000  0.003724  0.000000  \n",
       "27966  0.000000  0.000000  0.000000  0.000000  0.000559  0.000000  \n",
       "27967  0.000000  0.000000  0.000000  0.000000  0.001627  0.000000  \n",
       "27968  0.000000  0.000000  0.000000  0.000000  0.000000  0.000901  \n",
       "27969  0.000000  0.000000  0.000000  0.000000  0.000000  0.000603  \n",
       "\n",
       "[27970 rows x 13 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregated_df2 = aggregated_df2.reindex(sorted(aggregated_df2.columns), axis=1)\n",
    "aggregated_df2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c511d84-37f0-415c-a90f-5db113c148c1",
   "metadata": {},
   "source": [
    "### Split train & Test then Pipeline for pre-Phase 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "47e4d7ef-1b70-41fc-bfb5-70001eadcf08",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "def split_data(df, test_size=0.3, random_state=None):\n",
    "    #Splits a DataFrame into training and test sets.\n",
    "\n",
    "    train_df, test_df = train_test_split(df, test_size=test_size, random_state=random_state)\n",
    "\n",
    "    return train_df, test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b35217c7-57b5-4156-8508-91bba0fe4bd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PKD1</th>\n",
       "      <th>PKD2</th>\n",
       "      <th>PKD3</th>\n",
       "      <th>PKD4</th>\n",
       "      <th>PKD5</th>\n",
       "      <th>PKD6</th>\n",
       "      <th>PKD7</th>\n",
       "      <th>PKD8</th>\n",
       "      <th>control1</th>\n",
       "      <th>control2</th>\n",
       "      <th>control3</th>\n",
       "      <th>control4</th>\n",
       "      <th>control5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4682</th>\n",
       "      <td>0.058530</td>\n",
       "      <td>0.115151</td>\n",
       "      <td>0.049917</td>\n",
       "      <td>0.031766</td>\n",
       "      <td>0.099263</td>\n",
       "      <td>0.148055</td>\n",
       "      <td>0.060874</td>\n",
       "      <td>0.079229</td>\n",
       "      <td>0.181459</td>\n",
       "      <td>0.206016</td>\n",
       "      <td>0.137439</td>\n",
       "      <td>0.223114</td>\n",
       "      <td>0.089336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5478</th>\n",
       "      <td>0.005232</td>\n",
       "      <td>0.093129</td>\n",
       "      <td>0.010274</td>\n",
       "      <td>0.007166</td>\n",
       "      <td>0.008140</td>\n",
       "      <td>0.034626</td>\n",
       "      <td>0.077727</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000989</td>\n",
       "      <td>0.005326</td>\n",
       "      <td>0.001711</td>\n",
       "      <td>0.000400</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21732</th>\n",
       "      <td>0.001375</td>\n",
       "      <td>0.001636</td>\n",
       "      <td>0.000295</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001256</td>\n",
       "      <td>0.004061</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005882</td>\n",
       "      <td>0.003810</td>\n",
       "      <td>0.003133</td>\n",
       "      <td>0.006256</td>\n",
       "      <td>0.001839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24868</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000243</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000737</td>\n",
       "      <td>0.000622</td>\n",
       "      <td>0.000701</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001361</td>\n",
       "      <td>0.000406</td>\n",
       "      <td>0.000996</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18199</th>\n",
       "      <td>0.018895</td>\n",
       "      <td>0.006694</td>\n",
       "      <td>0.022041</td>\n",
       "      <td>0.005042</td>\n",
       "      <td>0.003015</td>\n",
       "      <td>0.039180</td>\n",
       "      <td>0.015758</td>\n",
       "      <td>0.089860</td>\n",
       "      <td>0.002654</td>\n",
       "      <td>0.004407</td>\n",
       "      <td>0.002012</td>\n",
       "      <td>0.002453</td>\n",
       "      <td>0.001323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16990</th>\n",
       "      <td>0.021699</td>\n",
       "      <td>0.070669</td>\n",
       "      <td>0.019897</td>\n",
       "      <td>0.016170</td>\n",
       "      <td>0.089503</td>\n",
       "      <td>0.083157</td>\n",
       "      <td>0.025825</td>\n",
       "      <td>0.063774</td>\n",
       "      <td>0.111850</td>\n",
       "      <td>0.144766</td>\n",
       "      <td>0.066968</td>\n",
       "      <td>0.145112</td>\n",
       "      <td>0.049720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24300</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000916</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000542</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14980</th>\n",
       "      <td>0.041159</td>\n",
       "      <td>0.048345</td>\n",
       "      <td>0.039836</td>\n",
       "      <td>0.021471</td>\n",
       "      <td>0.064940</td>\n",
       "      <td>0.032385</td>\n",
       "      <td>0.029096</td>\n",
       "      <td>0.030616</td>\n",
       "      <td>0.119178</td>\n",
       "      <td>0.161531</td>\n",
       "      <td>0.106662</td>\n",
       "      <td>0.160736</td>\n",
       "      <td>0.082489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18383</th>\n",
       "      <td>0.000514</td>\n",
       "      <td>0.001109</td>\n",
       "      <td>0.002086</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003125</td>\n",
       "      <td>0.002497</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001009</td>\n",
       "      <td>0.002684</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001674</td>\n",
       "      <td>0.001158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24015</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000641</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19579 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PKD1      PKD2      PKD3      PKD4      PKD5      PKD6      PKD7  \\\n",
       "4682   0.058530  0.115151  0.049917  0.031766  0.099263  0.148055  0.060874   \n",
       "5478   0.005232  0.093129  0.010274  0.007166  0.008140  0.034626  0.077727   \n",
       "21732  0.001375  0.001636  0.000295  0.000000  0.001256  0.004061  0.000000   \n",
       "24868  0.000000  0.000243  0.000000  0.000737  0.000622  0.000701  0.000000   \n",
       "18199  0.018895  0.006694  0.022041  0.005042  0.003015  0.039180  0.015758   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "16990  0.021699  0.070669  0.019897  0.016170  0.089503  0.083157  0.025825   \n",
       "24300  0.000000  0.000916  0.000000  0.000000  0.000542  0.000000  0.000000   \n",
       "14980  0.041159  0.048345  0.039836  0.021471  0.064940  0.032385  0.029096   \n",
       "18383  0.000514  0.001109  0.002086  0.000000  0.003125  0.002497  0.000000   \n",
       "24015  0.000000  0.000641  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "           PKD8  control1  control2  control3  control4  control5  \n",
       "4682   0.079229  0.181459  0.206016  0.137439  0.223114  0.089336  \n",
       "5478   0.000000  0.000989  0.005326  0.001711  0.000400  0.000000  \n",
       "21732  0.000000  0.005882  0.003810  0.003133  0.006256  0.001839  \n",
       "24868  0.000000  0.000000  0.001361  0.000406  0.000996  0.000000  \n",
       "18199  0.089860  0.002654  0.004407  0.002012  0.002453  0.001323  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "16990  0.063774  0.111850  0.144766  0.066968  0.145112  0.049720  \n",
       "24300  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "14980  0.030616  0.119178  0.161531  0.106662  0.160736  0.082489  \n",
       "18383  0.000000  0.001009  0.002684  0.000000  0.001674  0.001158  \n",
       "24015  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "\n",
       "[19579 rows x 13 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df, test_df = split_data(aggregated_df2)\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "2474b4a0-8044-4e16-b1ad-a5a03952c26a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def drop_duplicates(df): \n",
    "    print('\\nDuplicate row removal:')\n",
    "    print('Sample count before: ', len(df.index))\n",
    "    df_no_duplicates = df.drop_duplicates()\n",
    "    print('Sample count after: ', len(df_no_duplicates.index))\n",
    "\n",
    "    return df_no_duplicates\n",
    "def drop_null(df):\n",
    "     # Drop rows with null values\n",
    "    df_no_nulls = df.dropna()\n",
    "\n",
    "    # Drop rows with 0 entries\n",
    "    df_no_nulls = df_no_nulls[(df_no_nulls != 0).all(axis=1)]\n",
    "    print('\\nNull row removal:')\n",
    "    print('Sample count before: ', len(df.index))\n",
    "    print('Sample count after: ', len(df_no_nulls.index))\n",
    "    \n",
    "    return df_no_nulls\n",
    "\n",
    "def drop_out_of_domain(df, std = 2):\n",
    "    # Get only numeric data to identify rows with out-of-domain properties\n",
    "    df_numeric = numeric_only(df)\n",
    "    \n",
    "    # Calculate the mean and standard deviation of all row means\n",
    "    print('\\nOut-of-domain row removal:')\n",
    "    all_rows_mean = df_numeric.mean(axis=1)\n",
    "    all_rows_mean_mean = all_rows_mean.mean()\n",
    "    all_rows_mean_std = all_rows_mean.std()\n",
    "    threshold_std = std\n",
    "    threshold = all_rows_mean_mean + threshold_std * all_rows_mean_std\n",
    "    print('Threshold =', threshold_std, ' standard deviations')\n",
    "\n",
    "    out_of_domain_indices = []\n",
    "    \n",
    "    # Iterate over rows and check for out-of-domain properties\n",
    "    for idx, row in df_numeric.iterrows():\n",
    "        row_mean = row.mean()\n",
    "        if row_mean > threshold:\n",
    "            out_of_domain_indices.append(idx)\n",
    "            #print(f\"Row {idx} has out-of-domain properties.\") \n",
    "    \n",
    "    df_reduced = df.drop(out_of_domain_indices)\n",
    "    print('Number of rows with out-of-domain properties: ',len(out_of_domain_indices))\n",
    "    print('\\nSample count before: ', len(df.index))\n",
    "    print('Sample count after: ', len(df_reduced.index))\n",
    "\n",
    "    return df_reduced\n",
    "\n",
    "def numeric_only(df):\n",
    "    \"\"\"\n",
    "    Create a copy of the DataFrame with only numeric columns.\n",
    "    \"\"\"\n",
    "    # Select numeric columns\n",
    "    numeric_columns = df.select_dtypes(include='number')\n",
    "    \n",
    "    # Create a copy with only numeric columns\n",
    "    df_numeric = numeric_columns.copy()\n",
    "    \n",
    "    return df_numeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f2c909ad-5b99-42be-a2e8-3e5d57a0ccd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Duplicate row removal:\n",
      "Sample count before:  19579\n",
      "Sample count after:  19330\n",
      "\n",
      "Null row removal:\n",
      "Sample count before:  19330\n",
      "Sample count after:  10762\n",
      "\n",
      "Out-of-domain row removal:\n",
      "Threshold = 2  standard deviations\n",
      "Number of rows with out-of-domain properties:  20\n",
      "\n",
      "Sample count before:  10762\n",
      "Sample count after:  10742\n"
     ]
    }
   ],
   "source": [
    "train_clean = drop_out_of_domain(drop_null(drop_duplicates(train_df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "da7b0530-57d1-4ec0-a865-72b67c206bfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PKD1</th>\n",
       "      <th>PKD2</th>\n",
       "      <th>PKD3</th>\n",
       "      <th>PKD4</th>\n",
       "      <th>PKD5</th>\n",
       "      <th>PKD6</th>\n",
       "      <th>PKD7</th>\n",
       "      <th>PKD8</th>\n",
       "      <th>control1</th>\n",
       "      <th>control2</th>\n",
       "      <th>control3</th>\n",
       "      <th>control4</th>\n",
       "      <th>control5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4682</th>\n",
       "      <td>0.058530</td>\n",
       "      <td>0.115151</td>\n",
       "      <td>0.049917</td>\n",
       "      <td>0.031766</td>\n",
       "      <td>0.099263</td>\n",
       "      <td>0.148055</td>\n",
       "      <td>0.060874</td>\n",
       "      <td>0.079229</td>\n",
       "      <td>0.181459</td>\n",
       "      <td>0.206016</td>\n",
       "      <td>0.137439</td>\n",
       "      <td>0.223114</td>\n",
       "      <td>0.089336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18199</th>\n",
       "      <td>0.018895</td>\n",
       "      <td>0.006694</td>\n",
       "      <td>0.022041</td>\n",
       "      <td>0.005042</td>\n",
       "      <td>0.003015</td>\n",
       "      <td>0.039180</td>\n",
       "      <td>0.015758</td>\n",
       "      <td>0.089860</td>\n",
       "      <td>0.002654</td>\n",
       "      <td>0.004407</td>\n",
       "      <td>0.002012</td>\n",
       "      <td>0.002453</td>\n",
       "      <td>0.001323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20389</th>\n",
       "      <td>0.027754</td>\n",
       "      <td>0.066401</td>\n",
       "      <td>0.034147</td>\n",
       "      <td>0.026015</td>\n",
       "      <td>0.059349</td>\n",
       "      <td>0.091447</td>\n",
       "      <td>0.043421</td>\n",
       "      <td>0.053518</td>\n",
       "      <td>0.108313</td>\n",
       "      <td>0.135050</td>\n",
       "      <td>0.097840</td>\n",
       "      <td>0.150244</td>\n",
       "      <td>0.060836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21797</th>\n",
       "      <td>0.132074</td>\n",
       "      <td>0.240895</td>\n",
       "      <td>0.186061</td>\n",
       "      <td>0.102726</td>\n",
       "      <td>0.282600</td>\n",
       "      <td>0.370735</td>\n",
       "      <td>0.207769</td>\n",
       "      <td>0.347986</td>\n",
       "      <td>0.306554</td>\n",
       "      <td>0.328283</td>\n",
       "      <td>0.252108</td>\n",
       "      <td>0.341737</td>\n",
       "      <td>0.159480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2087</th>\n",
       "      <td>0.001342</td>\n",
       "      <td>0.005579</td>\n",
       "      <td>0.005105</td>\n",
       "      <td>0.001497</td>\n",
       "      <td>0.003724</td>\n",
       "      <td>0.001917</td>\n",
       "      <td>0.003834</td>\n",
       "      <td>0.002363</td>\n",
       "      <td>0.002789</td>\n",
       "      <td>0.004038</td>\n",
       "      <td>0.002666</td>\n",
       "      <td>0.003522</td>\n",
       "      <td>0.001846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8680</th>\n",
       "      <td>0.022339</td>\n",
       "      <td>0.151898</td>\n",
       "      <td>0.055156</td>\n",
       "      <td>0.033048</td>\n",
       "      <td>0.023833</td>\n",
       "      <td>0.041858</td>\n",
       "      <td>0.176863</td>\n",
       "      <td>0.023139</td>\n",
       "      <td>0.034133</td>\n",
       "      <td>0.123499</td>\n",
       "      <td>0.041477</td>\n",
       "      <td>0.197250</td>\n",
       "      <td>0.030462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3815</th>\n",
       "      <td>0.000868</td>\n",
       "      <td>0.014163</td>\n",
       "      <td>0.001706</td>\n",
       "      <td>0.010684</td>\n",
       "      <td>0.007468</td>\n",
       "      <td>0.003278</td>\n",
       "      <td>0.011897</td>\n",
       "      <td>0.007250</td>\n",
       "      <td>0.024857</td>\n",
       "      <td>0.016835</td>\n",
       "      <td>0.022873</td>\n",
       "      <td>0.014290</td>\n",
       "      <td>0.013376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9014</th>\n",
       "      <td>0.092936</td>\n",
       "      <td>0.277593</td>\n",
       "      <td>0.067824</td>\n",
       "      <td>0.096954</td>\n",
       "      <td>0.257435</td>\n",
       "      <td>0.239399</td>\n",
       "      <td>0.192993</td>\n",
       "      <td>0.362140</td>\n",
       "      <td>0.127688</td>\n",
       "      <td>0.165564</td>\n",
       "      <td>0.165822</td>\n",
       "      <td>0.177049</td>\n",
       "      <td>0.105570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16990</th>\n",
       "      <td>0.021699</td>\n",
       "      <td>0.070669</td>\n",
       "      <td>0.019897</td>\n",
       "      <td>0.016170</td>\n",
       "      <td>0.089503</td>\n",
       "      <td>0.083157</td>\n",
       "      <td>0.025825</td>\n",
       "      <td>0.063774</td>\n",
       "      <td>0.111850</td>\n",
       "      <td>0.144766</td>\n",
       "      <td>0.066968</td>\n",
       "      <td>0.145112</td>\n",
       "      <td>0.049720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14980</th>\n",
       "      <td>0.041159</td>\n",
       "      <td>0.048345</td>\n",
       "      <td>0.039836</td>\n",
       "      <td>0.021471</td>\n",
       "      <td>0.064940</td>\n",
       "      <td>0.032385</td>\n",
       "      <td>0.029096</td>\n",
       "      <td>0.030616</td>\n",
       "      <td>0.119178</td>\n",
       "      <td>0.161531</td>\n",
       "      <td>0.106662</td>\n",
       "      <td>0.160736</td>\n",
       "      <td>0.082489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10742 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PKD1      PKD2      PKD3      PKD4      PKD5      PKD6      PKD7  \\\n",
       "4682   0.058530  0.115151  0.049917  0.031766  0.099263  0.148055  0.060874   \n",
       "18199  0.018895  0.006694  0.022041  0.005042  0.003015  0.039180  0.015758   \n",
       "20389  0.027754  0.066401  0.034147  0.026015  0.059349  0.091447  0.043421   \n",
       "21797  0.132074  0.240895  0.186061  0.102726  0.282600  0.370735  0.207769   \n",
       "2087   0.001342  0.005579  0.005105  0.001497  0.003724  0.001917  0.003834   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "8680   0.022339  0.151898  0.055156  0.033048  0.023833  0.041858  0.176863   \n",
       "3815   0.000868  0.014163  0.001706  0.010684  0.007468  0.003278  0.011897   \n",
       "9014   0.092936  0.277593  0.067824  0.096954  0.257435  0.239399  0.192993   \n",
       "16990  0.021699  0.070669  0.019897  0.016170  0.089503  0.083157  0.025825   \n",
       "14980  0.041159  0.048345  0.039836  0.021471  0.064940  0.032385  0.029096   \n",
       "\n",
       "           PKD8  control1  control2  control3  control4  control5  \n",
       "4682   0.079229  0.181459  0.206016  0.137439  0.223114  0.089336  \n",
       "18199  0.089860  0.002654  0.004407  0.002012  0.002453  0.001323  \n",
       "20389  0.053518  0.108313  0.135050  0.097840  0.150244  0.060836  \n",
       "21797  0.347986  0.306554  0.328283  0.252108  0.341737  0.159480  \n",
       "2087   0.002363  0.002789  0.004038  0.002666  0.003522  0.001846  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "8680   0.023139  0.034133  0.123499  0.041477  0.197250  0.030462  \n",
       "3815   0.007250  0.024857  0.016835  0.022873  0.014290  0.013376  \n",
       "9014   0.362140  0.127688  0.165564  0.165822  0.177049  0.105570  \n",
       "16990  0.063774  0.111850  0.144766  0.066968  0.145112  0.049720  \n",
       "14980  0.030616  0.119178  0.161531  0.106662  0.160736  0.082489  \n",
       "\n",
       "[10742 rows x 13 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "374a8bc9-fc62-4d16-8046-2db252d7e11f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05a06b10-45b5-41a5-acdf-5dec3995c1dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e7b15b6-d542-4591-b437-6f9b67f8c1f1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c12ca77c-07e1-4814-aafc-7d165dec697b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transpose\n",
    "#transposed_df2 = mapped_df2.transpose(include_header=True)\n",
    "#transposed_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17fe2dfb-9ae7-4678-bb39-799be1de8dd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''def unique_group(df1, grouping_attribute, df2=None):\n",
    "\n",
    "    # create dictionary of unique instances of passed attribute in first dataframe\n",
    "    unique_instances = df1[grouping_attribute].unique() \n",
    "    grouped_columns = {grouping_attribute: [] for grouping_attribute in unique_instances} \n",
    "\n",
    "    # fill dictionary with second dataframe's data\n",
    "    if df2 is not None: \n",
    "        for col in df2.columns:\n",
    "            for grouping_attribute in unique_instances:\n",
    "                if grouping_attribute in col:\n",
    "                    grouped_columns[grouping_attribute].append(col)\n",
    "                    break\n",
    "    # fill dictionary with first dataframe's data\n",
    "    else:\n",
    "        for col in df1.columns:\n",
    "            for grouping_attribute in unique_instances:\n",
    "                if grouping_attribute in col:\n",
    "                    grouped_columns[grouping_attribute].append(col)\n",
    "                    break\n",
    "    return grouped_columns\n",
    "    \n",
    "def aggregate_by_group(df, grouping_attribute, grouped_columns):\n",
    "    aggregated_df = pl.DataFrame()\n",
    "    for grouping_attribute, cols in grouped_columns.items():\n",
    "        if cols:  # Check if any columns exist for that patient\n",
    "            aggregated_df[grouping_attribute] = df[cols].mean(axis=1)\n",
    "    return aggregated_df'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c22f346-708d-42e0-95c8-d02aff8f939e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# group columns by unique instances of attribute in df1 and fill dictionary with df2 data\n",
    "#grouped_columns = unique_group(df1, 'patient', df2=mapped_df2)\n",
    "#aggregated_df2 = aggregate_by_group(df2, 'patient', grouped_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f7d708a-9ec2-457d-8460-fa5f2c320470",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ae0f711-d4b3-4a2c-a81c-ee0edd68bb40",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "60743dd7-2368-47d2-8a9a-7b60664d50f4",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02eec872-e923-48c6-b1e4-7d1e959dbffe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e7f8532-1a97-4499-b519-19eb2d07c176",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b8697ee-8089-42d5-bf06-a7e140b314f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a7d15e6-240a-4d16-abcb-78b8036b02e6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "048d77a2-fcf2-431a-8cd5-986d3010c983",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c82ced75-91c5-418f-8cfb-f9989c0233f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''# File 1: Read URL of the compressed CSV \n",
    "df1 = mu.read_url(url)\n",
    "mu.save_to_csv(df1, input_dir, \"demographic_clinical_data.csv\")\n",
    "matrix1 = df1.values\n",
    "\n",
    "# File 2: Read rds from filepath in 'data' directory\n",
    "rds_path = os.path.join(input_dir, rds) # create path to download existing rds file\n",
    "matrix2, cell, gene = my_utils.read_rds(rds_path)\n",
    "df2 = pd.DataFrame(matrix2, columns=cell, index = gene)\n",
    "#my_utils.save_to_csv(df2, input_dir, \"raw_molecular_data.csv\") #save to 'data' directory\n",
    "'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}