Scoring_Code_model1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import islice\n",
    "import os\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "# !cat /proc/sys/vm/overcommit_memory\n",
    "# !echo 1 > /proc/sys/vm/overcommit_memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read Genomic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>Sample Name</th>\n",
       "      <th>TISSUE_FACTOR</th>\n",
       "      <th>MSI_FACTOR</th>\n",
       "      <th>ABCB1_mut</th>\n",
       "      <th>ABL2_mut</th>\n",
       "      <th>ACACA_mut</th>\n",
       "      <th>ACVR1B_mut</th>\n",
       "      <th>ACVR2A_mut</th>\n",
       "      <th>AFF4_mut</th>\n",
       "      <th>...</th>\n",
       "      <th>BAZ2B_mut</th>\n",
       "      <th>BCLAF1_mut</th>\n",
       "      <th>BCOR_mut</th>\n",
       "      <th>BCR-ABL_mut</th>\n",
       "      <th>BLM_mut</th>\n",
       "      <th>BMPR2_mut</th>\n",
       "      <th>BNC2_mut</th>\n",
       "      <th>BPTF_mut</th>\n",
       "      <th>BRAF_mut</th>\n",
       "      <th>TP53_mut</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1287381</td>\n",
       "      <td>201T</td>\n",
       "      <td>lung_NSCLC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>924100</td>\n",
       "      <td>22RV1</td>\n",
       "      <td>prostate</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   COSMIC_ID Sample Name TISSUE_FACTOR  MSI_FACTOR  ABCB1_mut  ABL2_mut  \\\n",
       "0    1287381        201T    lung_NSCLC           0          0         0   \n",
       "1     924100       22RV1      prostate           1          0         0   \n",
       "\n",
       "   ACACA_mut  ACVR1B_mut  ACVR2A_mut  AFF4_mut  ...  BAZ2B_mut  BCLAF1_mut  \\\n",
       "0          0           0           0         0  ...          0           0   \n",
       "1          0           0           0         0  ...          0           0   \n",
       "\n",
       "   BCOR_mut  BCR-ABL_mut  BLM_mut  BMPR2_mut  BNC2_mut  BPTF_mut  BRAF_mut  \\\n",
       "0         0            0        0          0         0         0         0   \n",
       "1         0            0        0          0         0         0         0   \n",
       "\n",
       "   TP53_mut  \n",
       "0         1  \n",
       "1         1  \n",
       "\n",
       "[2 rows x 51 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen_data=pd.read_csv(\"./data/genomic_features.tsv\",sep=\"\\t\",nrows=10)\n",
    "\n",
    "#Optimize datatypes to save memory\n",
    "gene_dict=gen_data.dtypes.apply(lambda x: x.name).to_dict()\n",
    "new_gene_dict=gene_dict.copy()\n",
    "vlist=['COSMIC_ID','Sample Name','TISSUE_FACTOR']\n",
    "i=1\n",
    "for k in new_gene_dict.keys():\n",
    "        if k in vlist:\n",
    "            i=0\n",
    "        else:\n",
    "            new_gene_dict[k]='int16'\n",
    "#         print(k,new_wes_dict[k])\n",
    "\n",
    "genomic_features = pd.read_csv(\"./data/genomic_features.tsv\",sep = \"\\t\",dtype=new_gene_dict)\n",
    "\n",
    "gen_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cell line details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sample Name</th>\n",
       "      <th>COSMIC identifier</th>\n",
       "      <th>Whole Exome Sequencing (WES)</th>\n",
       "      <th>Copy Number Alterations (CNA)</th>\n",
       "      <th>Gene Expression</th>\n",
       "      <th>Methylation</th>\n",
       "      <th>Drug\\nResponse</th>\n",
       "      <th>GDSC\\nTissue descriptor 1</th>\n",
       "      <th>GDSC\\nTissue\\ndescriptor 2</th>\n",
       "      <th>Cancer Type\\n(matching TCGA label)</th>\n",
       "      <th>Microsatellite \\ninstability Status (MSI)</th>\n",
       "      <th>Screen Medium</th>\n",
       "      <th>Growth Properties</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>A253</td>\n",
       "      <td>906794.0</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>aero_dig_tract</td>\n",
       "      <td>head and neck</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MSS/MSI-L</td>\n",
       "      <td>D/F12</td>\n",
       "      <td>Adherent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>BB30-HNC</td>\n",
       "      <td>753531.0</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>aero_dig_tract</td>\n",
       "      <td>head and neck</td>\n",
       "      <td>HNSC</td>\n",
       "      <td>MSS/MSI-L</td>\n",
       "      <td>D/F12</td>\n",
       "      <td>Adherent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Sample Name  COSMIC identifier Whole Exome Sequencing (WES)  \\\n",
       "0        A253           906794.0                            Y   \n",
       "1    BB30-HNC           753531.0                            Y   \n",
       "\n",
       "  Copy Number Alterations (CNA) Gene Expression Methylation Drug\\nResponse  \\\n",
       "0                             Y               Y           Y              Y   \n",
       "1                             Y               Y           Y              Y   \n",
       "\n",
       "  GDSC\\nTissue descriptor 1 GDSC\\nTissue\\ndescriptor 2  \\\n",
       "0            aero_dig_tract              head and neck   \n",
       "1            aero_dig_tract              head and neck   \n",
       "\n",
       "  Cancer Type\\n(matching TCGA label)  \\\n",
       "0                                NaN   \n",
       "1                               HNSC   \n",
       "\n",
       "  Microsatellite \\ninstability Status (MSI) Screen Medium Growth Properties  \n",
       "0                                 MSS/MSI-L         D/F12          Adherent  \n",
       "1                                 MSS/MSI-L         D/F12          Adherent  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "celld=pd.read_csv('./data/Cell_Lines_Details.csv')\n",
    "celld.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>DRUG_NAME</th>\n",
       "      <th>SYNONYMS</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>TARGET_PATHWAY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Erlotinib</td>\n",
       "      <td>Tarceva, RG-1415, CP-358774, OSI-774, Ro-50823...</td>\n",
       "      <td>EGFR</td>\n",
       "      <td>EGFR signaling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Rapamycin</td>\n",
       "      <td>AY-22989, Sirolimus, WY-090217, Torisel, Rapamune</td>\n",
       "      <td>MTORC1</td>\n",
       "      <td>PI3K/MTOR signaling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>Sunitinib</td>\n",
       "      <td>Sutent, Sunitinib Malate, SU-11248</td>\n",
       "      <td>PDGFR, KIT, VEGFR, FLT3, RET, CSF1R</td>\n",
       "      <td>RTK signaling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>PHA-665752</td>\n",
       "      <td>PHA665752, PHA 665752</td>\n",
       "      <td>MET</td>\n",
       "      <td>RTK signaling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>MG-132</td>\n",
       "      <td>LLL cpd, MG 132, MG132</td>\n",
       "      <td>Proteasome, CAPN1</td>\n",
       "      <td>Protein stability and degradation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   DRUG_ID   DRUG_NAME                                           SYNONYMS  \\\n",
       "0        1   Erlotinib  Tarceva, RG-1415, CP-358774, OSI-774, Ro-50823...   \n",
       "1        3   Rapamycin  AY-22989, Sirolimus, WY-090217, Torisel, Rapamune   \n",
       "2        5   Sunitinib                 Sutent, Sunitinib Malate, SU-11248   \n",
       "3        6  PHA-665752                              PHA665752, PHA 665752   \n",
       "4        9      MG-132                             LLL cpd, MG 132, MG132   \n",
       "\n",
       "                                TARGET                     TARGET_PATHWAY  \n",
       "0                                 EGFR                     EGFR signaling  \n",
       "1                               MTORC1                PI3K/MTOR signaling  \n",
       "2  PDGFR, KIT, VEGFR, FLT3, RET, CSF1R                      RTK signaling  \n",
       "3                                  MET                      RTK signaling  \n",
       "4                    Proteasome, CAPN1  Protein stability and degradation  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in drug data\n",
    "drug_d=pd.read_csv(\"./data/Screened_Compounds.csv\",sep=',')\n",
    "drug_d.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATASET_VERSION</th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>CELL_LINE_NAME</th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>DRUG_NAME_x</th>\n",
       "      <th>PUTATIVE_TARGET</th>\n",
       "      <th>MAX_CONC_MICROMOLAR</th>\n",
       "      <th>MIN_CONC_MICROMOLAR</th>\n",
       "      <th>LN_IC50</th>\n",
       "      <th>AUC</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>Z_SCORE</th>\n",
       "      <th>BINARY_RESPONSE</th>\n",
       "      <th>DRUG_NAME_y</th>\n",
       "      <th>SYNONYMS</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>TARGET_PATHWAY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>17.3</td>\n",
       "      <td>683665</td>\n",
       "      <td>MC-CAR</td>\n",
       "      <td>1</td>\n",
       "      <td>Erlotinib</td>\n",
       "      <td>EGFR</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.007812</td>\n",
       "      <td>2.453524</td>\n",
       "      <td>0.982610</td>\n",
       "      <td>0.021678</td>\n",
       "      <td>-0.015505</td>\n",
       "      <td>R</td>\n",
       "      <td>Erlotinib</td>\n",
       "      <td>Tarceva, RG-1415, CP-358774, OSI-774, Ro-50823...</td>\n",
       "      <td>EGFR</td>\n",
       "      <td>EGFR signaling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>17.3</td>\n",
       "      <td>684055</td>\n",
       "      <td>ES3</td>\n",
       "      <td>1</td>\n",
       "      <td>Erlotinib</td>\n",
       "      <td>EGFR</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.007812</td>\n",
       "      <td>3.376592</td>\n",
       "      <td>0.985169</td>\n",
       "      <td>0.029915</td>\n",
       "      <td>0.779999</td>\n",
       "      <td>R</td>\n",
       "      <td>Erlotinib</td>\n",
       "      <td>Tarceva, RG-1415, CP-358774, OSI-774, Ro-50823...</td>\n",
       "      <td>EGFR</td>\n",
       "      <td>EGFR signaling</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   DATASET_VERSION  COSMIC_ID CELL_LINE_NAME  DRUG_ID DRUG_NAME_x  \\\n",
       "0             17.3     683665         MC-CAR        1   Erlotinib   \n",
       "1             17.3     684055            ES3        1   Erlotinib   \n",
       "\n",
       "  PUTATIVE_TARGET  MAX_CONC_MICROMOLAR  MIN_CONC_MICROMOLAR   LN_IC50  \\\n",
       "0            EGFR                  2.0             0.007812  2.453524   \n",
       "1            EGFR                  2.0             0.007812  3.376592   \n",
       "\n",
       "        AUC      RMSE   Z_SCORE BINARY_RESPONSE DRUG_NAME_y  \\\n",
       "0  0.982610  0.021678 -0.015505               R   Erlotinib   \n",
       "1  0.985169  0.029915  0.779999               R   Erlotinib   \n",
       "\n",
       "                                            SYNONYMS TARGET  TARGET_PATHWAY  \n",
       "0  Tarceva, RG-1415, CP-358774, OSI-774, Ro-50823...   EGFR  EGFR signaling  \n",
       "1  Tarceva, RG-1415, CP-358774, OSI-774, Ro-50823...   EGFR  EGFR signaling  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in drug response data\n",
    "dose_response_df_all = pd.read_csv('./data/v17.3_fitted_dose_response_binary.csv', index_col='IC50_RESULTS_ID')\n",
    "dose_response_df=pd.merge(dose_response_df_all,drug_d,how='left',on='DRUG_ID')\n",
    "dose_response_df['TARGET_PATHWAY']=dose_response_df['TARGET_PATHWAY'].fillna(\"UNK\")\n",
    "dose_response_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>LN_IC50</th>\n",
       "      <th>BINARY_RESPONSE</th>\n",
       "      <th>TCGA_Label</th>\n",
       "      <th>Cancer_Type</th>\n",
       "      <th>TARGET_PATHWAY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>683665</td>\n",
       "      <td>1</td>\n",
       "      <td>2.453524</td>\n",
       "      <td>R</td>\n",
       "      <td>MM</td>\n",
       "      <td>Multiple Myeloma</td>\n",
       "      <td>EGFR signaling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>684055</td>\n",
       "      <td>1</td>\n",
       "      <td>3.376592</td>\n",
       "      <td>R</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "      <td>EGFR signaling</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   COSMIC_ID  DRUG_ID   LN_IC50 BINARY_RESPONSE TCGA_Label       Cancer_Type  \\\n",
       "0     683665        1  2.453524               R         MM  Multiple Myeloma   \n",
       "1     684055        1  3.376592               R        NaN               UNK   \n",
       "\n",
       "   TARGET_PATHWAY  \n",
       "0  EGFR signaling  \n",
       "1  EGFR signaling  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Read in cancer labels, clean data\n",
    "tcga_labels=pd.read_csv(\"./data/TCGA_Labels.csv\")\n",
    "celld=pd.read_csv('./data/Cell_Lines_Details.csv')\n",
    "celld=celld[['COSMIC identifier','GDSC\\nTissue descriptor 1','GDSC\\nTissue\\ndescriptor 2','Cancer Type\\n(matching TCGA label)']]\n",
    "celld_desc=pd.merge(celld,tcga_labels,how='outer',left_on='Cancer Type\\n(matching TCGA label)',right_on=\"Sigle\",indicator=True)\n",
    "celld_desc['Program']=celld_desc['Program'].fillna('UNK')\n",
    "celld_desc=celld_desc.rename(columns={'COSMIC identifier':'COSMIC_ID','GDSC\\nTissue descriptor 1':'GDSC_DESC1','GDSC\\nTissue\\ndescriptor 2':'GDSC_DESC2','Cancer Type\\n(matching TCGA label)':'TCGA_Label', 'Program':'Cancer_Type'                                     })\n",
    "dose_resp_caname=pd.merge(dose_response_df,celld_desc,how='left',on='COSMIC_ID')\n",
    "dose_response_trimmed_df = dose_resp_caname[['COSMIC_ID', 'DRUG_ID', 'LN_IC50', 'BINARY_RESPONSE','TCGA_Label','Cancer_Type','TARGET_PATHWAY']]\n",
    "dose_response_trimmed_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WES prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>101060321</th>\n",
       "      <th>101927546</th>\n",
       "      <th>101927722</th>\n",
       "      <th>101928638</th>\n",
       "      <th>102724473</th>\n",
       "      <th>102724928</th>\n",
       "      <th>105375355</th>\n",
       "      <th>105378803</th>\n",
       "      <th>107403068</th>\n",
       "      <th>109731405</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>907268</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>907269</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>907270</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>907271</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>907272</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 18382 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0    1    2    9   10   12   13   14   15   16  ...  101060321  \\\n",
       "0      907268  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...        0.0   \n",
       "1      907269  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...        0.0   \n",
       "2      907270  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...        0.0   \n",
       "3      907271  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...        0.0   \n",
       "4      907272  1.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  ...        0.0   \n",
       "\n",
       "   101927546  101927722  101928638  102724473  102724928  105375355  \\\n",
       "0        0.0        0.0        0.0        0.0        0.0        0.0   \n",
       "1        0.0        0.0        0.0        0.0        0.0        0.0   \n",
       "2        0.0        0.0        0.0        0.0        0.0        0.0   \n",
       "3        0.0        0.0        0.0        0.0        0.0        0.0   \n",
       "4        0.0        0.0        0.0        0.0        0.0        0.0   \n",
       "\n",
       "   105378803  107403068  109731405  \n",
       "0        0.0        0.0        0.0  \n",
       "1        0.0        0.0        0.0  \n",
       "2        0.0        0.0        0.0  \n",
       "3        0.0        0.0        0.0  \n",
       "4        0.0        0.0        0.0  \n",
       "\n",
       "[5 rows x 18382 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read Wes data first few rows\n",
    "wes_scored_temp = pd.read_csv(\"./data/data2/wes_scored_transposed.tsv\",sep = \"\\t\",nrows=10)\n",
    "wes_scored_temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read full WES data with right data types\n",
    "wes_dict=wes_scored_temp.dtypes.apply(lambda x: x.name).to_dict()\n",
    "new_wes_dict=wes_dict.copy()\n",
    "i=1\n",
    "for k in new_wes_dict.keys():\n",
    "        if k=='Unnamed: 0':\n",
    "            i=0\n",
    "        else:\n",
    "            new_wes_dict[k]='int16'\n",
    "#         print(k,new_wes_dict[k])\n",
    "\n",
    "wes_scored = pd.read_csv(\"./data/data2/wes_scored_transposed.tsv\",sep = \"\\t\",dtype=new_wes_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "wes_scored=wes_scored.rename(columns={\"Unnamed: 0\":\"COSMIC_ID\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1096, 23)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Subset one pathway RTK signaling and lung cancer - THIS MAY NEED TO BE PARAMETERS\n",
    "rtk=dose_resp_caname[(dose_resp_caname['TARGET_PATHWAY']=='RTK signaling')  & (dose_resp_caname['GDSC_DESC1']=='lung_SCLC')]\n",
    "# rtk['Cancer_Type'].value_counts()\n",
    "rtk.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1096, 18404)\n",
      "(1096, 23)\n"
     ]
    }
   ],
   "source": [
    "rtk_wes_joined_df = pd.merge(wes_scored, rtk,how='inner', on='COSMIC_ID')\n",
    "print(rtk_wes_joined_df.shape)\n",
    "print(rtk.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "rtk_wes_joined_df['COSMIC_DRUG_ID']=rtk_wes_joined_df['COSMIC_ID'].map(str)+\"_\"+rtk_wes_joined_df['DRUG_ID'].map(str)\n",
    "wes_subset=rtk_wes_joined_df[['COSMIC_DRUG_ID','347733','440560','4633','2019','5706','192683','1063','2220','4829','5137','7767','7982','9994','23033','51585','55051','57509','65083','84695','119395','148137','387266']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>107133486</th>\n",
       "      <th>107133502</th>\n",
       "      <th>107133524</th>\n",
       "      <th>107161145</th>\n",
       "      <th>107985535</th>\n",
       "      <th>107986809</th>\n",
       "      <th>107987337</th>\n",
       "      <th>107987341</th>\n",
       "      <th>109731405</th>\n",
       "      <th>112441434</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.584963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240122</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>0.584963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240123</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.321928</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.321928</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.415037</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.321928</td>\n",
       "      <td>-0.415037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1240124</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>-4.32</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1240125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.415037</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.415037</td>\n",
       "      <td>-2.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.321928</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.321928</td>\n",
       "      <td>-2.00</td>\n",
       "      <td>-2.00</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.415037</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24428 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   COSMIC_ID    1         2    9   10        12        13   14   15        16  \\\n",
       "0    1240121  0.0  0.000000 -1.0 -1.0  0.000000  1.584963  0.0  0.0  0.000000   \n",
       "1    1240122  0.0  0.584963  0.0  0.0  0.000000  1.000000  0.0  0.0  0.584963   \n",
       "2    1240123  0.0  0.000000  0.0  0.0  0.000000  0.321928  0.0  0.0  0.321928   \n",
       "3    1240124  0.0  0.584963  0.0  0.0  0.000000  0.000000  0.0  0.0  0.000000   \n",
       "4    1240125  0.0  0.000000  0.0  0.0 -0.415037  0.000000  0.0  0.0 -1.000000   \n",
       "\n",
       "   ...  107133486  107133502  107133524  107161145  107985535  107986809  \\\n",
       "0  ...   0.000000      -4.32        0.0   0.000000        0.0   0.000000   \n",
       "1  ...   0.584963       0.00        0.0   0.584963        0.0   0.584963   \n",
       "2  ...  -0.415037      -4.32        0.0   0.000000        0.0   0.000000   \n",
       "3  ...   0.000000      -4.32        0.0   0.000000        0.0   0.000000   \n",
       "4  ...  -0.415037      -2.00        0.0   0.321928        0.0   0.321928   \n",
       "\n",
       "   107987337  107987341  109731405  112441434  \n",
       "0      -4.32      -4.32   0.000000   0.000000  \n",
       "1      -4.32      -4.32   0.584963   0.584963  \n",
       "2      -4.32      -4.32   0.321928  -0.415037  \n",
       "3      -4.32      -4.32   0.000000   0.000000  \n",
       "4      -2.00      -2.00  -1.000000  -0.415037  \n",
       "\n",
       "[5 rows x 24428 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cna_scored_temp = pd.read_csv(\"./data/data2/cna_scored_transposed.tsv\",sep = \"\\t\", nrows=10)\n",
    "cna_scored_temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "cna_dict=cna_scored_temp.dtypes.apply(lambda x: x.name).to_dict()\n",
    "new_cna_dict=cna_dict.copy()\n",
    "i=1\n",
    "for k in new_cna_dict.keys():\n",
    "        if k=='COSMIC_ID':\n",
    "            i=0\n",
    "        else:\n",
    "            new_cna_dict[k]='float16'\n",
    "#         print(k,new_cna_dict[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>...</th>\n",
       "      <th>107133486</th>\n",
       "      <th>107133502</th>\n",
       "      <th>107133524</th>\n",
       "      <th>107161145</th>\n",
       "      <th>107985535</th>\n",
       "      <th>107986809</th>\n",
       "      <th>107987337</th>\n",
       "      <th>107987341</th>\n",
       "      <th>109731405</th>\n",
       "      <th>112441434</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240121</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.584961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240122</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>...</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>0.584961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240123</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.322021</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.322021</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.415039</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.322021</td>\n",
       "      <td>-0.415039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1240124</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1240125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.415039</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.415039</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.322021</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.322021</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.415039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24428 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   COSMIC_ID    1         2    9   10        12        13   14   15        16  \\\n",
       "0    1240121  0.0  0.000000 -1.0 -1.0  0.000000  1.584961  0.0  0.0  0.000000   \n",
       "1    1240122  0.0  0.584961  0.0  0.0  0.000000  1.000000  0.0  0.0  0.584961   \n",
       "2    1240123  0.0  0.000000  0.0  0.0  0.000000  0.322021  0.0  0.0  0.322021   \n",
       "3    1240124  0.0  0.584961  0.0  0.0  0.000000  0.000000  0.0  0.0  0.000000   \n",
       "4    1240125  0.0  0.000000  0.0  0.0 -0.415039  0.000000  0.0  0.0 -1.000000   \n",
       "\n",
       "   ...  107133486  107133502  107133524  107161145  107985535  107986809  \\\n",
       "0  ...   0.000000  -4.320312        0.0   0.000000        0.0   0.000000   \n",
       "1  ...   0.584961   0.000000        0.0   0.584961        0.0   0.584961   \n",
       "2  ...  -0.415039  -4.320312        0.0   0.000000        0.0   0.000000   \n",
       "3  ...   0.000000  -4.320312        0.0   0.000000        0.0   0.000000   \n",
       "4  ...  -0.415039  -2.000000        0.0   0.322021        0.0   0.322021   \n",
       "\n",
       "   107987337  107987341  109731405  112441434  \n",
       "0  -4.320312  -4.320312   0.000000   0.000000  \n",
       "1  -4.320312  -4.320312   0.584961   0.584961  \n",
       "2  -4.320312  -4.320312   0.322021  -0.415039  \n",
       "3  -4.320312  -4.320312   0.000000   0.000000  \n",
       "4  -2.000000  -2.000000  -1.000000  -0.415039  \n",
       "\n",
       "[5 rows x 24428 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cna_scored = pd.read_csv(\"./data/data2/cna_scored_transposed.tsv\",sep = \"\\t\", dtype=new_cna_dict)\n",
    "cna_scored.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "rtk=rtk.drop(['_merge'],axis=1)\n",
    "rtk_cna_joined_df = pd.merge(cna_scored, rtk, how='inner', on='COSMIC_ID',indicator=True)\n",
    "rtk_cna_joined_df['COSMIC_DRUG_ID']=rtk_cna_joined_df['COSMIC_ID'].map(str)+\"_\"+rtk_cna_joined_df['DRUG_ID'].map(str)\n",
    "\n",
    "#Make a copy with original labels and other fields\n",
    "rtk_cna_joined_df_copy_with_label=rtk_cna_joined_df.copy()\n",
    "\n",
    "rtk_cna_joined_df=rtk_cna_joined_df.drop({\"LN_IC50\",\"COSMIC_ID\",\"CELL_LINE_NAME\",\"DRUG_ID\",\"_merge\",\"Sigle\",\"GDSC_DESC1\",\"GDSC_DESC2\",\"TCGA_Label\",\"Cancer_Type\",'MAX_CONC_MICROMOLAR','MIN_CONC_MICROMOLAR','AUC','RMSE','Z_SCORE','DATASET_VERSION','PUTATIVE_TARGET','DRUG_NAME_x','DRUG_NAME_y','SYNONYMS','TARGET','TARGET_PATHWAY','BINARY_RESPONSE'},axis=1)\n",
    "rtk_cna_joined_df=rtk_cna_joined_df.set_index(\"COSMIC_DRUG_ID\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>18</th>\n",
       "      <th>...</th>\n",
       "      <th>107133486</th>\n",
       "      <th>107133502</th>\n",
       "      <th>107133524</th>\n",
       "      <th>107161145</th>\n",
       "      <th>107985535</th>\n",
       "      <th>107986809</th>\n",
       "      <th>107987337</th>\n",
       "      <th>107987341</th>\n",
       "      <th>109731405</th>\n",
       "      <th>112441434</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1240182_5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>-0.584961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.584961</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>-0.584961</td>\n",
       "      <td>-0.584961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1240182_6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>-0.584961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>0.415039</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.584961</td>\n",
       "      <td>-4.320312</td>\n",
       "      <td>-0.584961</td>\n",
       "      <td>-0.584961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 24427 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  1         2         9        10        12        13   14  \\\n",
       "COSMIC_DRUG_ID                                                               \n",
       "1240182_5       0.0  0.415039  0.415039  0.415039  0.415039  0.415039  0.0   \n",
       "1240182_6       0.0  0.415039  0.415039  0.415039  0.415039  0.415039  0.0   \n",
       "\n",
       "                      15        16   18  ...  107133486  107133502  107133524  \\\n",
       "COSMIC_DRUG_ID                           ...                                    \n",
       "1240182_5       0.415039 -0.584961  0.0  ...   0.415039  -4.320312   0.415039   \n",
       "1240182_6       0.415039 -0.584961  0.0  ...   0.415039  -4.320312   0.415039   \n",
       "\n",
       "                107161145  107985535  107986809  107987337  107987341  \\\n",
       "COSMIC_DRUG_ID                                                          \n",
       "1240182_5             0.0        0.0        0.0  -1.584961  -4.320312   \n",
       "1240182_6             0.0        0.0        0.0  -1.584961  -4.320312   \n",
       "\n",
       "                109731405  112441434  \n",
       "COSMIC_DRUG_ID                        \n",
       "1240182_5       -0.584961  -0.584961  \n",
       "1240182_6       -0.584961  -0.584961  \n",
       "\n",
       "[2 rows x 24427 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rtk_cna_joined_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th>cna_princ1</th>\n",
       "      <th>cna_princ2</th>\n",
       "      <th>cna_princ3</th>\n",
       "      <th>cna_princ4</th>\n",
       "      <th>cna_princ5</th>\n",
       "      <th>cna_princ6</th>\n",
       "      <th>cna_princ7</th>\n",
       "      <th>cna_princ8</th>\n",
       "      <th>cna_princ9</th>\n",
       "      <th>...</th>\n",
       "      <th>cna_princ31</th>\n",
       "      <th>cna_princ32</th>\n",
       "      <th>cna_princ33</th>\n",
       "      <th>cna_princ34</th>\n",
       "      <th>cna_princ35</th>\n",
       "      <th>cna_princ36</th>\n",
       "      <th>cna_princ37</th>\n",
       "      <th>cna_princ38</th>\n",
       "      <th>cna_princ39</th>\n",
       "      <th>cna_princ40</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240182_5</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.581619</td>\n",
       "      <td>-3.781948</td>\n",
       "      <td>0.471001</td>\n",
       "      <td>0.031296</td>\n",
       "      <td>2.542634</td>\n",
       "      <td>0.184568</td>\n",
       "      <td>-4.506054</td>\n",
       "      <td>-0.517173</td>\n",
       "      <td>0.496336</td>\n",
       "      <td>-1.216798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240182_6</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.581619</td>\n",
       "      <td>-3.781948</td>\n",
       "      <td>0.471001</td>\n",
       "      <td>0.031296</td>\n",
       "      <td>2.542634</td>\n",
       "      <td>0.184568</td>\n",
       "      <td>-4.506054</td>\n",
       "      <td>-0.517173</td>\n",
       "      <td>0.496336</td>\n",
       "      <td>-1.216798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240182_30</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.581619</td>\n",
       "      <td>-3.781948</td>\n",
       "      <td>0.471001</td>\n",
       "      <td>0.031296</td>\n",
       "      <td>2.542634</td>\n",
       "      <td>0.184568</td>\n",
       "      <td>-4.506054</td>\n",
       "      <td>-0.517173</td>\n",
       "      <td>0.496336</td>\n",
       "      <td>-1.216798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1240182_34</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.581619</td>\n",
       "      <td>-3.781948</td>\n",
       "      <td>0.471001</td>\n",
       "      <td>0.031296</td>\n",
       "      <td>2.542634</td>\n",
       "      <td>0.184568</td>\n",
       "      <td>-4.506054</td>\n",
       "      <td>-0.517173</td>\n",
       "      <td>0.496336</td>\n",
       "      <td>-1.216798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1240182_35</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.581619</td>\n",
       "      <td>-3.781948</td>\n",
       "      <td>0.471001</td>\n",
       "      <td>0.031296</td>\n",
       "      <td>2.542634</td>\n",
       "      <td>0.184568</td>\n",
       "      <td>-4.506054</td>\n",
       "      <td>-0.517173</td>\n",
       "      <td>0.496336</td>\n",
       "      <td>-1.216798</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  COSMIC_DRUG_ID  cna_princ1  cna_princ2  cna_princ3  cna_princ4  cna_princ5  \\\n",
       "0      1240182_5  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "1      1240182_6  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "2     1240182_30  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "3     1240182_34  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "4     1240182_35  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "\n",
       "   cna_princ6  cna_princ7  cna_princ8  cna_princ9  ...  cna_princ31  \\\n",
       "0   -9.017959    7.364445    0.337049   14.199088  ...    -6.581619   \n",
       "1   -9.017959    7.364445    0.337049   14.199088  ...    -6.581619   \n",
       "2   -9.017959    7.364445    0.337049   14.199088  ...    -6.581619   \n",
       "3   -9.017959    7.364445    0.337049   14.199088  ...    -6.581619   \n",
       "4   -9.017959    7.364445    0.337049   14.199088  ...    -6.581619   \n",
       "\n",
       "   cna_princ32  cna_princ33  cna_princ34  cna_princ35  cna_princ36  \\\n",
       "0    -3.781948     0.471001     0.031296     2.542634     0.184568   \n",
       "1    -3.781948     0.471001     0.031296     2.542634     0.184568   \n",
       "2    -3.781948     0.471001     0.031296     2.542634     0.184568   \n",
       "3    -3.781948     0.471001     0.031296     2.542634     0.184568   \n",
       "4    -3.781948     0.471001     0.031296     2.542634     0.184568   \n",
       "\n",
       "   cna_princ37  cna_princ38  cna_princ39  cna_princ40  \n",
       "0    -4.506054    -0.517173     0.496336    -1.216798  \n",
       "1    -4.506054    -0.517173     0.496336    -1.216798  \n",
       "2    -4.506054    -0.517173     0.496336    -1.216798  \n",
       "3    -4.506054    -0.517173     0.496336    -1.216798  \n",
       "4    -4.506054    -0.517173     0.496336    -1.216798  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Apply PCA on CNA test\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "# Load from file\n",
    "joblib_file = \"model1_cna_pca.pkl\"\n",
    "joblib_pca = joblib.load(joblib_file)\n",
    "\n",
    "# Transform test file\n",
    "devcnapca=joblib_pca.transform(rtk_cna_joined_df)  \n",
    "\n",
    "i=1\n",
    "varlist=[]\n",
    "for i in range(1, 41):\n",
    "    var=\"cna_princ\"+str(i)\n",
    "    varlist.append(var)\n",
    "    \n",
    "cna_subset=pd.DataFrame(data=devcnapca,columns=varlist,index=rtk_cna_joined_df.index)\n",
    "cna_subset=cna_subset.reset_index()\n",
    "cna_subset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th>cna_princ1</th>\n",
       "      <th>cna_princ2</th>\n",
       "      <th>cna_princ3</th>\n",
       "      <th>cna_princ4</th>\n",
       "      <th>cna_princ5</th>\n",
       "      <th>cna_princ6</th>\n",
       "      <th>cna_princ7</th>\n",
       "      <th>cna_princ8</th>\n",
       "      <th>cna_princ9</th>\n",
       "      <th>...</th>\n",
       "      <th>51585</th>\n",
       "      <th>55051</th>\n",
       "      <th>57509</th>\n",
       "      <th>65083</th>\n",
       "      <th>84695</th>\n",
       "      <th>119395</th>\n",
       "      <th>148137</th>\n",
       "      <th>387266</th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>COSMIC_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240182_5</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1240182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240182_6</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1240182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240182_30</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1240182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1240182_34</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>1240182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1240182_35</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.61564</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>1240182</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 65 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  COSMIC_DRUG_ID  cna_princ1  cna_princ2  cna_princ3  cna_princ4  cna_princ5  \\\n",
       "0      1240182_5  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "1      1240182_6  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "2     1240182_30  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "3     1240182_34  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "4     1240182_35  -27.474993   25.294332  -21.021211    13.61564   11.126272   \n",
       "\n",
       "   cna_princ6  cna_princ7  cna_princ8  cna_princ9  ...  51585  55051  57509  \\\n",
       "0   -9.017959    7.364445    0.337049   14.199088  ...      0      0      0   \n",
       "1   -9.017959    7.364445    0.337049   14.199088  ...      0      0      0   \n",
       "2   -9.017959    7.364445    0.337049   14.199088  ...      0      0      0   \n",
       "3   -9.017959    7.364445    0.337049   14.199088  ...      0      0      0   \n",
       "4   -9.017959    7.364445    0.337049   14.199088  ...      0      0      0   \n",
       "\n",
       "   65083  84695  119395  148137  387266  DRUG_ID  COSMIC_ID  \n",
       "0      0      0       0       0       0        5    1240182  \n",
       "1      0      0       0       0       0        6    1240182  \n",
       "2      0      0       0       0       0       30    1240182  \n",
       "3      0      0       0       0       0       34    1240182  \n",
       "4      0      0       0       0       0       35    1240182  \n",
       "\n",
       "[5 rows x 65 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Join the WES and CNA, split COSMIC_DRUG_ID to COSMIC_ID and DRUG_ID\n",
    "cna_wes_joined=pd.merge(cna_subset,wes_subset,how='inner',on='COSMIC_DRUG_ID')\n",
    "\n",
    "cna_wes_joined[\"DRUG_ID\"]=(cna_wes_joined['COSMIC_DRUG_ID'].str.extract(pat = '([\"_\"].+)'))\n",
    "cna_wes_joined[\"DRUG_ID\"]=cna_wes_joined[\"DRUG_ID\"].str.replace('_','').astype(int)\n",
    "\n",
    "cna_wes_joined[\"COSMIC_ID\"]=(cna_wes_joined['COSMIC_DRUG_ID'].str.extract(pat = '(.+[\"_\"])'))\n",
    "cna_wes_joined[\"COSMIC_ID\"]=cna_wes_joined[\"COSMIC_ID\"].str.replace('_','').astype(int)\n",
    "\n",
    "cna_wes_joined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transform drug_id to indicator columns\n",
    "cna_wes_joined = pd.concat([cna_wes_joined, pd.get_dummies(cna_wes_joined['DRUG_ID'], prefix=\"drug_id_\" )],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>Num_Targets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   DRUG_ID  Num_Targets\n",
       "0        1            1\n",
       "1        3            1\n",
       "2        5            6"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Import number of targets with drug id\n",
    "drug_target=pd.read_csv(\"./data/drugs_num_target.csv\")\n",
    "drug_target.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>ABL</th>\n",
       "      <th>ABL(T315I)</th>\n",
       "      <th>ADCK4</th>\n",
       "      <th>AKT1</th>\n",
       "      <th>AKT2</th>\n",
       "      <th>AKT3</th>\n",
       "      <th>ALK</th>\n",
       "      <th>ALK4</th>\n",
       "      <th>ALK5</th>\n",
       "      <th>...</th>\n",
       "      <th>WEE1</th>\n",
       "      <th>XIAP</th>\n",
       "      <th>c-FGR</th>\n",
       "      <th>dsDNAbreakinduction</th>\n",
       "      <th>gamma-secretase</th>\n",
       "      <th>notdefined</th>\n",
       "      <th>others</th>\n",
       "      <th>p38</th>\n",
       "      <th>p38alpha</th>\n",
       "      <th>p38beta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 237 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   DRUG_ID  ABL  ABL(T315I)  ADCK4  AKT1  AKT2  AKT3  ALK  ALK4  ALK5  ...  \\\n",
       "0        1  0.0         0.0    0.0   0.0   0.0   0.0  0.0   0.0   0.0  ...   \n",
       "1        3  0.0         0.0    0.0   0.0   0.0   0.0  0.0   0.0   0.0  ...   \n",
       "2        5  0.0         0.0    0.0   0.0   0.0   0.0  0.0   0.0   0.0  ...   \n",
       "\n",
       "   WEE1  XIAP  c-FGR  dsDNAbreakinduction  gamma-secretase  notdefined  \\\n",
       "0   0.0   0.0    0.0                  0.0              0.0         0.0   \n",
       "1   0.0   0.0    0.0                  0.0              0.0         0.0   \n",
       "2   0.0   0.0    0.0                  0.0              0.0         0.0   \n",
       "\n",
       "   others  p38  p38alpha  p38beta  \n",
       "0     0.0  0.0       0.0      0.0  \n",
       "1     0.0  0.0       0.0      0.0  \n",
       "2     0.0  0.0       0.0      0.0  \n",
       "\n",
       "[3 rows x 237 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Import drugs with target indicator\n",
    "drug_with_target=pd.read_csv(\"./data/drugs_with_target.csv\")\n",
    "drug_with_target=drug_with_target.drop(['num_targets'],axis=1)\n",
    "drug_with_target.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th>cna_princ1</th>\n",
       "      <th>cna_princ2</th>\n",
       "      <th>cna_princ3</th>\n",
       "      <th>cna_princ4</th>\n",
       "      <th>cna_princ5</th>\n",
       "      <th>cna_princ6</th>\n",
       "      <th>cna_princ7</th>\n",
       "      <th>cna_princ8</th>\n",
       "      <th>cna_princ9</th>\n",
       "      <th>...</th>\n",
       "      <th>WEE1</th>\n",
       "      <th>XIAP</th>\n",
       "      <th>c-FGR</th>\n",
       "      <th>dsDNAbreakinduction</th>\n",
       "      <th>gamma-secretase</th>\n",
       "      <th>notdefined</th>\n",
       "      <th>others</th>\n",
       "      <th>p38</th>\n",
       "      <th>p38alpha</th>\n",
       "      <th>p38beta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240182_5</td>\n",
       "      <td>-27.474993</td>\n",
       "      <td>25.294332</td>\n",
       "      <td>-21.021211</td>\n",
       "      <td>13.615640</td>\n",
       "      <td>11.126272</td>\n",
       "      <td>-9.017959</td>\n",
       "      <td>7.364445</td>\n",
       "      <td>0.337049</td>\n",
       "      <td>14.199088</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240189_5</td>\n",
       "      <td>-49.592893</td>\n",
       "      <td>-21.417866</td>\n",
       "      <td>-2.153763</td>\n",
       "      <td>-12.290173</td>\n",
       "      <td>35.043548</td>\n",
       "      <td>-11.792283</td>\n",
       "      <td>39.207136</td>\n",
       "      <td>12.739830</td>\n",
       "      <td>-4.703019</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240193_5</td>\n",
       "      <td>8.211324</td>\n",
       "      <td>26.039560</td>\n",
       "      <td>5.100081</td>\n",
       "      <td>-10.903063</td>\n",
       "      <td>-9.877014</td>\n",
       "      <td>-5.397095</td>\n",
       "      <td>9.684463</td>\n",
       "      <td>2.665763</td>\n",
       "      <td>-5.264648</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1297439_5</td>\n",
       "      <td>51.929641</td>\n",
       "      <td>-13.293321</td>\n",
       "      <td>-15.197194</td>\n",
       "      <td>-5.165354</td>\n",
       "      <td>-3.819563</td>\n",
       "      <td>1.191913</td>\n",
       "      <td>-0.049136</td>\n",
       "      <td>0.591174</td>\n",
       "      <td>-4.021995</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1303901_5</td>\n",
       "      <td>-26.112060</td>\n",
       "      <td>-11.083376</td>\n",
       "      <td>5.386099</td>\n",
       "      <td>9.754193</td>\n",
       "      <td>0.372270</td>\n",
       "      <td>2.745538</td>\n",
       "      <td>-15.419270</td>\n",
       "      <td>9.987002</td>\n",
       "      <td>12.310934</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 371 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  COSMIC_DRUG_ID  cna_princ1  cna_princ2  cna_princ3  cna_princ4  cna_princ5  \\\n",
       "0      1240182_5  -27.474993   25.294332  -21.021211   13.615640   11.126272   \n",
       "1      1240189_5  -49.592893  -21.417866   -2.153763  -12.290173   35.043548   \n",
       "2      1240193_5    8.211324   26.039560    5.100081  -10.903063   -9.877014   \n",
       "3      1297439_5   51.929641  -13.293321  -15.197194   -5.165354   -3.819563   \n",
       "4      1303901_5  -26.112060  -11.083376    5.386099    9.754193    0.372270   \n",
       "\n",
       "   cna_princ6  cna_princ7  cna_princ8  cna_princ9  ...  WEE1  XIAP  c-FGR  \\\n",
       "0   -9.017959    7.364445    0.337049   14.199088  ...   0.0   0.0    0.0   \n",
       "1  -11.792283   39.207136   12.739830   -4.703019  ...   0.0   0.0    0.0   \n",
       "2   -5.397095    9.684463    2.665763   -5.264648  ...   0.0   0.0    0.0   \n",
       "3    1.191913   -0.049136    0.591174   -4.021995  ...   0.0   0.0    0.0   \n",
       "4    2.745538  -15.419270    9.987002   12.310934  ...   0.0   0.0    0.0   \n",
       "\n",
       "   dsDNAbreakinduction  gamma-secretase  notdefined  others  p38  p38alpha  \\\n",
       "0                  0.0              0.0         0.0     0.0  0.0       0.0   \n",
       "1                  0.0              0.0         0.0     0.0  0.0       0.0   \n",
       "2                  0.0              0.0         0.0     0.0  0.0       0.0   \n",
       "3                  0.0              0.0         0.0     0.0  0.0       0.0   \n",
       "4                  0.0              0.0         0.0     0.0  0.0       0.0   \n",
       "\n",
       "   p38beta  \n",
       "0      0.0  \n",
       "1      0.0  \n",
       "2      0.0  \n",
       "3      0.0  \n",
       "4      0.0  \n",
       "\n",
       "[5 rows x 371 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Join all \n",
    "cna_wes_gen_joined=pd.merge(cna_wes_joined,genomic_features,how='inner',on='COSMIC_ID')\n",
    "all_joined=pd.merge(cna_wes_gen_joined,drug_target,how='inner',on='DRUG_ID')\n",
    "all_joined1=pd.merge(all_joined,drug_with_target,how='inner',on='DRUG_ID')\n",
    "all_joined2=all_joined1.drop(['Sample Name','TISSUE_FACTOR','DRUG_ID'],axis=1)\n",
    "all_joined2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_joined2.shape\n",
    "all_joined2.to_csv(\"./data/model1_to_score.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Score with model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: For input data encodings other than the standard utf-8, see documentation at http://10.132.37.7/docs/users-guide/more-info/tabs/prime-examples.html"
     ]
    }
   ],
   "source": [
    "%run -i model1_scoring.py ./data/model1_to_score.csv scored_model1.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>COSMIC_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240182_5</td>\n",
       "      <td>3.165768</td>\n",
       "      <td>5</td>\n",
       "      <td>1240182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240189_5</td>\n",
       "      <td>2.658180</td>\n",
       "      <td>5</td>\n",
       "      <td>1240189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240193_5</td>\n",
       "      <td>2.386022</td>\n",
       "      <td>5</td>\n",
       "      <td>1240193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1297439_5</td>\n",
       "      <td>3.622384</td>\n",
       "      <td>5</td>\n",
       "      <td>1297439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1303901_5</td>\n",
       "      <td>3.548154</td>\n",
       "      <td>5</td>\n",
       "      <td>1303901</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  COSMIC_DRUG_ID  Prediction  DRUG_ID  COSMIC_ID\n",
       "0      1240182_5    3.165768        5    1240182\n",
       "1      1240189_5    2.658180        5    1240189\n",
       "2      1240193_5    2.386022        5    1240193\n",
       "3      1297439_5    3.622384        5    1297439\n",
       "4      1303901_5    3.548154        5    1303901"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoreddf=pd.read_csv(\"./scored_model1.csv\")\n",
    "\n",
    "scored=pd.concat([all_joined2,scoreddf],axis=1)[['COSMIC_DRUG_ID','Prediction']]\n",
    "scored[\"DRUG_ID\"]=(scored['COSMIC_DRUG_ID'].str.extract(pat = '([\"_\"].+)'))\n",
    "scored[\"DRUG_ID\"]=scored[\"DRUG_ID\"].str.replace('_','').astype(int)\n",
    "\n",
    "scored[\"COSMIC_ID\"]=(scored['COSMIC_DRUG_ID'].str.extract(pat = '(.+[\"_\"])'))\n",
    "scored[\"COSMIC_ID\"]=scored[\"COSMIC_ID\"].str.replace('_','').astype(int)\n",
    "scored.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1096, 4)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scored.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf=pd.read_csv(\"./Drug_Response_threshold.csv\")\n",
    "tdf=tdf.drop(['Screened Compounds:'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th>Predicted_IC50</th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>IC50_Threshold</th>\n",
       "      <th>Predicted_Resp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1240182_5</td>\n",
       "      <td>3.165768</td>\n",
       "      <td>5</td>\n",
       "      <td>1240182</td>\n",
       "      <td>0.50064</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1240189_5</td>\n",
       "      <td>2.658180</td>\n",
       "      <td>5</td>\n",
       "      <td>1240189</td>\n",
       "      <td>0.50064</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1240193_5</td>\n",
       "      <td>2.386022</td>\n",
       "      <td>5</td>\n",
       "      <td>1240193</td>\n",
       "      <td>0.50064</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1297439_5</td>\n",
       "      <td>3.622384</td>\n",
       "      <td>5</td>\n",
       "      <td>1297439</td>\n",
       "      <td>0.50064</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1303901_5</td>\n",
       "      <td>3.548154</td>\n",
       "      <td>5</td>\n",
       "      <td>1303901</td>\n",
       "      <td>0.50064</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  COSMIC_DRUG_ID  Predicted_IC50  DRUG_ID  COSMIC_ID  IC50_Threshold  \\\n",
       "0      1240182_5        3.165768        5    1240182         0.50064   \n",
       "1      1240189_5        2.658180        5    1240189         0.50064   \n",
       "2      1240193_5        2.386022        5    1240193         0.50064   \n",
       "3      1297439_5        3.622384        5    1297439         0.50064   \n",
       "4      1303901_5        3.548154        5    1303901         0.50064   \n",
       "\n",
       "  Predicted_Resp  \n",
       "0              S  \n",
       "1              S  \n",
       "2              S  \n",
       "3              S  \n",
       "4              S  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scored_resp=pd.merge(scored,tdf,how='left',on='DRUG_ID')\n",
    "scored_resp=scored_resp.rename(columns={'Prediction':'Predicted_IC50'})\n",
    "scored_resp['Predicted_Resp']=np.where(scored_resp['Predicted_IC50']>=scored_resp['IC50_Threshold'],\"S\",\"R\")\n",
    "scored_resp['Predicted_Resp']=np.where(scored_resp['IC50_Threshold'].isnull(),\"UNK\",scored_resp['Predicted_Resp'])\n",
    "scored_resp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COSMIC_DRUG_ID</th>\n",
       "      <th>Predicted_IC50</th>\n",
       "      <th>DRUG_ID</th>\n",
       "      <th>COSMIC_ID</th>\n",
       "      <th>IC50_Threshold</th>\n",
       "      <th>Predicted_Resp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>303</td>\n",
       "      <td>1240186_155</td>\n",
       "      <td>1.321044</td>\n",
       "      <td>155</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>357</td>\n",
       "      <td>1240186_193</td>\n",
       "      <td>4.602482</td>\n",
       "      <td>193</td>\n",
       "      <td>1240186</td>\n",
       "      <td>4.5004</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>411</td>\n",
       "      <td>1240186_199</td>\n",
       "      <td>3.679495</td>\n",
       "      <td>199</td>\n",
       "      <td>1240186</td>\n",
       "      <td>1.6529</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>465</td>\n",
       "      <td>1240186_254</td>\n",
       "      <td>1.999565</td>\n",
       "      <td>254</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>519</td>\n",
       "      <td>1240186_277</td>\n",
       "      <td>2.347424</td>\n",
       "      <td>277</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>573</td>\n",
       "      <td>1240186_281</td>\n",
       "      <td>3.582908</td>\n",
       "      <td>281</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>627</td>\n",
       "      <td>1240186_298</td>\n",
       "      <td>4.060090</td>\n",
       "      <td>298</td>\n",
       "      <td>1240186</td>\n",
       "      <td>2.7919</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>681</td>\n",
       "      <td>1240186_304</td>\n",
       "      <td>3.540771</td>\n",
       "      <td>304</td>\n",
       "      <td>1240186</td>\n",
       "      <td>2.7263</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>734</td>\n",
       "      <td>1240186_308</td>\n",
       "      <td>0.409808</td>\n",
       "      <td>308</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>788</td>\n",
       "      <td>1240186_312</td>\n",
       "      <td>0.620808</td>\n",
       "      <td>312</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1045</td>\n",
       "      <td>1240186_1194</td>\n",
       "      <td>4.095321</td>\n",
       "      <td>1194</td>\n",
       "      <td>1240186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     COSMIC_DRUG_ID  Predicted_IC50  DRUG_ID  COSMIC_ID  IC50_Threshold  \\\n",
       "303     1240186_155        1.321044      155    1240186             NaN   \n",
       "357     1240186_193        4.602482      193    1240186          4.5004   \n",
       "411     1240186_199        3.679495      199    1240186          1.6529   \n",
       "465     1240186_254        1.999565      254    1240186             NaN   \n",
       "519     1240186_277        2.347424      277    1240186             NaN   \n",
       "573     1240186_281        3.582908      281    1240186             NaN   \n",
       "627     1240186_298        4.060090      298    1240186          2.7919   \n",
       "681     1240186_304        3.540771      304    1240186          2.7263   \n",
       "734     1240186_308        0.409808      308    1240186             NaN   \n",
       "788     1240186_312        0.620808      312    1240186             NaN   \n",
       "1045   1240186_1194        4.095321     1194    1240186             NaN   \n",
       "\n",
       "     Predicted_Resp  \n",
       "303             UNK  \n",
       "357               S  \n",
       "411               S  \n",
       "465             UNK  \n",
       "519             UNK  \n",
       "573             UNK  \n",
       "627               S  \n",
       "681               S  \n",
       "734             UNK  \n",
       "788             UNK  \n",
       "1045            UNK  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scored_resp[scored_resp['COSMIC_ID']==1240186]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}