split_data_gen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import json \n",
    "import pdb \n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_comp = pd.read_csv('./CNN_data/training_dataset/training_compound.csv')\n",
    "tr_prot = pd.read_csv('./CNN_data/training_dataset/training_protein.csv')\n",
    "tr_dti = pd.read_csv('./CNN_data/training_dataset/training_dti.csv')\n",
    "\n",
    "\n",
    "vl_comp = pd.read_csv('./CNN_data/validation_dataset/validation_compound.csv')\n",
    "vl_prot = pd.read_csv('./CNN_data/validation_dataset/validation_protein.csv')\n",
    "vl_dti = pd.read_csv('./CNN_data/validation_dataset/validation_dti.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9586, 3)\n",
      "(3117, 2)\n",
      "(17751, 3)\n",
      "(3704, 3)\n",
      "(1826, 2)\n",
      "(4445, 3)\n"
     ]
    }
   ],
   "source": [
    "print(tr_comp.shape)\n",
    "print(tr_prot.shape)\n",
    "print(tr_dti.shape)\n",
    "print(vl_comp.shape)\n",
    "print(vl_prot.shape)\n",
    "print(vl_dti.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_comp = pd.concat([tr_comp, vl_comp])\n",
    "total_prot = pd.concat([tr_prot, vl_prot])\n",
    "total_dti = pd.concat([tr_dti, vl_dti])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13290, 3)\n",
      "(4943, 2)\n",
      "(22196, 3)\n"
     ]
    }
   ],
   "source": [
    "print(total_comp.shape)\n",
    "print(total_prot.shape)\n",
    "print(total_dti.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## removing duplicates "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10347, 3)\n",
      "(3171, 2)\n",
      "(22196, 3)\n"
     ]
    }
   ],
   "source": [
    "comp_nodupl = total_comp.drop_duplicates()\n",
    "prot_nodupl = total_prot.drop_duplicates()\n",
    "dti_nodupl = total_dti.drop_duplicates()\n",
    "\n",
    "print(comp_nodupl.shape)\n",
    "print(prot_nodupl.shape)\n",
    "print(dti_nodupl.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generating a file to test novel pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "pid_list = list(dti_nodupl['Protein_ID'])\n",
    "cid_list = list(dti_nodupl['Compound_ID'])\n",
    "\n",
    "\n",
    "cid_full_list = list(comp_nodupl['Compound_ID'])\n",
    "pid_full_list = list(prot_nodupl['Protein_ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'P15309'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pid_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from more_itertools import locate \n",
    "list(locate(pid_list, lambda x: x == pid_list[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10347/10347 [00:05<00:00, 2067.91it/s]\n"
     ]
    }
   ],
   "source": [
    "pid_final_test = []\n",
    "cid_final_test = []\n",
    "\n",
    "\n",
    "for i in tqdm(cid_full_list):\n",
    "    for j in pid_full_list:\n",
    "#         if i in pid_list:\n",
    "#             ix = list(locate(pid_full_list, lambda x: x == i))\n",
    "#             cid_temp_list = []\n",
    "#             for k in ix:\n",
    "#                 cid_temp_list.append(cid_full_list[k])\n",
    "            \n",
    "#             if j in cid_temp_list:\n",
    "#                 pass\n",
    "#             else:\n",
    "        pid_final_test.append(j)\n",
    "        cid_final_test.append(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3171/3171 [16:16<00:00,  3.25it/s]\n"
     ]
    }
   ],
   "source": [
    "pid_test_list = []\n",
    "cid_test_list = []\n",
    "\n",
    "for i in tqdm(list(set(pid_final_test))):\n",
    "    if i in pid_full_list:\n",
    "        ix = list(locate(pid_list, lambda x: x == i))\n",
    "        cid_temp_list = []\n",
    "        for k in ix:\n",
    "            cid_temp_list.append(cid_list[k])\n",
    "\n",
    "        for j in list(set(cid_final_test)):\n",
    "            if j not in cid_temp_list:\n",
    "                pid_test_list.append(i)\n",
    "                cid_test_list.append(j)\n",
    "\n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32788141"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_file = pd.DataFrame({'Protein_ID': pid_test_list, 'Compound_ID': cid_test_list})\n",
    "df_file.head()\n",
    "\n",
    "df_file.to_csv('./test_random_pairs/dti_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "comp_nodupl.to_csv('./test_random_pairs/dti_comp_noidx.csv', index= False)\n",
    "prot_nodupl.to_csv('./test_random_pairs/dti_prot_noidx.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_file_1 = pd.DataFrame({'Protein_ID': pid_test_list[0:100000], 'CoFalseund_ID': cid_test_list[0:100000]})\n",
    "df_file_1.head()\n",
    "\n",
    "df_file_1.to_csv('./test_random_pairs/dti_test_100000_idx.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## equal split of positive and negative dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protein_ID</th>\n",
       "      <th>Compound_ID</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P15309</td>\n",
       "      <td>SUB-7688</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q0ZUL0</td>\n",
       "      <td>SUB-417</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q13093</td>\n",
       "      <td>SUB-5066</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>E5DD06</td>\n",
       "      <td>SUB-7215</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S5M6X9</td>\n",
       "      <td>SUB-1824</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Protein_ID Compound_ID  Label\n",
       "0     P15309    SUB-7688      0\n",
       "1     Q0ZUL0     SUB-417      1\n",
       "2     Q13093    SUB-5066      1\n",
       "3     E5DD06    SUB-7215      0\n",
       "4     S5M6X9    SUB-1824      1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dti_nodupl.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = dti_nodupl['Label'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total positive activity pairs:  11100\n",
      "total negative activity pairs:  11096\n"
     ]
    }
   ],
   "source": [
    "pos_act = sum(labels)\n",
    "neg_act = len(labels) - pos_act\n",
    "\n",
    "print('total positive activity pairs: ', pos_act)\n",
    "print('total negative activity pairs: ', neg_act)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_df = dti_nodupl[dti_nodupl['Label'] == 1]\n",
    "negative_df = dti_nodupl[dti_nodupl['Label'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of total positive pairs:  (11100, 3)\n",
      "shape of total negative pairs:  (11096, 3)\n"
     ]
    }
   ],
   "source": [
    "print('shape of total positive pairs: ', positive_df.shape)\n",
    "print('shape of total negative pairs: ', negative_df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# splitting 80-20 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(11):\n",
    "    train_pos, test_pos = train_test_split(positive_df, test_size = 0.1, random_state = i)\n",
    "    train_neg, test_neg = train_test_split(negative_df, test_size = 0.1, random_state = i)\n",
    "    \n",
    "    train_positive, validation_positive = train_test_split(train_pos, test_size = 1/9, random_state = i)\n",
    "    train_negative, validation_negative = train_test_split(train_neg, test_size = 1/9, random_state = i)\n",
    "    \n",
    "    train_concat = pd.concat([train_positive, train_negative])\n",
    "    test_concat = pd.concat([test_pos, test_neg])\n",
    "    val_concat = pd.concat([validation_positive, validation_negative])\n",
    "\n",
    "    train_shuffle = shuffle(train_concat, random_state = i) #.sample(frac=1, random_state = i)\n",
    "    test_shuffle = shuffle(test_concat, random_state = i) #.sample(frac =1, random_state = i)\n",
    "    val_shuffle = shuffle(val_concat, random_state = i) #.sample(frac =1, random_state = i)\n",
    "    \n",
    "#     print(type(train_shuffle))\n",
    "#     print(train_shuffle.shape)\n",
    "    \n",
    "    train_reindex = train_shuffle.reset_index(drop=True)\n",
    "    test_reindex = test_shuffle.reset_index(drop=True)\n",
    "    val_reindex = val_shuffle.reset_index(drop=True)\n",
    "    \n",
    "    \n",
    "    tr_folder = './CNN_data_split/CNN_data_' + str(i) + '/training_dataset/training_act.csv'\n",
    "    val_folder = './CNN_data_split/CNN_data_' + str(i) + '/validation_dataset/validation_act.csv' \n",
    "    test_folder = './CNN_data_split/CNN_data_' + str(i) + '/test_dataset/test_act.csv'\n",
    "    \n",
    "    train_reindex.to_csv(tr_folder, index =False)\n",
    "    val_reindex.to_csv(val_folder, index = False)\n",
    "    test_reindex.to_csv(test_folder, index = False)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17756, 3)\n",
      "(2220, 3)\n",
      "(2220, 3)\n"
     ]
    }
   ],
   "source": [
    "print(train_reindex.shape)\n",
    "print(val_reindex.shape)\n",
    "print(test_reindex.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8880.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "11100*0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8876.800000000001"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "11096*0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17756"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "8880+8876"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 1/11 [00:32<05:22, 32.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9607\n",
      "df shape (9607, 3)\n",
      "len pid train 3113\n",
      "df shape (3113, 2)\n",
      "len cid val 2002\n",
      "df shape (2002, 3)\n",
      "len pid val 1237\n",
      "df shape (1237, 2)\n",
      "len cid test 2005\n",
      "df shape (2005, 3)\n",
      "len pid test 1234\n",
      "df shape (1234, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 2/11 [01:03<04:47, 31.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9555\n",
      "df shape (9555, 3)\n",
      "len pid train 3103\n",
      "df shape (3103, 2)\n",
      "len cid val 1994\n",
      "df shape (1994, 3)\n",
      "len pid val 1224\n",
      "df shape (1224, 2)\n",
      "len cid test 2005\n",
      "df shape (2005, 3)\n",
      "len pid test 1249\n",
      "df shape (1249, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 3/11 [01:35<04:14, 31.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9584\n",
      "df shape (9584, 3)\n",
      "len pid train 3106\n",
      "df shape (3106, 2)\n",
      "len cid val 2007\n",
      "df shape (2007, 3)\n",
      "len pid val 1209\n",
      "df shape (1209, 2)\n",
      "len cid test 1996\n",
      "df shape (1996, 3)\n",
      "len pid test 1237\n",
      "df shape (1237, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▋      | 4/11 [02:08<03:45, 32.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9571\n",
      "df shape (9571, 3)\n",
      "len pid train 3112\n",
      "df shape (3112, 2)\n",
      "len cid val 1995\n",
      "df shape (1995, 3)\n",
      "len pid val 1212\n",
      "df shape (1212, 2)\n",
      "len cid test 2006\n",
      "df shape (2006, 3)\n",
      "len pid test 1228\n",
      "df shape (1228, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 5/11 [02:40<03:11, 31.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9546\n",
      "df shape (9546, 3)\n",
      "len pid train 3118\n",
      "df shape (3118, 2)\n",
      "len cid val 2026\n",
      "df shape (2026, 3)\n",
      "len pid val 1204\n",
      "df shape (1204, 2)\n",
      "len cid test 1997\n",
      "df shape (1997, 3)\n",
      "len pid test 1240\n",
      "df shape (1240, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 6/11 [03:11<02:39, 31.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9550\n",
      "df shape (9550, 3)\n",
      "len pid train 3118\n",
      "df shape (3118, 2)\n",
      "len cid val 1990\n",
      "df shape (1990, 3)\n",
      "len pid val 1214\n",
      "df shape (1214, 2)\n",
      "len cid test 1963\n",
      "df shape (1963, 3)\n",
      "len pid test 1240\n",
      "df shape (1240, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▎   | 7/11 [03:43<02:07, 31.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9552\n",
      "df shape (9552, 3)\n",
      "len pid train 3105\n",
      "df shape (3105, 2)\n",
      "len cid val 1967\n",
      "df shape (1967, 3)\n",
      "len pid val 1266\n",
      "df shape (1266, 2)\n",
      "len cid test 1979\n",
      "df shape (1979, 3)\n",
      "len pid test 1232\n",
      "df shape (1232, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 8/11 [04:15<01:35, 31.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9594\n",
      "df shape (9594, 3)\n",
      "len pid train 3101\n",
      "df shape (3101, 2)\n",
      "len cid val 2000\n",
      "df shape (2000, 3)\n",
      "len pid val 1242\n",
      "df shape (1242, 2)\n",
      "len cid test 1992\n",
      "df shape (1992, 3)\n",
      "len pid test 1241\n",
      "df shape (1241, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 9/11 [04:46<01:03, 31.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9533\n",
      "df shape (9533, 3)\n",
      "len pid train 3115\n",
      "df shape (3115, 2)\n",
      "len cid val 1980\n",
      "df shape (1980, 3)\n",
      "len pid val 1191\n",
      "df shape (1191, 2)\n",
      "len cid test 2009\n",
      "df shape (2009, 3)\n",
      "len pid test 1231\n",
      "df shape (1231, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 10/11 [05:18<00:31, 31.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9602\n",
      "df shape (9602, 3)\n",
      "len pid train 3119\n",
      "df shape (3119, 2)\n",
      "len cid val 2011\n",
      "df shape (2011, 3)\n",
      "len pid val 1210\n",
      "df shape (1210, 2)\n",
      "len cid test 2010\n",
      "df shape (2010, 3)\n",
      "len pid test 1191\n",
      "df shape (1191, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11/11 [05:50<00:00, 31.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---------------------------------------#\n",
      "len cid train 9554\n",
      "df shape (9554, 3)\n",
      "len pid train 3109\n",
      "df shape (3109, 2)\n",
      "len cid val 2012\n",
      "df shape (2012, 3)\n",
      "len pid val 1238\n",
      "df shape (1238, 2)\n",
      "len cid test 1995\n",
      "df shape (1995, 3)\n",
      "len pid test 1253\n",
      "df shape (1253, 2)\n",
      "#---------------------------------------#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for k in tqdm(range(11)):\n",
    "    train_folder = './CNN_data_split/CNN_data_' + str(k) + '/training_dataset/training_act'\n",
    "    valid_folder = './CNN_data_split/CNN_data_' + str(k) + '/validation_dataset/validation_act' \n",
    "    tes_folder = './CNN_data_split/CNN_data_' + str(k) + '/test_dataset/test_act'\n",
    "    \n",
    "    tr_read = pd.read_csv(train_folder)\n",
    "    val_read = pd.read_csv(valid_folder)\n",
    "    tes_read = pd.read_csv(tes_folder)\n",
    "\n",
    "    pid_tr = tr_read['Protein_ID'].to_list()\n",
    "    pid_val = val_read['Protein_ID'].to_list()\n",
    "    pid_tes = tes_read['Protein_ID'].to_list()\n",
    "    \n",
    "    cid_tr = tr_read['Compound_ID'].to_list()\n",
    "    cid_val = val_read['Compound_ID'].to_list()\n",
    "    cid_tes = tes_read['Compound_ID'].to_list()\n",
    "    \n",
    "    uniq_pid_tr = list(set(pid_tr))\n",
    "    uniq_pid_val = list(set(pid_val))\n",
    "    uniq_pid_tes = list(set(pid_tes))\n",
    "    \n",
    "    uniq_cid_tr = list(set(cid_tr))\n",
    "    uniq_cid_val = list(set(cid_val))\n",
    "    uniq_cid_tes = list(set(cid_tes))\n",
    "    \n",
    "## train    \n",
    "    for i in range(len(uniq_cid_tr)):\n",
    "        if i==0:\n",
    "            train_comp_df = comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid_tr[i]]\n",
    "        else:\n",
    "            train_comp_df = pd.concat([comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid_tr[i]], train_comp_df])\n",
    "    \n",
    "\n",
    "    for i in range(len(uniq_pid_tr)):\n",
    "        if i==0:\n",
    "            train_prot_df = prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid_tr[i]]\n",
    "        else:\n",
    "            train_prot_df = pd.concat([prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid_tr[i]], train_prot_df])\n",
    "             \n",
    "## valid      \n",
    "    for i in range(len(uniq_cid_val)):\n",
    "        if i==0:\n",
    "            valid_comp_df = comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid_val[i]]\n",
    "        else:\n",
    "            valid_comp_df = pd.concat([comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid_val[i]], valid_comp_df])\n",
    "    \n",
    "\n",
    "    for i in range(len(uniq_pid_val)):\n",
    "        if i==0:\n",
    "            valid_prot_df = prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid_val[i]]\n",
    "        else:\n",
    "            valid_prot_df = pd.concat([prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid_val[i]], valid_prot_df])\n",
    "            \n",
    "## test                \n",
    "    for i in range(len(uniq_cid_tes)):\n",
    "        if i==0:\n",
    "            test_comp_df = comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid_tes[i]]\n",
    "        else:\n",
    "            test_comp_df = pd.concat([comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid_tes[i]], test_comp_df])\n",
    "    \n",
    "\n",
    "    for i in range(len(uniq_pid_tes)):\n",
    "        if i==0:\n",
    "            test_prot_df = prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid_tes[i]]\n",
    "        else:\n",
    "            test_prot_df = pd.concat([prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid_tes[i]], test_prot_df])\n",
    "            \n",
    "\n",
    "    train_comp_df = train_comp_df.reset_index(drop=True)\n",
    "    train_prot_df = train_prot_df.reset_index(drop=True)\n",
    "    \n",
    "    valid_comp_df = valid_comp_df.reset_index(drop=True)\n",
    "    valid_prot_df = valid_prot_df.reset_index(drop=True)\n",
    "    \n",
    "    test_comp_df = test_comp_df.reset_index(drop=True)\n",
    "    test_prot_df = test_prot_df.reset_index(drop=True)\n",
    "    \n",
    "\n",
    "    \n",
    "    train_fname_cid = './CNN_data_split/CNN_data_' + str(k) + '/training_dataset/training_compound.csv'\n",
    "    train_fname_pid = './CNN_data_split/CNN_data_' + str(k) + '/training_dataset/training_protein.csv'\n",
    "    \n",
    "    val_fname_cid = './CNN_data_split/CNN_data_' + str(k) + '/validation_dataset/validation_compound.csv' \n",
    "    val_fname_pid = './CNN_data_split/CNN_data_' + str(k) + '/validation_dataset/validation_protein.csv' \n",
    "    \n",
    "    test_fname_cid = './CNN_data_split/CNN_data_' + str(k) + '/test_dataset/test_compound.csv'\n",
    "    test_fname_pid = './CNN_data_split/CNN_data_' + str(k) + '/test_dataset/test_protein.csv'\n",
    "        \n",
    "    train_comp_df.to_csv(train_fname_cid, index =False)\n",
    "    train_prot_df.to_csv(train_fname_pid, index =False)\n",
    "    \n",
    "    valid_comp_df.to_csv(val_fname_cid, index = False)\n",
    "    valid_prot_df.to_csv(val_fname_pid, index =False)\n",
    "    \n",
    "    test_comp_df.to_csv(test_fname_cid, index = False)\n",
    "    test_prot_df.to_csv(test_fname_pid, index =False)\n",
    "    \n",
    "    print('#---------------------------------------#')\n",
    "    \n",
    "    print('len cid train', len(uniq_cid_tr))\n",
    "    print('df shape', train_comp_df.shape)\n",
    "\n",
    "    print('len pid train', len(uniq_pid_tr))\n",
    "    print('df shape', train_prot_df.shape)\n",
    "\n",
    "    print('len cid val', len(uniq_cid_val))\n",
    "    print('df shape', valid_comp_df.shape)\n",
    "\n",
    "    print('len pid val', len(uniq_pid_val))\n",
    "    print('df shape', valid_prot_df.shape)\n",
    "    \n",
    "    print('len cid test', len(uniq_cid_tes))\n",
    "    print('df shape', test_comp_df.shape)\n",
    "\n",
    "    print('len pid test', len(uniq_pid_tes))\n",
    "    print('df shape', test_prot_df.shape)\n",
    "    \n",
    "    print('#---------------------------------------#')\n",
    "\n",
    "# A = pd.read_csv('./CNN_data_split/CNN_data_0/training_dataset/training_act')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1234"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(uniq_pid_tes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "pidA = A['Protein_ID'].to_list()\n",
    "cidA = A['Compound_ID'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kjjfkadsjf 3112\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniq_pid = list(set(pidA))\n",
    "uniq_cid = list(set(cidA))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protein_ID</th>\n",
       "      <th>Sequence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P04335</td>\n",
       "      <td>MTQANLSETLFKPRFKHPETSTLVRRFNHGAQPPVQSALDGKTIPH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>R9WYJ0</td>\n",
       "      <td>MSTRKAVIGYYFIPTNQINNYTETDTSVVPFPVSNITPAKAKQLTH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P0A111</td>\n",
       "      <td>MNYNNKILVSESGLSQKHLIHGDEELFQHELKTIFARNWLFLTHDS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>P54315</td>\n",
       "      <td>MLIFWTITLFLLGAAKGKEVCYEDLGCFSDTEPWGGTAIRPLKILP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>P76270</td>\n",
       "      <td>MNKTEFYADLNRDFNALMAGETSFLATLANTSALLYERLTDINWAG...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Protein_ID                                           Sequence\n",
       "0     P04335  MTQANLSETLFKPRFKHPETSTLVRRFNHGAQPPVQSALDGKTIPH...\n",
       "1     R9WYJ0  MSTRKAVIGYYFIPTNQINNYTETDTSVVPFPVSNITPAKAKQLTH...\n",
       "2     P0A111  MNYNNKILVSESGLSQKHLIHGDEELFQHELKTIFARNWLFLTHDS...\n",
       "3     P54315  MLIFWTITLFLLGAAKGKEVCYEDLGCFSDTEPWGGTAIRPLKILP...\n",
       "4     P76270  MNKTEFYADLNRDFNALMAGETSFLATLANTSALLYERLTDINWAG..."
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prot_nodupl.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Compound_ID</th>\n",
       "      <th>smiles</th>\n",
       "      <th>morgan_fp_r2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SUB-6711</td>\n",
       "      <td>C=C</td>\n",
       "      <td>0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SUB-9324</td>\n",
       "      <td>C1=CC(=CC=C1[N+](=O)[O-])OC2C(C(C(C(O2)CO)O)O)O</td>\n",
       "      <td>0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SUB-6923</td>\n",
       "      <td>C1(C(C(C(C(C1OP(=O)(O)O)OP(=O)(O)O)OP(=O)(O)O)...</td>\n",
       "      <td>0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SUB-3175</td>\n",
       "      <td>ClC1=C([C@H](C#N)O)C=CC=C1</td>\n",
       "      <td>0.0\\t1.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SUB-6690</td>\n",
       "      <td>COC(=O)C=CC1=CC=C(C=C1)O</td>\n",
       "      <td>0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Compound_ID                                             smiles  \\\n",
       "0    SUB-6711                                                C=C   \n",
       "1    SUB-9324    C1=CC(=CC=C1[N+](=O)[O-])OC2C(C(C(C(O2)CO)O)O)O   \n",
       "2    SUB-6923  C1(C(C(C(C(C1OP(=O)(O)O)OP(=O)(O)O)OP(=O)(O)O)...   \n",
       "3    SUB-3175                         ClC1=C([C@H](C#N)O)C=CC=C1   \n",
       "4    SUB-6690                           COC(=O)C=CC1=CC=C(C=C1)O   \n",
       "\n",
       "                                        morgan_fp_r2  \n",
       "0  0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...  \n",
       "1  0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...  \n",
       "2  0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...  \n",
       "3  0.0\\t1.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...  \n",
       "4  0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0.0\\t0...  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comp_nodupl.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(uniq_cid)):\n",
    "    if i==0:\n",
    "        train_comp_df = comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid[i]]\n",
    "    else:\n",
    "        train_comp_df = pd.concat([comp_nodupl[comp_nodupl['Compound_ID'] == uniq_cid[i]], train_comp_df])\n",
    "    \n",
    "    \n",
    "for i in range(len(uniq_pid)):\n",
    "    if i==0:\n",
    "        train_prot_df = prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid[i]]\n",
    "    else:\n",
    "        train_prot_df = pd.concat([prot_nodupl[prot_nodupl['Protein_ID'] == uniq_pid[i]], train_prot_df])\n",
    "        \n",
    "        \n",
    "train_comp_df = train_comp_df.reset_index(drop=True)\n",
    "train_prot_df = train_prot_df.reset_index(drop=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protein_ID</th>\n",
       "      <th>Sequence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P21513</td>\n",
       "      <td>MKRMLINATQQEELRVALVDGQRLYDLDIESPGHEQKKANIYKGKI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>P62623</td>\n",
       "      <td>MQILLANPRGFCAGVDRAISIVENALAIYGAPIYVRHEVVHNRYVV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q552R4</td>\n",
       "      <td>MNKIFGIGNDIVKISRLESSFKRHGDKFLKRAFNEVEISIFKSLNP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A5LGH2</td>\n",
       "      <td>MHGNRTALKIDVLIVGTGPAGAASAALLGTYGVRALVINKYGWTAP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q9HLD0</td>\n",
       "      <td>MLDDIRGFMNTFSETMFMDVINYSLTRDRYDSVFLQRQNYRDLGQL...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Protein_ID                                           Sequence\n",
       "0     P21513  MKRMLINATQQEELRVALVDGQRLYDLDIESPGHEQKKANIYKGKI...\n",
       "1     P62623  MQILLANPRGFCAGVDRAISIVENALAIYGAPIYVRHEVVHNRYVV...\n",
       "2     Q552R4  MNKIFGIGNDIVKISRLESSFKRHGDKFLKRAFNEVEISIFKSLNP...\n",
       "3     A5LGH2  MHGNRTALKIDVLIVGTGPAGAASAALLGTYGVRALVINKYGWTAP...\n",
       "4     Q9HLD0  MLDDIRGFMNTFSETMFMDVINYSLTRDRYDSVFLQRQNYRDLGQL..."
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3113"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(uniq_pid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_prot_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## shell script for folder generation \n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for ((i=0; i<5; ++i)); do  cd \"CNN_data_$i\"; mkdir training_dataset; mkdir validation_dataset; mkdir test_dataset; cd ..; done \n",
    "for ((i=5; i<=10; ++i)); do  mkdir \"CNN_data_$i\"; cd \"CNN_data_$i\"; mkdir training_dataset; mkdir validation_dataset; mkdir test_dataset; cd ..; done \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}