Kfold-RFR_for_computing_Overfiting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:358: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if token is '':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:2437: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if options['min_type'] is 'min' and options['min_value'] == 0:\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:2440: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if options['max_type'] is 'max' and options['max_value'] == 0:\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:4999: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if props[i]['type'] is 'number':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:6827: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
      "  if data_bar['bar_axis_position'] is not 'none':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:6862: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if data_bar['bar_direction'] is 'left':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:6865: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if data_bar['bar_direction'] is 'right':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:6875: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if data_bar['bar_axis_position'] is 'middle':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/worksheet.py:6878: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if data_bar['bar_axis_position'] is 'none':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/chart.py:2497: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if val is 'right':\n",
      "/usr/lib/python3/dist-packages/xlsxwriter/chart.py:2500: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "  if val is 'left':\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "import xlsxwriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tg</th>\n",
       "      <th>Tx</th>\n",
       "      <th>Tl</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>...</th>\n",
       "      <th>X9</th>\n",
       "      <th>X10</th>\n",
       "      <th>X11</th>\n",
       "      <th>X12</th>\n",
       "      <th>X13</th>\n",
       "      <th>X14</th>\n",
       "      <th>X15</th>\n",
       "      <th>X16</th>\n",
       "      <th>X17</th>\n",
       "      <th>Y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>740.0</td>\n",
       "      <td>768.0</td>\n",
       "      <td>1191.0</td>\n",
       "      <td>0.621327</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.644836</td>\n",
       "      <td>1.659164</td>\n",
       "      <td>3.176232</td>\n",
       "      <td>0.397721</td>\n",
       "      <td>0.668346</td>\n",
       "      <td>...</td>\n",
       "      <td>0.389012</td>\n",
       "      <td>0.197099</td>\n",
       "      <td>0.406330</td>\n",
       "      <td>0.963643</td>\n",
       "      <td>0.657785</td>\n",
       "      <td>0.485598</td>\n",
       "      <td>0.062084</td>\n",
       "      <td>0.115800</td>\n",
       "      <td>0.691856</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>709.0</td>\n",
       "      <td>738.0</td>\n",
       "      <td>1135.0</td>\n",
       "      <td>0.624670</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.650220</td>\n",
       "      <td>1.665572</td>\n",
       "      <td>3.319874</td>\n",
       "      <td>0.400217</td>\n",
       "      <td>0.675771</td>\n",
       "      <td>...</td>\n",
       "      <td>0.395486</td>\n",
       "      <td>0.191724</td>\n",
       "      <td>0.409503</td>\n",
       "      <td>0.976184</td>\n",
       "      <td>0.663965</td>\n",
       "      <td>0.480190</td>\n",
       "      <td>0.068075</td>\n",
       "      <td>0.130456</td>\n",
       "      <td>0.701322</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>689.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>1119.0</td>\n",
       "      <td>0.615728</td>\n",
       "      <td>43.0</td>\n",
       "      <td>0.654156</td>\n",
       "      <td>1.678138</td>\n",
       "      <td>3.367506</td>\n",
       "      <td>0.404867</td>\n",
       "      <td>0.692583</td>\n",
       "      <td>...</td>\n",
       "      <td>0.414104</td>\n",
       "      <td>0.179089</td>\n",
       "      <td>0.418693</td>\n",
       "      <td>1.001672</td>\n",
       "      <td>0.674472</td>\n",
       "      <td>0.467620</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.197818</td>\n",
       "      <td>0.731010</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>693.0</td>\n",
       "      <td>730.0</td>\n",
       "      <td>1112.0</td>\n",
       "      <td>0.623201</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.656475</td>\n",
       "      <td>1.676592</td>\n",
       "      <td>3.466805</td>\n",
       "      <td>0.404432</td>\n",
       "      <td>0.689748</td>\n",
       "      <td>...</td>\n",
       "      <td>0.409880</td>\n",
       "      <td>0.181448</td>\n",
       "      <td>0.416395</td>\n",
       "      <td>0.998870</td>\n",
       "      <td>0.673886</td>\n",
       "      <td>0.469929</td>\n",
       "      <td>0.088305</td>\n",
       "      <td>0.175715</td>\n",
       "      <td>0.723022</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>700.0</td>\n",
       "      <td>734.0</td>\n",
       "      <td>1115.0</td>\n",
       "      <td>0.627803</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.658296</td>\n",
       "      <td>1.676374</td>\n",
       "      <td>3.539518</td>\n",
       "      <td>0.404408</td>\n",
       "      <td>0.688789</td>\n",
       "      <td>...</td>\n",
       "      <td>0.407357</td>\n",
       "      <td>0.182329</td>\n",
       "      <td>0.415360</td>\n",
       "      <td>0.997526</td>\n",
       "      <td>0.674124</td>\n",
       "      <td>0.470753</td>\n",
       "      <td>0.081928</td>\n",
       "      <td>0.163956</td>\n",
       "      <td>0.719283</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Tg     Tx      Tl        X1    X2        X3        X4        X5  \\\n",
       "0  740.0  768.0  1191.0  0.621327  28.0  0.644836  1.659164  3.176232   \n",
       "1  709.0  738.0  1135.0  0.624670  29.0  0.650220  1.665572  3.319874   \n",
       "2  689.0  732.0  1119.0  0.615728  43.0  0.654156  1.678138  3.367506   \n",
       "3  693.0  730.0  1112.0  0.623201  37.0  0.656475  1.676592  3.466805   \n",
       "4  700.0  734.0  1115.0  0.627803  34.0  0.658296  1.676374  3.539518   \n",
       "\n",
       "         X6        X7  ...        X9       X10       X11       X12       X13  \\\n",
       "0  0.397721  0.668346  ...  0.389012  0.197099  0.406330  0.963643  0.657785   \n",
       "1  0.400217  0.675771  ...  0.395486  0.191724  0.409503  0.976184  0.663965   \n",
       "2  0.404867  0.692583  ...  0.414104  0.179089  0.418693  1.001672  0.674472   \n",
       "3  0.404432  0.689748  ...  0.409880  0.181448  0.416395  0.998870  0.673886   \n",
       "4  0.404408  0.688789  ...  0.407357  0.182329  0.415360  0.997526  0.674124   \n",
       "\n",
       "        X14       X15       X16       X17     Y  \n",
       "0  0.485598  0.062084  0.115800  0.691856   3.0  \n",
       "1  0.480190  0.068075  0.130456  0.701322   6.0  \n",
       "2  0.467620  0.100000  0.197818  0.731010   6.0  \n",
       "3  0.469929  0.088305  0.175715  0.723022   8.0  \n",
       "4  0.470753  0.081928  0.163956  0.719283  10.0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pd.read_csv(\"Final Dataset.csv\",usecols = ['X1','X2','X3','X4','X5','X6','X7','X8','X9','X10','X11','X12','X13','X14','X15','X16','X17','Tg','Tx','Tl','Y'])\n",
    "workbook = xlsxwriter.Workbook('best13+3.xlsx')\n",
    "worksheet = workbook.add_worksheet()\n",
    "dataset.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7.40000000e+02 7.68000000e+02 1.19100000e+03 6.21326616e-01\n",
      " 2.80000000e+01 6.44836270e-01 1.65916445e+00 3.17623191e+00\n",
      " 3.97721390e-01 6.68345930e-01 1.70288248e+00 3.89012380e-01\n",
      " 1.97099410e-01 4.06329760e-01 9.63643310e-01 6.57784950e-01\n",
      " 4.85598120e-01 6.20842600e-02 1.15800120e-01 6.91855580e-01]\n"
     ]
    }
   ],
   "source": [
    "m=[]\n",
    "X = dataset.iloc[:,0:20].values.astype(float)\n",
    "print(X[0,:])\n",
    "m.append(min(X[:,0]))\n",
    "m.append(max(X[:,0]))\n",
    "X[:,0] = (X[:,0] - min(X[:,0]))/(max(X[:,0])-min(X[:,0]))\n",
    "m.append(min(X[:,1]))\n",
    "m.append(max(X[:,1]))\n",
    "X[:,1] = (X[:,1] - min(X[:,1]))/(max(X[:,1])-min(X[:,1]))\n",
    "m.append(min(X[:,2]))\n",
    "m.append(max(X[:,2]))\n",
    "X[:,2] = (X[:,2] - min(X[:,2]))/(max(X[:,2])-min(X[:,2]))\n",
    "m.append(min(X[:,3]))\n",
    "m.append(max(X[:,3]))\n",
    "X[:,3] = (X[:,3] - min(X[:,3]))/(max(X[:,3])-min(X[:,3]))\n",
    "m.append(min(X[:,4]))\n",
    "m.append(max(X[:,4]))\n",
    "X[:,4] = (X[:,4] - min(X[:,4]))/(max(X[:,4])-min(X[:,4]))\n",
    "m.append(min(X[:,5]))\n",
    "m.append(max(X[:,5]))\n",
    "X[:,5] = (X[:,5] - min(X[:,5]))/(max(X[:,5])-min(X[:,5]))\n",
    "m.append(min(X[:,6]))\n",
    "m.append(max(X[:,6]))\n",
    "X[:,6] = (X[:,6] - min(X[:,6]))/(max(X[:,6])-min(X[:,6]))\n",
    "m.append(min(X[:,7]))\n",
    "m.append(max(X[:,7]))\n",
    "X[:,7] = (X[:,7] - min(X[:,7]))/(max(X[:,7])-min(X[:,7]))\n",
    "m.append(min(X[:,8]))\n",
    "m.append(max(X[:,8]))\n",
    "X[:,8] = (X[:,8] - min(X[:,8]))/(max(X[:,8])-min(X[:,8]))\n",
    "m.append(min(X[:,9]))\n",
    "m.append(max(X[:,9]))\n",
    "X[:,9] = (X[:,9] - min(X[:,9]))/(max(X[:,9])-min(X[:,9]))\n",
    "m.append(min(X[:,10]))\n",
    "m.append(max(X[:,10]))\n",
    "X[:,10] = (X[:,10] - min(X[:,10]))/(max(X[:,10])-min(X[:,10]))\n",
    "m.append(min(X[:,11]))\n",
    "m.append(max(X[:,11]))\n",
    "X[:,11] = (X[:,11] - min(X[:,11]))/(max(X[:,11])-min(X[:,11]))\n",
    "m.append(min(X[:,12]))\n",
    "m.append(max(X[:,12]))\n",
    "X[:,12] = (X[:,12] - min(X[:,12]))/(max(X[:,12])-min(X[:,12]))\n",
    "m.append(min(X[:,13]))\n",
    "m.append(max(X[:,13]))\n",
    "X[:,13] = (X[:,13] - min(X[:,13]))/(max(X[:,13])-min(X[:,13]))\n",
    "m.append(min(X[:,14]))\n",
    "m.append(max(X[:,14]))\n",
    "X[:,14] = (X[:,14] - min(X[:,14]))/(max(X[:,14])-min(X[:,14]))\n",
    "m.append(min(X[:,15]))\n",
    "m.append(max(X[:,15]))\n",
    "X[:,15] = (X[:,15] - min(X[:,15]))/(max(X[:,15])-min(X[:,15]))\n",
    "m.append(min(X[:,16]))\n",
    "m.append(max(X[:,16]))\n",
    "X[:,16] = (X[:,16] - min(X[:,16]))/(max(X[:,16])-min(X[:,16]))\n",
    "m.append(min(X[:,17]))\n",
    "m.append(max(X[:,17]))\n",
    "X[:,17] = (X[:,17] - min(X[:,17]))/(max(X[:,17])-min(X[:,17]))\n",
    "m.append(min(X[:,18]))\n",
    "m.append(max(X[:,18]))\n",
    "X[:,18] = (X[:,18] - min(X[:,18]))/(max(X[:,18])-min(X[:,18]))\n",
    "m.append(min(X[:,19]))\n",
    "m.append(max(X[:,19]))\n",
    "X[:,19] = (X[:,19] - min(X[:,19]))/(max(X[:,19])-min(X[:,19]))\n",
    "\n",
    "y = dataset.iloc[:,20].values.astype(float)\n",
    "#y = (y - min(y))/(max(y)-min(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.949079961646447\n",
      "[0.64790997 0.61205564 0.59781421 0.75481835 0.15547025 0.60513405\n",
      " 0.54115792 0.2369204  0.58087253 0.50679012 0.54019126 0.48124851\n",
      " 0.437283   0.5119773  0.54174228 0.56588553 0.47673743 0.14744453\n",
      " 0.06472791 0.43723936]\n",
      "{0: 24, 5: 66, 10: 9, 100: 1}\n",
      "[0.05497759 0.04846529 0.08780012 0.0560756  0.04148325 0.03711666\n",
      " 0.02116097 0.04226153 0.02877959 0.05326074 0.03933098 0.08661205\n",
      " 0.0659095  0.03209046 0.01807744 0.02440583 0.03627043 0.06587416\n",
      " 0.12248496 0.03756284]\n",
      "Test accuracy average in RandomForest: 0.9065452217139237\n",
      "Train accuracy average in RandomForest: 0.9089334421584929\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVR\n",
    "from sklearn.model_selection import RepeatedKFold,ShuffleSplit\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.feature_selection import RFE \n",
    "from sklearn import metrics\n",
    "d={0:0,\n",
    "  5:0,\n",
    "  10:0,\n",
    "    100:0\n",
    "}\n",
    "cv = RepeatedKFold(n_splits=5, n_repeats=20, random_state=123)\n",
    "count1RF=0\n",
    "count2RF=0\n",
    "maxR2=0\n",
    "selector = RandomForestRegressor(max_depth= 21,n_estimators=36, random_state=False)\n",
    "for train, test in cv.split(y):\n",
    "    selector = selector.fit(X, y)\n",
    "    y_pred2 = selector.predict(X[test])\n",
    "    x_pred2 = selector.predict(X[train])\n",
    "    R2test2=1-(((y[test] - y_pred2) ** 2).sum())/((y[test] - y[test].mean()) ** 2).sum()\n",
    "    R2train2=1-(((y[train] - x_pred2) ** 2).sum())/((y[train] - y[train].mean()) ** 2).sum()\n",
    "    if abs(R2train2-R2test2)*100>10:\n",
    "        d[100]+=1\n",
    "    elif abs(R2train2-R2test2)*100>5:\n",
    "        d[10]+=1\n",
    "    elif abs(R2train2-R2test2)*100>1:\n",
    "        d[5]+=1\n",
    "    else:\n",
    "        d[0]+=1\n",
    "    count1RF+=R2test2\n",
    "    count2RF+=R2train2\n",
    "    maxR2=max(maxR2,R2test2)\n",
    "    if R2test2>0.95:\n",
    "        plt.show()\n",
    "        print(\"------------------------------------------------------------\")\n",
    "        row = 1\n",
    "        col = 0\n",
    "        worksheet.write(row, 0, 'test number')\n",
    "        worksheet.write(row, 1, 'real value')\n",
    "        worksheet.write(row, 2, 'predicted value')\n",
    "        for num, real, pred in zip(test,y[test],y_pred2):\n",
    "            worksheet.write(row, col,     num)\n",
    "            worksheet.write(row, col + 1, real)\n",
    "            worksheet.write(row, col + 2, pred)\n",
    "            row += 1\n",
    "        worksheet.write(row, 0, 'Train R2')\n",
    "        worksheet.write(row, 1, R2train2)\n",
    "        worksheet.write(row+1, 0, 'Test R2')\n",
    "        worksheet.write(row+1, 1, R2test2)\n",
    "        mae = metrics.mean_absolute_error(y[test], y_pred2)\n",
    "        worksheet.write(row+2, 0, 'Mean absolute error')\n",
    "        worksheet.write(row+2, 1, mae)\n",
    "        \n",
    "        print(test)\n",
    "        print(\"predicted Value in test set with RandomForest:\")\n",
    "        print(y_pred2)\n",
    "        print(\"Test accuracy result in RandomForest:\",R2test2)\n",
    "        print('Mean Absolute Error in RandomForest:', mae)\n",
    "        print(\"Train accuracy result in RandomForest:\",R2train2)\n",
    "        print(\"------------------------------------------------------------\")\n",
    "print(maxR2)\n",
    "print(X[0,:])\n",
    "print(d)\n",
    "print(selector.feature_importances_)\n",
    "workbook.close()\n",
    "#print(selector.support_)\n",
    "#print(selector.ranking_)\n",
    "print(\"Test accuracy average in RandomForest:\",count1RF/100.0)\n",
    "print(\"Train accuracy average in RandomForest:\",count2RF/100.0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}