2 Deep Neural Network_06.2021.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "id": "LO10dGzVTVD7"
   },
   "source": [
    "# Metamodeling - DNN\n",
    "Objective: Develop and Optimize DNN to predict oil spill\n",
    "\n",
    "Outline of this script:\n",
    "1. Data Preparation\n",
    "2. Develop DNN model\n",
    "3. Fit and measure performance\n",
    "4. Apply Grid Search, Random Search, HyperOpt and TPOT\n",
    "5. Compare thier performance\n",
    "\n",
    "\n",
    "Input for this script: A dataset which include\n",
    "\n",
    "Output of this script:\n",
    "A few dataframe that contains performance measures (e.g. R^2 and RMSE) and design of the algorithm\n",
    "\n",
    "Date: May 01, 2021\n",
    "\n",
    "Written by: Tanmoy Das"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ea_hjCCMTVD8"
   },
   "source": [
    "# Data Preparation & Initial Modeling\n",
    "### Importing ADSAM dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "aKoOmcKWT3x9"
   },
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Algorithm related libraries\n",
    "# !pip install plotly\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "# metrics\n",
    "from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "JhR7x7vhzrmG"
   },
   "outputs": [],
   "source": [
    "# Import Training dataset  \n",
    "X_train_dev = pd.read_csv('Dataset for metamodeling/X_train_dev.csv', header=0).copy()\n",
    "X_val = pd.read_csv('Dataset for metamodeling/X_val.csv', header=0).copy()\n",
    "y_train_dev = pd.read_csv('Dataset for metamodeling/y_train_dev.csv', header=0).copy()\n",
    "y_val = pd.read_csv('Dataset for metamodeling/y_val.csv', header=0).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>$type_A$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.490008</td>\n",
       "      <td>0.334118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$type_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.491090</td>\n",
       "      <td>0.331719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$type_{oil}$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.492446</td>\n",
       "      <td>0.376486</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$velocity_A$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.498944</td>\n",
       "      <td>0.288640</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.247730</td>\n",
       "      <td>0.498735</td>\n",
       "      <td>0.749022</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$velocity_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.499480</td>\n",
       "      <td>0.288556</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.249848</td>\n",
       "      <td>0.499230</td>\n",
       "      <td>0.750018</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$angle_{collision}$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.500327</td>\n",
       "      <td>0.288654</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.250539</td>\n",
       "      <td>0.500202</td>\n",
       "      <td>0.749741</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$location_{impact.B}$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.398842</td>\n",
       "      <td>0.242892</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.197668</td>\n",
       "      <td>0.361345</td>\n",
       "      <td>0.575490</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$displacement_A$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.459782</td>\n",
       "      <td>0.325586</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.122403</td>\n",
       "      <td>0.457617</td>\n",
       "      <td>0.673070</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$displacement_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.405861</td>\n",
       "      <td>0.288774</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.219923</td>\n",
       "      <td>0.362168</td>\n",
       "      <td>0.543024</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$length_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.639653</td>\n",
       "      <td>0.318023</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.442238</td>\n",
       "      <td>0.748682</td>\n",
       "      <td>0.937961</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$width_B$</th>\n",
       "      <td>74880.0</td>\n",
       "      <td>0.589180</td>\n",
       "      <td>0.318618</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.592339</td>\n",
       "      <td>0.894425</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         count      mean       std  min       25%       50%  \\\n",
       "$type_A$               74880.0  0.490008  0.334118  0.0  0.166667  0.500000   \n",
       "$type_B$               74880.0  0.491090  0.331719  0.0  0.230769  0.538462   \n",
       "$type_{oil}$           74880.0  0.492446  0.376486  0.0  0.000000  0.333333   \n",
       "$velocity_A$           74880.0  0.498944  0.288640  0.0  0.247730  0.498735   \n",
       "$velocity_B$           74880.0  0.499480  0.288556  0.0  0.249848  0.499230   \n",
       "$angle_{collision}$    74880.0  0.500327  0.288654  0.0  0.250539  0.500202   \n",
       "$location_{impact.B}$  74880.0  0.398842  0.242892  0.0  0.197668  0.361345   \n",
       "$displacement_A$       74880.0  0.459782  0.325586  0.0  0.122403  0.457617   \n",
       "$displacement_B$       74880.0  0.405861  0.288774  0.0  0.219923  0.362168   \n",
       "$length_B$             74880.0  0.639653  0.318023  0.0  0.442238  0.748682   \n",
       "$width_B$              74880.0  0.589180  0.318618  0.0  0.421053  0.592339   \n",
       "\n",
       "                            75%  max  \n",
       "$type_A$               0.833333  1.0  \n",
       "$type_B$               0.769231  1.0  \n",
       "$type_{oil}$           1.000000  1.0  \n",
       "$velocity_A$           0.749022  1.0  \n",
       "$velocity_B$           0.750018  1.0  \n",
       "$angle_{collision}$    0.749741  1.0  \n",
       "$location_{impact.B}$  0.575490  1.0  \n",
       "$displacement_A$       0.673070  1.0  \n",
       "$displacement_B$       0.543024  1.0  \n",
       "$length_B$             0.937961  1.0  \n",
       "$width_B$              0.894425  1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# X_train_dev.info()\n",
    "display(X_train_dev.describe().T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WutmuTdlTVEH"
   },
   "source": [
    "### Initial Model\n",
    "Note: this performance is obtained without any hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5607,
     "status": "ok",
     "timestamp": 1612926268089,
     "user": {
      "displayName": "Tanmoy Das",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgMmxHcU_jnpEQnP9s8nrNXQJMgMWwxHelgPXC5=s64",
      "userId": "02309773046279713936"
     },
     "user_tz": 240
    },
    "id": "c8RLEAfZuPNg",
    "outputId": "3db9189b-2543-4d69-b219-05c3613de29e",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.10808269486506364"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# \n",
    "from sklearn.neural_network import MLPRegressor\n",
    "# from sklearn import metrics\n",
    "neural_net_wo_scl = MLPRegressor(hidden_layer_sizes=16, activation='logistic', solver='adam', alpha=0.01,\n",
    "                           batch_size='auto', learning_rate='adaptive',random_state=1, max_iter=20, \n",
    "                           learning_rate_init =.8, verbose= False)\n",
    "neural_net_wo_scl = neural_net_wo_scl.fit(X_train_dev, y_train_dev)\n",
    "neural_net_wo_scl.score(X_val, y_val) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p7rLCZYJTVEM"
   },
   "source": [
    "# Model Improvement by Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'activation': 'relu',\n",
       " 'alpha': 0.0001,\n",
       " 'batch_size': 'auto',\n",
       " 'beta_1': 0.9,\n",
       " 'beta_2': 0.999,\n",
       " 'early_stopping': False,\n",
       " 'epsilon': 1e-08,\n",
       " 'hidden_layer_sizes': (100,),\n",
       " 'learning_rate': 'constant',\n",
       " 'learning_rate_init': 0.001,\n",
       " 'max_fun': 15000,\n",
       " 'max_iter': 200,\n",
       " 'momentum': 0.9,\n",
       " 'n_iter_no_change': 10,\n",
       " 'nesterovs_momentum': True,\n",
       " 'power_t': 0.5,\n",
       " 'random_state': None,\n",
       " 'shuffle': True,\n",
       " 'solver': 'adam',\n",
       " 'tol': 0.0001,\n",
       " 'validation_fraction': 0.1,\n",
       " 'verbose': False,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MLPRegressor().get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PeJYXZEmTVEV"
   },
   "source": [
    "# Model Improvement by Random Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 4 (DNN part) & Figure 6: PCP "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup of the hyperparameters & Designing the DNN Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hidden layer sizes</th>\n",
       "      <th>Learning Rate</th>\n",
       "      <th>alpha</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0.1350</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.164258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.659051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>64.0</td>\n",
       "      <td>0.0775</td>\n",
       "      <td>0.000200</td>\n",
       "      <td>0.363508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>128.0</td>\n",
       "      <td>0.1925</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.271581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.700960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>16.0</td>\n",
       "      <td>0.1350</td>\n",
       "      <td>0.000200</td>\n",
       "      <td>0.293273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.421779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0775</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.255240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0.2500</td>\n",
       "      <td>0.000200</td>\n",
       "      <td>0.294387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.000175</td>\n",
       "      <td>0.358881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    hidden layer sizes  Learning Rate     alpha     Score\n",
       "0                 32.0         0.1350  0.000100  0.164258\n",
       "1                 16.0         0.0200  0.000100  0.659051\n",
       "2                 64.0         0.0775  0.000200  0.363508\n",
       "3                128.0         0.1925  0.000100  0.271581\n",
       "4                 32.0         0.0200  0.000100  0.700960\n",
       "..                 ...            ...       ...       ...\n",
       "95                16.0         0.1350  0.000200  0.293273\n",
       "96                 8.0         0.0200  0.000100  0.421779\n",
       "97                32.0         0.0775  0.000100  0.255240\n",
       "98                32.0         0.2500  0.000200  0.294387\n",
       "99                 2.0         0.0200  0.000175  0.358881\n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Performance from Random Search\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Best R^2</th>\n",
       "      <th>Number of hidden layers</th>\n",
       "      <th>Learning Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.814949</td>\n",
       "      <td>128.0</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Best R^2  Number of hidden layers  Learning Rate\n",
       "0  0.814949                    128.0           0.02"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Random Search for PCP\n",
    "start_random = time.time()\n",
    "# Set up the sample space for Random Search\n",
    "parameter_grid = { 'learning_rate_init' : np.linspace(0.020,.25,5),\n",
    "                    'hidden_layer_sizes' : [128, 64, 32, 16, 8, 2], # list(range(16, 32, 5)),\n",
    "                    'alpha' : np.linspace(0.0001,.0002,5) }      # range(start, stop, step) #(start=1,stop=1000, num=100))\n",
    "# Define how many samples\n",
    "number_models = 100\n",
    "\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "rand_search_dnn_model = RandomizedSearchCV(estimator=MLPRegressor(), # check with empty model e.g. SVR()\n",
    "                            param_distributions=parameter_grid,\n",
    "                            n_iter = number_models,\n",
    "                            scoring='r2', n_jobs=30,\n",
    "                            cv = 2,            refit=True,\n",
    "                            return_train_score = True)\n",
    "# Add alpha as hyperparatemeter later??\n",
    "rand_search_dnn_model.fit(X_train_dev, y_train_dev)\n",
    "\n",
    "# Calculate scores on validation training set \n",
    "learning_rate_init = [item['learning_rate_init'] for item in rand_search_dnn_model.cv_results_['params']]\n",
    "hidden_layer_sizes = [item['hidden_layer_sizes'] for item in rand_search_dnn_model.cv_results_['params']]\n",
    "alpha = [item['alpha'] for item in rand_search_dnn_model.cv_results_['params']]\n",
    "scores = list(rand_search_dnn_model.cv_results_['mean_test_score'])\n",
    "df_rands_dnn_perfm = pd.DataFrame([hidden_layer_sizes, learning_rate_init, alpha, scores]).T\n",
    "df_rands_dnn_perfm.columns = ['hidden layer sizes','Learning Rate', 'alpha', 'Score']\n",
    "# df_rands_dnn_perfm.groupby(['Learning Rate']).mean()\n",
    "display(df_rands_dnn_perfm)\n",
    "\n",
    "# Determine the best performance and associate hyperparameters\n",
    "df_rands_best = pd.DataFrame([rand_search_dnn_model.best_score_, rand_search_dnn_model.best_params_['hidden_layer_sizes'],\n",
    "                              rand_search_dnn_model.best_params_['learning_rate_init']]).T\n",
    "print('Best Performance from Random Search')\n",
    "df_rands_best.columns = ['Best R^2', 'Number of hidden layers', 'Learning Rate']\n",
    "display(df_rands_best)\n",
    "\n",
    "#runtime_random_search = time.time() - start_random\n",
    "#display(runtime_random_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rand_search_dnn_model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-40f0676e88a4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Save the Random Search model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpickle\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrand_search_dnn_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Models 05.2021/rand_search_dnn_model.sav'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#  Random Search\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mdf_rands_dnn_perfm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Models 05.2021/data_rands_dnn_perfm_PCP.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'rand_search_dnn_model' is not defined"
     ]
    }
   ],
   "source": [
    "# Save the Random Search model\n",
    "import pickle\n",
    "pickle.dump(rand_search_dnn_model, open('Models 05.2021/rand_search_dnn_model.sav', 'wb')) #  Random Search\n",
    "df_rands_dnn_perfm.to_csv('Models 05.2021/data_rands_dnn_perfm_PCP.csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rands_dnn_perfm = pd.read_csv('Models 05.2021/data_rands_dnn_perfm_PCP.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parallel Coordinat Plot during Random Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find a version that satisfies the requirement plotly-orca (from versions: none)\n",
      "ERROR: No matching distribution found for plotly-orca\n"
     ]
    }
   ],
   "source": [
    "!pip install plotly-orca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "dimensions": [
          {
           "label": "No. of Neurons",
           "values": [
            32,
            16,
            64,
            128,
            32,
            16,
            32,
            32,
            16,
            8,
            8,
            32,
            64,
            128,
            16,
            8,
            2,
            32,
            16,
            2,
            8,
            64,
            2,
            64,
            128,
            32,
            32,
            2,
            16,
            2,
            128,
            128,
            16,
            8,
            8,
            16,
            64,
            8,
            2,
            64,
            8,
            2,
            8,
            32,
            64,
            2,
            32,
            128,
            8,
            16,
            16,
            16,
            128,
            8,
            2,
            128,
            32,
            64,
            64,
            32,
            64,
            128,
            64,
            128,
            2,
            8,
            128,
            16,
            128,
            8,
            32,
            128,
            16,
            32,
            64,
            2,
            128,
            16,
            64,
            128,
            64,
            2,
            64,
            128,
            32,
            16,
            32,
            64,
            8,
            128,
            2,
            2,
            32,
            16,
            128,
            16,
            8,
            32,
            32,
            2
           ]
          },
          {
           "label": "Learning Rate",
           "values": [
            0.135,
            0.02,
            0.0775,
            0.1925,
            0.02,
            0.25,
            0.25,
            0.1925,
            0.1925,
            0.135,
            0.135,
            0.135,
            0.02,
            0.0775,
            0.0775,
            0.0775,
            0.25,
            0.25,
            0.135,
            0.0775,
            0.02,
            0.1925,
            0.1925,
            0.1925,
            0.1925,
            0.02,
            0.0775,
            0.1925,
            0.02,
            0.1925,
            0.135,
            0.0775,
            0.02,
            0.1925,
            0.0775,
            0.1925,
            0.0775,
            0.0775,
            0.1925,
            0.25,
            0.02,
            0.135,
            0.135,
            0.1925,
            0.135,
            0.135,
            0.0775,
            0.0775,
            0.25,
            0.135,
            0.0775,
            0.1925,
            0.02,
            0.135,
            0.02,
            0.02,
            0.25,
            0.135,
            0.135,
            0.02,
            0.0775,
            0.1925,
            0.1925,
            0.25,
            0.1925,
            0.1925,
            0.25,
            0.02,
            0.135,
            0.1925,
            0.1925,
            0.135,
            0.02,
            0.1925,
            0.02,
            0.25,
            0.0775,
            0.25,
            0.0775,
            0.135,
            0.25,
            0.25,
            0.25,
            0.135,
            0.1925,
            0.1925,
            0.25,
            0.02,
            0.25,
            0.1925,
            0.25,
            0.0775,
            0.02,
            0.0775,
            0.1925,
            0.135,
            0.02,
            0.0775,
            0.25,
            0.02
           ]
          },
          {
           "label": "Alpha",
           "values": [
            0.0001,
            0.0001,
            0.0002,
            0.0001,
            0.0001,
            0.00017500000000000005,
            0.00017500000000000005,
            0.000125,
            0.0001,
            0.0002,
            0.0001,
            0.00017500000000000005,
            0.00015,
            0.00015,
            0.0002,
            0.00017500000000000005,
            0.00015,
            0.000125,
            0.0001,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0002,
            0.0002,
            0.0001,
            0.0002,
            0.00017500000000000005,
            0.00015,
            0.000125,
            0.00015,
            0.00017500000000000005,
            0.000125,
            0.000125,
            0.00017500000000000005,
            0.00017500000000000005,
            0.000125,
            0.0002,
            0.0001,
            0.00015,
            0.0001,
            0.00015,
            0.00015,
            0.000125,
            0.000125,
            0.00015,
            0.00015,
            0.0001,
            0.000125,
            0.0001,
            0.0002,
            0.00017500000000000005,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0002,
            0.00017500000000000005,
            0.0001,
            0.00017500000000000005,
            0.0001,
            0.000125,
            0.0002,
            0.00015,
            0.000125,
            0.00015,
            0.000125,
            0.00017500000000000005,
            0.00015,
            0.0001,
            0.000125,
            0.000125,
            0.00015,
            0.0002,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0002,
            0.0002,
            0.00017500000000000005,
            0.0002,
            0.0002,
            0.00015,
            0.00015,
            0.0002,
            0.0001,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0001,
            0.0001,
            0.00015,
            0.00015,
            0.0002,
            0.0001,
            0.00017500000000000005,
            0.0001,
            0.000125,
            0.0002,
            0.0001,
            0.000125,
            0.0002,
            0.0001,
            0.0001,
            0.0002,
            0.00017500000000000005
           ]
          }
         ],
         "line": {
          "color": [
           0.16425847936115864,
           0.6590510448620013,
           0.3635079022018521,
           0.2715805970373783,
           0.7009596579533608,
           0.1518967867708017,
           0.2718402123666769,
           0.3162065806371457,
           0.1513976913793107,
           -0.0376566128558354,
           -0.011151527354621072,
           0.23573330982557605,
           0.7149813348130349,
           0.3909634982679493,
           0.162041779944567,
           0.14419220842421945,
           -0.007135854873072953,
           0.2627740264572056,
           -0.016273611713388602,
           0.15445145481339945,
           0.44499644609542255,
           0.28674486065947985,
           -0.017631666437996518,
           0.240333815431798,
           0.16501905455175092,
           0.7363210332116172,
           0.3693614458642888,
           -0.003745591685682258,
           0.5810564624935384,
           -0.026484521381059327,
           0.2499970731291053,
           0.4292221538942409,
           0.5637777536760377,
           0.2994371737131243,
           0.15646738703277116,
           0.11158887625654354,
           0.16016596088030066,
           0.36633428991608064,
           -0.026784257394702613,
           0.1012928349050868,
           0.5331279487625565,
           -0.008454077137812401,
           0.037634496953806516,
           0.35483172095984994,
           0.3533387299477633,
           -0.02483765917624372,
           0.3228674268538321,
           0.2971522834573239,
           -0.06762116369509633,
           -0.022035479935045968,
           0.13738014414319372,
           0.1320698738993187,
           0.8149492474896601,
           -0.04407530202629019,
           0.325011269016259,
           0.8128699148153082,
           0.0370524868942005,
           0.306738287711812,
           0.30407634200715816,
           0.7004900053934181,
           0.3916980587343845,
           0.2701439454453733,
           0.2213760865464786,
           0.1631650594058805,
           -0.011623645210101159,
           0.07343848904453583,
           0.2142598119833044,
           0.596191266013413,
           0.10303507539716927,
           -0.033038441531594645,
           0.2877555323235481,
           0.431240858146438,
           0.6336531983258568,
           0.11321303978713712,
           0.7560661409235161,
           -0.051025041204091376,
           0.369112376352861,
           -0.03190108059697645,
           0.24121383310498434,
           0.3677601011602805,
           0.2721907871247251,
           -0.029841807055732045,
           0.3092651919988009,
           0.3782087275542209,
           -0.028389940145782513,
           0.11757567554321295,
           0.1463037585364388,
           0.7822660071088914,
           -0.011853543080247808,
           0.2616846058538931,
           -0.10690527855078548,
           0.10871166644170513,
           0.7473839335907957,
           0.3744524718542495,
           0.1864507614355549,
           0.2932733423365895,
           0.4217789336888689,
           0.25523992108032084,
           0.2943866212647684,
           0.3588806269004384
          ],
          "showscale": true
         },
         "type": "parcoords"
        }
       ],
       "layout": {
        "autosize": true,
        "height": 500,
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 500,
        "yaxis": {
         "tickformat": ".02%"
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"ecf3bde0-4273-41e9-b34d-050c82073c89\" class=\"plotly-graph-div\" style=\"height:500px; width:500px;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ecf3bde0-4273-41e9-b34d-050c82073c89\")) {                    Plotly.newPlot(                        \"ecf3bde0-4273-41e9-b34d-050c82073c89\",                        [{\"dimensions\": [{\"label\": \"No. of Neurons\", \"values\": [32.0, 16.0, 64.0, 128.0, 32.0, 16.0, 32.0, 32.0, 16.0, 8.0, 8.0, 32.0, 64.0, 128.0, 16.0, 8.0, 2.0, 32.0, 16.0, 2.0, 8.0, 64.0, 2.0, 64.0, 128.0, 32.0, 32.0, 2.0, 16.0, 2.0, 128.0, 128.0, 16.0, 8.0, 8.0, 16.0, 64.0, 8.0, 2.0, 64.0, 8.0, 2.0, 8.0, 32.0, 64.0, 2.0, 32.0, 128.0, 8.0, 16.0, 16.0, 16.0, 128.0, 8.0, 2.0, 128.0, 32.0, 64.0, 64.0, 32.0, 64.0, 128.0, 64.0, 128.0, 2.0, 8.0, 128.0, 16.0, 128.0, 8.0, 32.0, 128.0, 16.0, 32.0, 64.0, 2.0, 128.0, 16.0, 64.0, 128.0, 64.0, 2.0, 64.0, 128.0, 32.0, 16.0, 32.0, 64.0, 8.0, 128.0, 2.0, 2.0, 32.0, 16.0, 128.0, 16.0, 8.0, 32.0, 32.0, 2.0]}, {\"label\": \"Learning Rate\", \"values\": [0.135, 0.02, 0.0775, 0.1925, 0.02, 0.25, 0.25, 0.1925, 0.1925, 0.135, 0.135, 0.135, 0.02, 0.0775, 0.0775, 0.0775, 0.25, 0.25, 0.135, 0.0775, 0.02, 0.1925, 0.1925, 0.1925, 0.1925, 0.02, 0.0775, 0.1925, 0.02, 0.1925, 0.135, 0.0775, 0.02, 0.1925, 0.0775, 0.1925, 0.0775, 0.0775, 0.1925, 0.25, 0.02, 0.135, 0.135, 0.1925, 0.135, 0.135, 0.0775, 0.0775, 0.25, 0.135, 0.0775, 0.1925, 0.02, 0.135, 0.02, 0.02, 0.25, 0.135, 0.135, 0.02, 0.0775, 0.1925, 0.1925, 0.25, 0.1925, 0.1925, 0.25, 0.02, 0.135, 0.1925, 0.1925, 0.135, 0.02, 0.1925, 0.02, 0.25, 0.0775, 0.25, 0.0775, 0.135, 0.25, 0.25, 0.25, 0.135, 0.1925, 0.1925, 0.25, 0.02, 0.25, 0.1925, 0.25, 0.0775, 0.02, 0.0775, 0.1925, 0.135, 0.02, 0.0775, 0.25, 0.02]}, {\"label\": \"Alpha\", \"values\": [0.0001, 0.0001, 0.0002, 0.0001, 0.0001, 0.00017500000000000005, 0.00017500000000000005, 0.000125, 0.0001, 0.0002, 0.0001, 0.00017500000000000005, 0.00015, 0.00015, 0.0002, 0.00017500000000000005, 0.00015, 0.000125, 0.0001, 0.00017500000000000005, 0.00017500000000000005, 0.0002, 0.0002, 0.0001, 0.0002, 0.00017500000000000005, 0.00015, 0.000125, 0.00015, 0.00017500000000000005, 0.000125, 0.000125, 0.00017500000000000005, 0.00017500000000000005, 0.000125, 0.0002, 0.0001, 0.00015, 0.0001, 0.00015, 0.00015, 0.000125, 0.000125, 0.00015, 0.00015, 0.0001, 0.000125, 0.0001, 0.0002, 0.00017500000000000005, 0.00017500000000000005, 0.00017500000000000005, 0.0002, 0.00017500000000000005, 0.0001, 0.00017500000000000005, 0.0001, 0.000125, 0.0002, 0.00015, 0.000125, 0.00015, 0.000125, 0.00017500000000000005, 0.00015, 0.0001, 0.000125, 0.000125, 0.00015, 0.0002, 0.00017500000000000005, 0.00017500000000000005, 0.0002, 0.0002, 0.00017500000000000005, 0.0002, 0.0002, 0.00015, 0.00015, 0.0002, 0.0001, 0.00017500000000000005, 0.00017500000000000005, 0.0001, 0.0001, 0.00015, 0.00015, 0.0002, 0.0001, 0.00017500000000000005, 0.0001, 0.000125, 0.0002, 0.0001, 0.000125, 0.0002, 0.0001, 0.0001, 0.0002, 0.00017500000000000005]}], \"line\": {\"color\": [0.16425847936115864, 0.6590510448620013, 0.3635079022018521, 0.2715805970373783, 0.7009596579533608, 0.1518967867708017, 0.2718402123666769, 0.3162065806371457, 0.1513976913793107, -0.0376566128558354, -0.011151527354621072, 0.23573330982557605, 0.7149813348130349, 0.3909634982679493, 0.162041779944567, 0.14419220842421945, -0.007135854873072953, 0.2627740264572056, -0.016273611713388602, 0.15445145481339945, 0.44499644609542255, 0.28674486065947985, -0.017631666437996518, 0.240333815431798, 0.16501905455175092, 0.7363210332116172, 0.3693614458642888, -0.003745591685682258, 0.5810564624935384, -0.026484521381059327, 0.2499970731291053, 0.4292221538942409, 0.5637777536760377, 0.2994371737131243, 0.15646738703277116, 0.11158887625654354, 0.16016596088030066, 0.36633428991608064, -0.026784257394702613, 0.1012928349050868, 0.5331279487625565, -0.008454077137812401, 0.037634496953806516, 0.35483172095984994, 0.3533387299477633, -0.02483765917624372, 0.3228674268538321, 0.2971522834573239, -0.06762116369509633, -0.022035479935045968, 0.13738014414319372, 0.1320698738993187, 0.8149492474896601, -0.04407530202629019, 0.325011269016259, 0.8128699148153082, 0.0370524868942005, 0.306738287711812, 0.30407634200715816, 0.7004900053934181, 0.3916980587343845, 0.2701439454453733, 0.2213760865464786, 0.1631650594058805, -0.011623645210101159, 0.07343848904453583, 0.2142598119833044, 0.596191266013413, 0.10303507539716927, -0.033038441531594645, 0.2877555323235481, 0.431240858146438, 0.6336531983258568, 0.11321303978713712, 0.7560661409235161, -0.051025041204091376, 0.369112376352861, -0.03190108059697645, 0.24121383310498434, 0.3677601011602805, 0.2721907871247251, -0.029841807055732045, 0.3092651919988009, 0.3782087275542209, -0.028389940145782513, 0.11757567554321295, 0.1463037585364388, 0.7822660071088914, -0.011853543080247808, 0.2616846058538931, -0.10690527855078548, 0.10871166644170513, 0.7473839335907957, 0.3744524718542495, 0.1864507614355549, 0.2932733423365895, 0.4217789336888689, 0.25523992108032084, 0.2943866212647684, 0.3588806269004384], \"showscale\": true}, \"type\": \"parcoords\"}],                        {\"autosize\": true, \"height\": 500, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"autotypenumbers\": \"strict\", \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"width\": 500, \"yaxis\": {\"tickformat\": \".02%\"}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('ecf3bde0-4273-41e9-b34d-050c82073c89');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# !pip install plotly\n",
    "# 33 \n",
    "# Parallel Coordinate Plot\n",
    "# https://plotly.com/python/parallel-coordinates-plot/\n",
    "#plt.clf()\n",
    "import plotly.graph_objects as go\n",
    "import os\n",
    "\n",
    "fig = plt.figure()\n",
    "fig = go.Figure(data=\n",
    "    go.Parcoords(#abelfont(size=[12]),\n",
    "        line = dict(color = df_rands_dnn_perfm['Score'], showscale=True), # showscale is for colorbar\n",
    "        dimensions = list([\n",
    "            dict(    #    constraintrange = [4,8],\n",
    "                label = 'No. of Neurons', values = df_rands_dnn_perfm['hidden layer sizes']),\n",
    "            dict(#range = [0.01,2],\n",
    "                label = 'Learning Rate', values = df_rands_dnn_perfm['Learning Rate']),\n",
    "            dict(# range = [0,8],\n",
    "                label = 'Alpha', values = df_rands_dnn_perfm['alpha'])\n",
    "        ]),\n",
    "        #parallelaxes(fontsize=14)\n",
    "    ) #colorbar\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    autosize=True,\n",
    "    width=500,\n",
    "    height=500)\n",
    " #   yaxis=dict(tickformat=\".02%\"))\n",
    "\n",
    "#go.Layout()\n",
    "#fig.write_image('Fig6.jpeg')\n",
    "\n",
    "#img_bytes = fig.to_image(format=\"jpeg\", width=600, height=350, scale=2)\n",
    "#Image(img_bytes)\n",
    "# fig.to_image(format=\"png\")\n",
    "\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "dimensions": [
          {
           "label": "No. of Neurons",
           "values": [
            32,
            16,
            64,
            128,
            32,
            16,
            32,
            32,
            16,
            8,
            8,
            32,
            64,
            128,
            16,
            8,
            2,
            32,
            16,
            2,
            8,
            64,
            2,
            64,
            128,
            32,
            32,
            2,
            16,
            2,
            128,
            128,
            16,
            8,
            8,
            16,
            64,
            8,
            2,
            64,
            8,
            2,
            8,
            32,
            64,
            2,
            32,
            128,
            8,
            16,
            16,
            16,
            128,
            8,
            2,
            128,
            32,
            64,
            64,
            32,
            64,
            128,
            64,
            128,
            2,
            8,
            128,
            16,
            128,
            8,
            32,
            128,
            16,
            32,
            64,
            2,
            128,
            16,
            64,
            128,
            64,
            2,
            64,
            128,
            32,
            16,
            32,
            64,
            8,
            128,
            2,
            2,
            32,
            16,
            128,
            16,
            8,
            32,
            32,
            2
           ]
          },
          {
           "label": "Learning Rate",
           "values": [
            0.135,
            0.02,
            0.0775,
            0.1925,
            0.02,
            0.25,
            0.25,
            0.1925,
            0.1925,
            0.135,
            0.135,
            0.135,
            0.02,
            0.0775,
            0.0775,
            0.0775,
            0.25,
            0.25,
            0.135,
            0.0775,
            0.02,
            0.1925,
            0.1925,
            0.1925,
            0.1925,
            0.02,
            0.0775,
            0.1925,
            0.02,
            0.1925,
            0.135,
            0.0775,
            0.02,
            0.1925,
            0.0775,
            0.1925,
            0.0775,
            0.0775,
            0.1925,
            0.25,
            0.02,
            0.135,
            0.135,
            0.1925,
            0.135,
            0.135,
            0.0775,
            0.0775,
            0.25,
            0.135,
            0.0775,
            0.1925,
            0.02,
            0.135,
            0.02,
            0.02,
            0.25,
            0.135,
            0.135,
            0.02,
            0.0775,
            0.1925,
            0.1925,
            0.25,
            0.1925,
            0.1925,
            0.25,
            0.02,
            0.135,
            0.1925,
            0.1925,
            0.135,
            0.02,
            0.1925,
            0.02,
            0.25,
            0.0775,
            0.25,
            0.0775,
            0.135,
            0.25,
            0.25,
            0.25,
            0.135,
            0.1925,
            0.1925,
            0.25,
            0.02,
            0.25,
            0.1925,
            0.25,
            0.0775,
            0.02,
            0.0775,
            0.1925,
            0.135,
            0.02,
            0.0775,
            0.25,
            0.02
           ]
          },
          {
           "label": "Alpha",
           "values": [
            0.0001,
            0.0001,
            0.0002,
            0.0001,
            0.0001,
            0.00017500000000000005,
            0.00017500000000000005,
            0.000125,
            0.0001,
            0.0002,
            0.0001,
            0.00017500000000000005,
            0.00015,
            0.00015,
            0.0002,
            0.00017500000000000005,
            0.00015,
            0.000125,
            0.0001,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0002,
            0.0002,
            0.0001,
            0.0002,
            0.00017500000000000005,
            0.00015,
            0.000125,
            0.00015,
            0.00017500000000000005,
            0.000125,
            0.000125,
            0.00017500000000000005,
            0.00017500000000000005,
            0.000125,
            0.0002,
            0.0001,
            0.00015,
            0.0001,
            0.00015,
            0.00015,
            0.000125,
            0.000125,
            0.00015,
            0.00015,
            0.0001,
            0.000125,
            0.0001,
            0.0002,
            0.00017500000000000005,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0002,
            0.00017500000000000005,
            0.0001,
            0.00017500000000000005,
            0.0001,
            0.000125,
            0.0002,
            0.00015,
            0.000125,
            0.00015,
            0.000125,
            0.00017500000000000005,
            0.00015,
            0.0001,
            0.000125,
            0.000125,
            0.00015,
            0.0002,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0002,
            0.0002,
            0.00017500000000000005,
            0.0002,
            0.0002,
            0.00015,
            0.00015,
            0.0002,
            0.0001,
            0.00017500000000000005,
            0.00017500000000000005,
            0.0001,
            0.0001,
            0.00015,
            0.00015,
            0.0002,
            0.0001,
            0.00017500000000000005,
            0.0001,
            0.000125,
            0.0002,
            0.0001,
            0.000125,
            0.0002,
            0.0001,
            0.0001,
            0.0002,
            0.00017500000000000005
           ]
          }
         ],
         "line": {
          "color": [
           0.16425847936115864,
           0.6590510448620013,
           0.3635079022018521,
           0.2715805970373783,
           0.7009596579533608,
           0.1518967867708017,
           0.2718402123666769,
           0.3162065806371457,
           0.1513976913793107,
           -0.0376566128558354,
           -0.011151527354621072,
           0.23573330982557605,
           0.7149813348130349,
           0.3909634982679493,
           0.162041779944567,
           0.14419220842421945,
           -0.007135854873072953,
           0.2627740264572056,
           -0.016273611713388602,
           0.15445145481339945,
           0.44499644609542255,
           0.28674486065947985,
           -0.017631666437996518,
           0.240333815431798,
           0.16501905455175092,
           0.7363210332116172,
           0.3693614458642888,
           -0.003745591685682258,
           0.5810564624935384,
           -0.026484521381059327,
           0.2499970731291053,
           0.4292221538942409,
           0.5637777536760377,
           0.2994371737131243,
           0.15646738703277116,
           0.11158887625654354,
           0.16016596088030066,
           0.36633428991608064,
           -0.026784257394702613,
           0.1012928349050868,
           0.5331279487625565,
           -0.008454077137812401,
           0.037634496953806516,
           0.35483172095984994,
           0.3533387299477633,
           -0.02483765917624372,
           0.3228674268538321,
           0.2971522834573239,
           -0.06762116369509633,
           -0.022035479935045968,
           0.13738014414319372,
           0.1320698738993187,
           0.8149492474896601,
           -0.04407530202629019,
           0.325011269016259,
           0.8128699148153082,
           0.0370524868942005,
           0.306738287711812,
           0.30407634200715816,
           0.7004900053934181,
           0.3916980587343845,
           0.2701439454453733,
           0.2213760865464786,
           0.1631650594058805,
           -0.011623645210101159,
           0.07343848904453583,
           0.2142598119833044,
           0.596191266013413,
           0.10303507539716927,
           -0.033038441531594645,
           0.2877555323235481,
           0.431240858146438,
           0.6336531983258568,
           0.11321303978713712,
           0.7560661409235161,
           -0.051025041204091376,
           0.369112376352861,
           -0.03190108059697645,
           0.24121383310498434,
           0.3677601011602805,
           0.2721907871247251,
           -0.029841807055732045,
           0.3092651919988009,
           0.3782087275542209,
           -0.028389940145782513,
           0.11757567554321295,
           0.1463037585364388,
           0.7822660071088914,
           -0.011853543080247808,
           0.2616846058538931,
           -0.10690527855078548,
           0.10871166644170513,
           0.7473839335907957,
           0.3744524718542495,
           0.1864507614355549,
           0.2932733423365895,
           0.4217789336888689,
           0.25523992108032084,
           0.2943866212647684,
           0.3588806269004384
          ],
          "showscale": true
         },
         "tickfont": {
          "size": 10
         },
         "type": "parcoords"
        }
       ],
       "layout": {
        "autosize": true,
        "font": {
         "color": "Black",
         "family": "Times New Roman",
         "size": 25
        },
        "height": 500,
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 500
       }
      },
      "text/html": [
       "<div>                            <div id=\"155646b5-0439-4554-aa63-322c05fff609\" class=\"plotly-graph-div\" style=\"height:500px; width:500px;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"155646b5-0439-4554-aa63-322c05fff609\")) {                    Plotly.newPlot(                        \"155646b5-0439-4554-aa63-322c05fff609\",                        [{\"dimensions\": [{\"label\": \"No. of Neurons\", \"values\": [32.0, 16.0, 64.0, 128.0, 32.0, 16.0, 32.0, 32.0, 16.0, 8.0, 8.0, 32.0, 64.0, 128.0, 16.0, 8.0, 2.0, 32.0, 16.0, 2.0, 8.0, 64.0, 2.0, 64.0, 128.0, 32.0, 32.0, 2.0, 16.0, 2.0, 128.0, 128.0, 16.0, 8.0, 8.0, 16.0, 64.0, 8.0, 2.0, 64.0, 8.0, 2.0, 8.0, 32.0, 64.0, 2.0, 32.0, 128.0, 8.0, 16.0, 16.0, 16.0, 128.0, 8.0, 2.0, 128.0, 32.0, 64.0, 64.0, 32.0, 64.0, 128.0, 64.0, 128.0, 2.0, 8.0, 128.0, 16.0, 128.0, 8.0, 32.0, 128.0, 16.0, 32.0, 64.0, 2.0, 128.0, 16.0, 64.0, 128.0, 64.0, 2.0, 64.0, 128.0, 32.0, 16.0, 32.0, 64.0, 8.0, 128.0, 2.0, 2.0, 32.0, 16.0, 128.0, 16.0, 8.0, 32.0, 32.0, 2.0]}, {\"label\": \"Learning Rate\", \"values\": [0.135, 0.02, 0.0775, 0.1925, 0.02, 0.25, 0.25, 0.1925, 0.1925, 0.135, 0.135, 0.135, 0.02, 0.0775, 0.0775, 0.0775, 0.25, 0.25, 0.135, 0.0775, 0.02, 0.1925, 0.1925, 0.1925, 0.1925, 0.02, 0.0775, 0.1925, 0.02, 0.1925, 0.135, 0.0775, 0.02, 0.1925, 0.0775, 0.1925, 0.0775, 0.0775, 0.1925, 0.25, 0.02, 0.135, 0.135, 0.1925, 0.135, 0.135, 0.0775, 0.0775, 0.25, 0.135, 0.0775, 0.1925, 0.02, 0.135, 0.02, 0.02, 0.25, 0.135, 0.135, 0.02, 0.0775, 0.1925, 0.1925, 0.25, 0.1925, 0.1925, 0.25, 0.02, 0.135, 0.1925, 0.1925, 0.135, 0.02, 0.1925, 0.02, 0.25, 0.0775, 0.25, 0.0775, 0.135, 0.25, 0.25, 0.25, 0.135, 0.1925, 0.1925, 0.25, 0.02, 0.25, 0.1925, 0.25, 0.0775, 0.02, 0.0775, 0.1925, 0.135, 0.02, 0.0775, 0.25, 0.02]}, {\"label\": \"Alpha\", \"values\": [0.0001, 0.0001, 0.0002, 0.0001, 0.0001, 0.00017500000000000005, 0.00017500000000000005, 0.000125, 0.0001, 0.0002, 0.0001, 0.00017500000000000005, 0.00015, 0.00015, 0.0002, 0.00017500000000000005, 0.00015, 0.000125, 0.0001, 0.00017500000000000005, 0.00017500000000000005, 0.0002, 0.0002, 0.0001, 0.0002, 0.00017500000000000005, 0.00015, 0.000125, 0.00015, 0.00017500000000000005, 0.000125, 0.000125, 0.00017500000000000005, 0.00017500000000000005, 0.000125, 0.0002, 0.0001, 0.00015, 0.0001, 0.00015, 0.00015, 0.000125, 0.000125, 0.00015, 0.00015, 0.0001, 0.000125, 0.0001, 0.0002, 0.00017500000000000005, 0.00017500000000000005, 0.00017500000000000005, 0.0002, 0.00017500000000000005, 0.0001, 0.00017500000000000005, 0.0001, 0.000125, 0.0002, 0.00015, 0.000125, 0.00015, 0.000125, 0.00017500000000000005, 0.00015, 0.0001, 0.000125, 0.000125, 0.00015, 0.0002, 0.00017500000000000005, 0.00017500000000000005, 0.0002, 0.0002, 0.00017500000000000005, 0.0002, 0.0002, 0.00015, 0.00015, 0.0002, 0.0001, 0.00017500000000000005, 0.00017500000000000005, 0.0001, 0.0001, 0.00015, 0.00015, 0.0002, 0.0001, 0.00017500000000000005, 0.0001, 0.000125, 0.0002, 0.0001, 0.000125, 0.0002, 0.0001, 0.0001, 0.0002, 0.00017500000000000005]}], \"line\": {\"color\": [0.16425847936115864, 0.6590510448620013, 0.3635079022018521, 0.2715805970373783, 0.7009596579533608, 0.1518967867708017, 0.2718402123666769, 0.3162065806371457, 0.1513976913793107, -0.0376566128558354, -0.011151527354621072, 0.23573330982557605, 0.7149813348130349, 0.3909634982679493, 0.162041779944567, 0.14419220842421945, -0.007135854873072953, 0.2627740264572056, -0.016273611713388602, 0.15445145481339945, 0.44499644609542255, 0.28674486065947985, -0.017631666437996518, 0.240333815431798, 0.16501905455175092, 0.7363210332116172, 0.3693614458642888, -0.003745591685682258, 0.5810564624935384, -0.026484521381059327, 0.2499970731291053, 0.4292221538942409, 0.5637777536760377, 0.2994371737131243, 0.15646738703277116, 0.11158887625654354, 0.16016596088030066, 0.36633428991608064, -0.026784257394702613, 0.1012928349050868, 0.5331279487625565, -0.008454077137812401, 0.037634496953806516, 0.35483172095984994, 0.3533387299477633, -0.02483765917624372, 0.3228674268538321, 0.2971522834573239, -0.06762116369509633, -0.022035479935045968, 0.13738014414319372, 0.1320698738993187, 0.8149492474896601, -0.04407530202629019, 0.325011269016259, 0.8128699148153082, 0.0370524868942005, 0.306738287711812, 0.30407634200715816, 0.7004900053934181, 0.3916980587343845, 0.2701439454453733, 0.2213760865464786, 0.1631650594058805, -0.011623645210101159, 0.07343848904453583, 0.2142598119833044, 0.596191266013413, 0.10303507539716927, -0.033038441531594645, 0.2877555323235481, 0.431240858146438, 0.6336531983258568, 0.11321303978713712, 0.7560661409235161, -0.051025041204091376, 0.369112376352861, -0.03190108059697645, 0.24121383310498434, 0.3677601011602805, 0.2721907871247251, -0.029841807055732045, 0.3092651919988009, 0.3782087275542209, -0.028389940145782513, 0.11757567554321295, 0.1463037585364388, 0.7822660071088914, -0.011853543080247808, 0.2616846058538931, -0.10690527855078548, 0.10871166644170513, 0.7473839335907957, 0.3744524718542495, 0.1864507614355549, 0.2932733423365895, 0.4217789336888689, 0.25523992108032084, 0.2943866212647684, 0.3588806269004384], \"showscale\": true}, \"tickfont\": {\"size\": 10}, \"type\": \"parcoords\"}],                        {\"autosize\": true, \"font\": {\"color\": \"Black\", \"family\": \"Times New Roman\", \"size\": 25}, \"height\": 500, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"autotypenumbers\": \"strict\", \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"width\": 500},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('155646b5-0439-4554-aa63-322c05fff609');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "fig.update_traces(tickfont_size=20)\n",
    "fig.update_layout(\n",
    "    font=dict(\n",
    "        family=\"Times New Roman\",\n",
    "        size=25,\n",
    "        color=\"Black\" \n",
    "    )\n",
    ")\n",
    "plt.savefig('Figures 06.2021/Figure6_PCP.png', dpi=600)\n",
    "fig.show()\n",
    "\n",
    "# Documentation: https://plotly.github.io/plotly.py-docs/generated/plotly.graph_objects.Parcoords.html\n",
    "# https://plotly.com/python/reference/parcoords/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hidden layer sizes</th>\n",
       "      <th>Learning Rate</th>\n",
       "      <th>alpha</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(32, 32)</td>\n",
       "      <td>0.1925</td>\n",
       "      <td>0.00015</td>\n",
       "      <td>-0.0813055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(32, 16)</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.0002</td>\n",
       "      <td>-0.0149412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(32, 32, 32)</td>\n",
       "      <td>0.0775</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.3767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(32, 32)</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.000175</td>\n",
       "      <td>0.917569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.799876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>(32, 32, 32)</td>\n",
       "      <td>0.0775</td>\n",
       "      <td>0.00015</td>\n",
       "      <td>0.740451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>(32, 32)</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.347403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>(32, 32, 32)</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.000125</td>\n",
       "      <td>0.35823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>(32, 16)</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.000175</td>\n",
       "      <td>0.846108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>(32, 32, 32)</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.000175</td>\n",
       "      <td>0.0997675</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   hidden layer sizes Learning Rate     alpha      Score\n",
       "0            (32, 32)        0.1925   0.00015 -0.0813055\n",
       "1            (32, 16)         0.135    0.0002 -0.0149412\n",
       "2        (32, 32, 32)        0.0775    0.0001     0.3767\n",
       "3            (32, 32)          0.02  0.000175   0.917569\n",
       "4                  32          0.02    0.0001   0.799876\n",
       "..                ...           ...       ...        ...\n",
       "95       (32, 32, 32)        0.0775   0.00015   0.740451\n",
       "96           (32, 32)          0.25    0.0001   0.347403\n",
       "97       (32, 32, 32)         0.135  0.000125    0.35823\n",
       "98           (32, 16)          0.02  0.000175   0.846108\n",
       "99       (32, 32, 32)         0.135  0.000175  0.0997675\n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Performance from Random Search\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Best R^2</th>\n",
       "      <th>Number of hidden layers</th>\n",
       "      <th>Learning Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.927299</td>\n",
       "      <td>(32, 32, 32)</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Best R^2 Number of hidden layers Learning Rate\n",
       "0  0.927299            (32, 32, 32)          0.02"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "44.97604560852051"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start_random = time.time()\n",
    "# Set up the sample space for Random Search\n",
    "parameter_grid = { 'learning_rate_init' : np.linspace(0.020,.25,5),\n",
    "                    'hidden_layer_sizes' : [(32,32, 32),(32,32), (32, 16), (32), (16,8)], # list(range(16, 32, 5)),\n",
    "                    'alpha' : np.linspace(0.0001,.0002,5) }      # range(start, stop, step) #(start=1,stop=1000, num=100))\n",
    "# Define how many samples\n",
    "number_models = 100\n",
    "\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "rand_search_dnn_model = RandomizedSearchCV(estimator=MLPRegressor(), # check with empty model e.g. SVR()\n",
    "                            param_distributions=parameter_grid,\n",
    "                            n_iter = number_models,\n",
    "                            scoring='r2', n_jobs=30,\n",
    "                            cv = 2,            refit=True,\n",
    "                            return_train_score = True)\n",
    "# Add alpha as hyperparatemeter later??\n",
    "rand_search_dnn_model.fit(X_train_dev, y_train_dev)\n",
    "\n",
    "# Calculate scores on validation training set \n",
    "learning_rate_init = [item['learning_rate_init'] for item in rand_search_dnn_model.cv_results_['params']]\n",
    "hidden_layer_sizes = [item['hidden_layer_sizes'] for item in rand_search_dnn_model.cv_results_['params']]\n",
    "alpha = [item['alpha'] for item in rand_search_dnn_model.cv_results_['params']]\n",
    "scores = list(rand_search_dnn_model.cv_results_['mean_test_score'])\n",
    "\n",
    "df_rands_dnn_perfm = pd.DataFrame([hidden_layer_sizes, learning_rate_init, alpha, scores]).T\n",
    "df_rands_dnn_perfm.columns = ['hidden layer sizes','Learning Rate', 'alpha', 'Score']\n",
    "# df_rands_dnn_perfm.groupby(['Learning Rate']).mean()\n",
    "display(df_rands_dnn_perfm)\n",
    "\n",
    "# Determine the best performance and associate hyperparameters\n",
    "df_rands_best = pd.DataFrame([rand_search_dnn_model.best_score_, rand_search_dnn_model.best_params_['hidden_layer_sizes'],\n",
    "                              rand_search_dnn_model.best_params_['learning_rate_init']]).T\n",
    "print('Best Performance from Random Search')\n",
    "df_rands_best.columns = ['Best R^2', 'Number of hidden layers', 'Learning Rate']\n",
    "display(df_rands_best)\n",
    "\n",
    "\n",
    "# r2_test = .80 for hidden_layer 56,32  ++ good one\n",
    "df_rands_dnn_perfm.to_csv('df_rands_dnn_perfm.csv')\n",
    "df_rands_dnn_perfm.sort_values(by=['r2'],  ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UPPs9qFrTVFl"
   },
   "source": [
    "## TPOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tpot\n",
      "  Downloading TPOT-0.11.7-py3-none-any.whl (87 kB)\n",
      "Requirement already satisfied: numpy>=1.16.3 in c:\\programdata\\anaconda3\\lib\\site-packages (from tpot) (1.19.2)\n",
      "Requirement already satisfied: joblib>=0.13.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from tpot) (0.17.0)\n",
      "Requirement already satisfied: tqdm>=4.36.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from tpot) (4.50.2)\n",
      "Requirement already satisfied: pandas>=0.24.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from tpot) (1.1.3)\n",
      "Collecting xgboost>=1.1.0\n",
      "  Downloading xgboost-1.4.0-py3-none-win_amd64.whl (97.8 MB)\n",
      "Collecting stopit>=1.1.1\n",
      "  Downloading stopit-1.1.2.tar.gz (18 kB)\n",
      "Requirement already satisfied: scipy>=1.3.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from tpot) (1.5.2)\n",
      "Requirement already satisfied: scikit-learn>=0.22.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from tpot) (0.23.2)\n",
      "Collecting update-checker>=0.16\n",
      "  Downloading update_checker-0.18.0-py3-none-any.whl (7.0 kB)\n",
      "Collecting deap>=1.2\n",
      "  Downloading deap-1.3.1-cp38-cp38-win_amd64.whl (108 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\programdata\\anaconda3\\lib\\site-packages (from pandas>=0.24.2->tpot) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from pandas>=0.24.2->tpot) (2020.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from scikit-learn>=0.22.0->tpot) (2.1.0)\n",
      "Requirement already satisfied: requests>=2.3.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from update-checker>=0.16->tpot) (2.24.0)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7.3->pandas>=0.24.2->tpot) (1.15.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests>=2.3.0->update-checker>=0.16->tpot) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests>=2.3.0->update-checker>=0.16->tpot) (1.25.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests>=2.3.0->update-checker>=0.16->tpot) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests>=2.3.0->update-checker>=0.16->tpot) (3.0.4)\n",
      "Building wheels for collected packages: stopit\n",
      "  Building wheel for stopit (setup.py): started\n",
      "  Building wheel for stopit (setup.py): finished with status 'done'\n",
      "  Created wheel for stopit: filename=stopit-1.1.2-py3-none-any.whl size=11959 sha256=5b187a829a8564644e10138ca13ac67ac7e69a71825637eb1847543fc8f1b08c\n",
      "  Stored in directory: c:\\users\\tn753037\\appdata\\local\\pip\\cache\\wheels\\a8\\bb\\8f\\6b9328d23c2dcedbfeb8498b9f650d55d463089e3b8fc0bfb2\n",
      "Successfully built stopit\n",
      "Installing collected packages: xgboost, stopit, update-checker, deap, tpot\n",
      "Successfully installed deap-1.3.1 stopit-1.1.2 tpot-0.11.7 update-checker-0.18.0 xgboost-1.4.0\n"
     ]
    }
   ],
   "source": [
    "!pip install tpot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 368
    },
    "executionInfo": {
     "elapsed": 435,
     "status": "error",
     "timestamp": 1606113041714,
     "user": {
      "displayName": "Tanmoy Das",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgMmxHcU_jnpEQnP9s8nrNXQJMgMWwxHelgPXC5=s64",
      "userId": "02309773046279713936"
     },
     "user_tz": 240
    },
    "id": "iW-v5Xn0TVFl",
    "outputId": "9f497eb8-c342-4598-baa1-e4d883375664"
   },
   "outputs": [],
   "source": [
    "# TPOT\n",
    "from tpot import TPOTRegressor\n",
    "\n",
    "config_dict = { 'sklearn.neural_network.MLPRegressor':  { 'learning_rate_init' : np.linspace(0.015,1,100),\n",
    "                    'hidden_layer_sizes' : list(range(10, 1000, 10)) }}\n",
    "tpot_dnn_model = TPOTRegressor(generations=3, population_size=5,\n",
    "                    verbosity=2, offspring_size=10,\n",
    "                    scoring='r2', cv=2, config_dict = config_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TPOT\n",
    "from tpot import TPOTRegressor\n",
    "\n",
    "config_dict = { 'sklearn.neural_network.MLPRegressor':  { 'learning_rate_init' : np.linspace(0.015,.03,10),\n",
    "                    'hidden_layer_sizes' : [(64,32),(32,32), (32, 16)]}}  #[(32,32, 32),(32,32), (32, 16), (32), (16,8)]\n",
    "tpot_dnn_model = TPOTRegressor(generations=1, population_size=2,#\n",
    "                    verbosity=2, offspring_size=2,\n",
    "                    scoring='r2', cv=2, config_dict = config_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.7938345478237014\n",
      "\n",
      "Best pipeline: MLPRegressor(CombineDFs(CombineDFs(input_matrix, input_matrix), input_matrix), hidden_layer_sizes=(32, 32), learning_rate_init=0.019999999999999997)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.8661029343068397\n",
      "\n",
      "Best pipeline: MLPRegressor(input_matrix, hidden_layer_sizes=(64, 32), learning_rate_init=0.019999999999999997)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.8514794272705797\n",
      "\n",
      "Best pipeline: MLPRegressor(input_matrix, hidden_layer_sizes=(64, 32), learning_rate_init=0.021666666666666667)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.937040454877772\n",
      "\n",
      "Best pipeline: MLPRegressor(input_matrix, hidden_layer_sizes=(32, 32), learning_rate_init=0.015)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9577056577519354\n",
      "\n",
      "Best pipeline: MLPRegressor(MLPRegressor(input_matrix, hidden_layer_sizes=(32, 32), learning_rate_init=0.015), hidden_layer_sizes=(32, 32), learning_rate_init=0.019999999999999997)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9562192247899501\n",
      "\n",
      "Best pipeline: MLPRegressor(MLPRegressor(input_matrix, hidden_layer_sizes=(32, 32), learning_rate_init=0.018333333333333333), hidden_layer_sizes=(32, 16), learning_rate_init=0.03)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9466699950240665\n",
      "\n",
      "Best pipeline: MLPRegressor(input_matrix, hidden_layer_sizes=(64, 32), learning_rate_init=0.019999999999999997)\n",
      "TPOT EA on DNN: \n",
      "0.8917692712446847\n"
     ]
    }
   ],
   "source": [
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "TPOT_multi = MultiOutputRegressor(tpot_dnn_model)\n",
    "TPOT_multi.fit(X_train_dev, y_train_dev)\n",
    "\n",
    "print('TPOT EA on DNN: ')\n",
    "print(TPOT_multi.score(X_val, y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0xRFGtjdTVEW"
   },
   "source": [
    "## Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse_grid:  0.04773447079973153\n",
      "r2_grid:  0.9334571783201253\n",
      "Best Performance from Grid Search\n"
     ]
    }
   ],
   "source": [
    "# Develop GridSearchCV for the DNN\n",
    "\n",
    "params_grid = {\n",
    "    'hidden_layer_sizes':  [(64,32, 32),(64,64), (64, 32)], #[(64,32, 32),(64,32), (32, 16), (32), (16,8)]\n",
    "    'activation': ['relu'],\n",
    "    'solver': ['adam'], # max_iter=200\n",
    "    'learning_rate_init': [0.02,0.01]\n",
    "}\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "grid_search_dnn_model = GridSearchCV(MLPRegressor(), param_grid= params_grid,\n",
    "                        scoring='r2')\n",
    "# Add alpha as hyperparatemeter later??\n",
    "grid_search_dnn_model.fit(X_train_dev, y_train_dev)\n",
    "\n",
    "# Suppress ignore messages??\n",
    "\n",
    "# -------------------------- Performanec metrics\n",
    "# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error\n",
    "# RMSE_grid\n",
    "y_pred_grid = grid_search_dnn_model.predict(X_val)\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "rmse_grid = mean_squared_error(y_val, y_pred_grid, squared=False) # If False, returns RMSE value\n",
    "print('rmse_grid: ', rmse_grid)\n",
    "# R2_grid\n",
    "r2_grid = r2_score(y_val, y_pred_grid) # R^2\n",
    "print('r2_grid: ', r2_grid)\n",
    "\n",
    "\n",
    "# ----------------------------\n",
    "# Performance metric & accociated hyperparameters\n",
    "hidden_layer_sizes = [item['hidden_layer_sizes'] for item in grid_search_dnn_model.cv_results_['params']]\n",
    "hidden_layer_sizes2 =[]\n",
    "for i in range (len(hidden_layer_sizes)):\n",
    "    results = hidden_layer_sizes[i]\n",
    "    hidden_layer_sizes2.append(results[0])\n",
    "learning_rate_init = [item['learning_rate_init'] for item in grid_search_dnn_model.cv_results_['params']]\n",
    "scores = list(grid_search_dnn_model.cv_results_['mean_test_score'])\n",
    "df_grids_dnn_perfm = pd.DataFrame([hidden_layer_sizes2, learning_rate_init, scores]).T\n",
    "df_grids_dnn_perfm.columns = ['hidden layer sizes', 'learning rate', 'Score']\n",
    "df_grids_dnn_perfm\n",
    "\n",
    "# Determine the best performance and associate hyperparameters\n",
    "df_grids_best = pd.DataFrame([grid_search_dnn_model.best_score_, grid_search_dnn_model.best_params_['hidden_layer_sizes'],\n",
    "                              grid_search_dnn_model.best_params_['learning_rate_init']]).T\n",
    "print('Best Performance from Grid Search')\n",
    "df_grids_best.columns = ['Best R^2', 'Number of hidden layers', 'Learning Rate']\n",
    "df_grids_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Best R^2</th>\n",
       "      <th>Number of hidden layers</th>\n",
       "      <th>Learning Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.927181</td>\n",
       "      <td>(64, 32, 32)</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Best R^2 Number of hidden layers Learning Rate\n",
       "0  0.927181            (64, 32, 32)          0.01"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_grids_best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from hyperopt import hp\n",
    "# Setting the space\n",
    "space = {\n",
    "    'hidden_layer_sizes': hp.uniform('hidden_layer_sizes', 128, 256),\n",
    "    'learning_rate_init': hp.uniform('learning_rate_init', 0.01, .1),\n",
    "}# r2_score\n",
    "def objective_r2(params_bo):\n",
    "    params_bo = {'hidden_layer_sizes': int(params_bo['hidden_layer_sizes']),\n",
    "        'learning_rate_init': params_bo['learning_rate_init']}\n",
    "    neural_net_bo = MLPRegressor(activation='relu', solver='adam', alpha=0.001,\n",
    "                              batch_size='auto',  random_state=1, max_iter=100, verbose= False,\n",
    "                              **params_bo)\n",
    "    #\n",
    "    neural_net_bo.fit(X_train_dev, y_train_dev)\n",
    "    #r2_bo = (-1)* cross_val_score(neural_net_bo, X_train, y_train, scoring='r2', cv=5).mean()\n",
    "    y_pred_bo = neural_net_bo.predict(X_val)\n",
    "    r2_bo = r2_score(y_val, y_pred_bo) # If False, returns RMSE value\n",
    "    print('r2_bo: ', r2_bo)\n",
    "    print('hidden_layer_sizes, learning_rate_init', params_bo['hidden_layer_sizes'], params_bo['learning_rate_init'])\n",
    "    return r2_bo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2_bo:                                                                                                                 \n",
      "0.39840887907173556                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "134                                                                                                                    \n",
      "0.09542400141507949                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.655816927844316                                                                                                      \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "206                                                                                                                    \n",
      "0.042783816889806155                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.7705793406682504                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "159                                                                                                                    \n",
      "0.030666944504768416                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.8280820039551636                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "237                                                                                                                    \n",
      "0.018517365374393302                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.4989032716689679                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "207                                                                                                                    \n",
      "0.0899433458283459                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.6842132559633506                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "246                                                                                                                    \n",
      "0.04740692083433723                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.8347433192815817                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "178                                                                                                                    \n",
      "0.01774072572665357                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5129195729537824                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "210                                                                                                                    \n",
      "0.07197502849988491                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.8031285863857243                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "149                                                                                                                    \n",
      "0.015334414593443608                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.46010335265141833                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "132                                                                                                                    \n",
      "0.05219449198394748                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.44646995462506744                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "129                                                                                                                    \n",
      "0.09600758227709538                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6086808390896378                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "142                                                                                                                    \n",
      "0.0341980944124391                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.6649693882230674                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "248                                                                                                                    \n",
      "0.031013823745413796                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.48393316832404526                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "236                                                                                                                    \n",
      "0.0695706832784942                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.6865139975349145                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "157                                                                                                                    \n",
      "0.03586038099864275                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5954534196744016                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "168                                                                                                                    \n",
      "0.06610940598571181                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5597043535278249                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "174                                                                                                                    \n",
      "0.06687269146495421                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4398102996834762                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "208                                                                                                                    \n",
      "0.05379097899887204                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6471337138806723                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "134                                                                                                                    \n",
      "0.03638741686496069                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.8235407567229915                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "152                                                                                                                    \n",
      "0.024662212633582873                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.5654559379779193                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "191                                                                                                                    \n",
      "0.08311199919192279                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4566711010981007                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "222                                                                                                                    \n",
      "0.05901381815538745                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5134864940778425                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "190                                                                                                                    \n",
      "0.08068691180890256                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6106382775745871                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "223                                                                                                                    \n",
      "0.09997595792830671                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4368286815655682                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "185                                                                                                                    \n",
      "0.06088606989885918                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5905347209526148                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "183                                                                                                                    \n",
      "0.07874499446203824                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.2693653643677981                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "141                                                                                                                    \n",
      "0.08937513574941026                                                                                                    \n",
      "r2_bo:                                                                                                                 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4945677121182753                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "140                                                                                                                    \n",
      "0.09206481676037932                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.3770795349054194                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "166                                                                                                                    \n",
      "0.08756967748827167                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5300311386040281                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "168                                                                                                                    \n",
      "0.07547460645328473                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.3672435960999722                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "147                                                                                                                    \n",
      "0.08655759701936405                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5307349662828036                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "143                                                                                                                    \n",
      "0.09954787603117633                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5090398174962766                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "199                                                                                                                    \n",
      "0.08644234218067108                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4931719129894753                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "150                                                                                                                    \n",
      "0.07604455536337223                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.47739562715943523                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "162                                                                                                                    \n",
      "0.09423728654838454                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.32057404142380397                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "128                                                                                                                    \n",
      "0.0846682722153686                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.33097339177072654                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "128                                                                                                                    \n",
      "0.09801906946804151                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.3411023913644546                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "138                                                                                                                    \n",
      "0.04380836311956644                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4465650527153455                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "156                                                                                                                    \n",
      "0.06241601321938969                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5278903121669828                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "176                                                                                                                    \n",
      "0.08297610302667101                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.3757041312730732                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "128                                                                                                                    \n",
      "0.07450954384919119                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.41662759947129135                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "201                                                                                                                    \n",
      "0.05733277455851771                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.3077416322117616                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "133                                                                                                                    \n",
      "0.09098540474902761                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5254655202346472                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "135                                                                                                                    \n",
      "0.09244772630385228                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6274025356878566                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "219                                                                                                                    \n",
      "0.09013719034769388                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5358814372802004                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "240                                                                                                                    \n",
      "0.04865975020389745                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.39561720127281685                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "145                                                                                                                    \n",
      "0.07004634706348474                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4334668514810726                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "162                                                                                                                    \n",
      "0.06542149416602186                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.8137772178313876                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "173                                                                                                                    \n",
      "0.022994677168433175                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.5596066986461815                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "154                                                                                                                    \n",
      "0.095761184044418                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.8566541499424991                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "135                                                                                                                    \n",
      "0.013040385276392168                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.41239393476942027                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "183                                                                                                                    \n",
      "0.04229420701131681                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5772399287541814                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "230                                                                                                                    \n",
      "0.07216543191768884                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.554021008549571                                                                                                      \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "195                                                                                                                    \n",
      "0.07967019334178846                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5195502408331362                                                                                                     \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "160                                                                                                                    \n",
      "0.04870037371617788                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5403084725025368                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "253                                                                                                                    \n",
      "0.08926967872744813                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.41626901735933036                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "171                                                                                                                    \n",
      "0.05500714092093695                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5462761046825982                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "151                                                                                                                    \n",
      "0.06350275728461567                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.40575383226484185                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "213                                                                                                                    \n",
      "0.068080808410476                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.5446062111685177                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "132                                                                                                                    \n",
      "0.0777993064576689                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.48706310614720516                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "144                                                                                                                    \n",
      "0.09722935273831218                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.41483052806897236                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "180                                                                                                                    \n",
      "0.08195650508394864                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.40666431642623174                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "186                                                                                                                    \n",
      "0.0723799607103378                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.31021051289611606                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "141                                                                                                                    \n",
      "0.05179545456257314                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.8291203362997771                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "148                                                                                                                    \n",
      "0.010004640084429764                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.6619274318480534                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "140                                                                                                                    \n",
      "0.03999579559914699                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.7235109439488158                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "165                                                                                                                    \n",
      "0.029395404511413502                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.43256209482133123                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "137                                                                                                                    \n",
      "0.05675712005620799                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.7274300226315354                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "152                                                                                                                    \n",
      "0.03311846364180366                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.48521301172024883                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "159                                                                                                                    \n",
      "0.060892922928625734                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.46703983067725724                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "132                                                                                                                    \n",
      "0.04617267683544985                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6280287849316842                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "143                                                                                                                    \n",
      "0.038688195198249564                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.7567762003689497                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "157                                                                                                                    \n",
      "0.025014056033490703                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.5038604802980192                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "131                                                                                                                    \n",
      "0.09332994997480154                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4441552860898431                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "164                                                                                                                    \n",
      "0.05227296992007913                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4788177464983359                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "137                                                                                                                    \n",
      "0.05079544781263143                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5835038602404603                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "149                                                                                                                    \n",
      "0.045229994755025614                                                                                                   \n",
      "r2_bo:                                                                                                                 \n",
      "0.46171369797824                                                                                                       \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "176                                                                                                                    \n",
      "0.08505026958765238                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4455443678544343                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "170                                                                                                                    \n",
      "0.08947292890351884                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.26432724352574105                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "204                                                                                                                    \n",
      "0.09962797592840056                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5494376241533764                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "205                                                                                                                    \n",
      "0.09185725943329495                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4042629828893586                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "218                                                                                                                    \n",
      "0.09945462353039781                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4298423724129469                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "195                                                                                                                    \n",
      "0.09605496066567845                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4785519683376765                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "230                                                                                                                    \n",
      "0.09992267738730995                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5460205878289183                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "203                                                                                                                    \n",
      "0.07750495392371307                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4452229627741818                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "210                                                                                                                    \n",
      "0.08441245294345302                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.48708608328035546                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "188                                                                                                                    \n",
      "0.08716805235369028                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6023605622106195                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "213                                                                                                                    \n",
      "0.08154984771760974                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.44323199151390924                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "200                                                                                                                    \n",
      "0.09448300613184642                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.4980633398108771                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "196                                                                                                                    \n",
      "0.07422618033173925                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.496527870503928                                                                                                      \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "228                                                                                                                    \n",
      "0.09083648769611954                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.44259116467881243                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "181                                                                                                                    \n",
      "0.0794277926822729                                                                                                     \n",
      "r2_bo:                                                                                                                 \n",
      "0.6225072457287535                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "236                                                                                                                    \n",
      "0.09786169200929698                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6087258180218335                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "246                                                                                                                    \n",
      "0.069331992896658                                                                                                      \n",
      "r2_bo:                                                                                                                 \n",
      "0.5222103600540569                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "192                                                                                                                    \n",
      "0.08878715554637581                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.44188080373431715                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "217                                                                                                                    \n",
      "0.06497458092895125                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.49371708687494925                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "241                                                                                                                    \n",
      "0.07660306104437128                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.6116883844197545                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "206                                                                                                                    \n",
      "0.07379396804903862                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.5249438735577493                                                                                                     \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "226                                                                                                                    \n",
      "0.08370865980077925                                                                                                    \n",
      "r2_bo:                                                                                                                 \n",
      "0.46530612425084594                                                                                                    \n",
      "hidden_layer_sizes, learning_rate_init                                                                                 \n",
      "215                                                                                                                    \n",
      "0.09666203539805829                                                                                                    \n",
      "100%|█████████████████████████████████████████████| 100/100 [30:21<00:00, 18.21s/trial, best loss: 0.26432724352574105]\n",
      "r2_bo:  0.26432724352574105\n",
      "hidden_layer_sizes, learning_rate_init 204 0.09962797592840056\n",
      "Best R^2 {:.3f} params {} 0.26432724352574105 {'hidden_layer_sizes': 204.28690994704888, 'learning_rate_init': 0.09962797592840056}\n"
     ]
    }
   ],
   "source": [
    "from hyperopt import fmin, tpe\n",
    "best_result = fmin(fn=objective_r2,\n",
    "            space=space,\n",
    "            max_evals=100,\n",
    "            rstate=np.random.RandomState(22),\n",
    "            algo=tpe.suggest)\n",
    "\n",
    "# print the best parameter\n",
    "print(\"Best R^2 {:.3f} params {}\", objective_r2(best_result), best_result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XeBbGn-1TVF7"
   },
   "source": [
    "# Save the best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kMscTLKNTVF7"
   },
   "outputs": [],
   "source": [
    "# pickle.dump(grid_s, open('grid_s.sav', 'wb')) #  Grid Search\n",
    "\n",
    "# load the model from disk\n",
    "loaded_model = pickle.load(open('grid_search_dnn.sav', 'rb'))\n",
    "result = loaded_model.score(X_test, y_test)\n",
    "print(result)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "DNN_m.ipynb",
   "provenance": [
    {
     "file_id": "1fLLmd4654NLeHWeTUiqrYeqPNJaCERu0",
     "timestamp": 1612926314792
    }
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}