spam_filter_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import accuracy_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv(\"D:\\Desktop\\ML with pyhton\\datasets\\emails.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's observe the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Email No.  the  to  ect  and  for  of    a  you  hou     ...      connevey  \\\n",
      "0   Email 1    0   0    1    0    0   0    2    0    0     ...             0   \n",
      "1   Email 2    8  13   24    6    6   2  102    1   27     ...             0   \n",
      "2   Email 3    0   0    1    0    0   0    8    0    0     ...             0   \n",
      "3   Email 4    0   5   22    0    5   1   51    2   10     ...             0   \n",
      "4   Email 5    7   6   17    1    5   2   57    0    9     ...             0   \n",
      "\n",
      "   jay  valued  lay  infrastructure  military  allowing  ff  dry  Prediction  \n",
      "0    0       0    0               0         0         0   0    0           0  \n",
      "1    0       0    0               0         0         0   1    0           0  \n",
      "2    0       0    0               0         0         0   0    0           0  \n",
      "3    0       0    0               0         0         0   0    0           0  \n",
      "4    0       0    0               0         0         0   1    0           0  \n",
      "\n",
      "[5 rows x 3002 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(5172, 3002)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(data.head()) \n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.reset_index(drop=True)\n",
    "data.drop(\"Email No.\",axis=1,inplace=True)\n",
    "data=data.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      the  to  ect  and  for  of    a  you  hou  in     ...      connevey  \\\n",
      "445     5   1    1    1    7   0   19    1    0   1     ...             0   \n",
      "3232    6   6    1    3    1   2   44    3    0  12     ...             0   \n",
      "4387    0   1    2    1    0   0   17    0    1   6     ...             0   \n",
      "5068   53  49    3   25   47  30  433    0    2  78     ...             0   \n",
      "1541    3   5    1    0    0   6   37    1    0   4     ...             0   \n",
      "\n",
      "      jay  valued  lay  infrastructure  military  allowing  ff  dry  \\\n",
      "445     0       0    0               0         0         0   0    0   \n",
      "3232    0       0    0               0         0         0   0    1   \n",
      "4387    0       0    0               0         0         0   0    0   \n",
      "5068    0       0    0               0         0         0   0    0   \n",
      "1541    0       0    0               0         0         0   4    0   \n",
      "\n",
      "      Prediction  \n",
      "445            0  \n",
      "3232           0  \n",
      "4387           0  \n",
      "5068           0  \n",
      "1541           0  \n",
      "\n",
      "[5 rows x 3001 columns]\n"
     ]
    }
   ],
   "source": [
    " print(data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>the</th>\n",
       "      <th>to</th>\n",
       "      <th>ect</th>\n",
       "      <th>and</th>\n",
       "      <th>for</th>\n",
       "      <th>of</th>\n",
       "      <th>a</th>\n",
       "      <th>you</th>\n",
       "      <th>hou</th>\n",
       "      <th>in</th>\n",
       "      <th>...</th>\n",
       "      <th>connevey</th>\n",
       "      <th>jay</th>\n",
       "      <th>valued</th>\n",
       "      <th>lay</th>\n",
       "      <th>infrastructure</th>\n",
       "      <th>military</th>\n",
       "      <th>allowing</th>\n",
       "      <th>ff</th>\n",
       "      <th>dry</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.640565</td>\n",
       "      <td>6.188128</td>\n",
       "      <td>5.143852</td>\n",
       "      <td>3.075599</td>\n",
       "      <td>3.124710</td>\n",
       "      <td>2.627030</td>\n",
       "      <td>55.517401</td>\n",
       "      <td>2.466551</td>\n",
       "      <td>2.024362</td>\n",
       "      <td>10.600155</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005027</td>\n",
       "      <td>0.012568</td>\n",
       "      <td>0.010634</td>\n",
       "      <td>0.098028</td>\n",
       "      <td>0.004254</td>\n",
       "      <td>0.006574</td>\n",
       "      <td>0.004060</td>\n",
       "      <td>0.914733</td>\n",
       "      <td>0.006961</td>\n",
       "      <td>0.290023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>11.745009</td>\n",
       "      <td>9.534576</td>\n",
       "      <td>14.101142</td>\n",
       "      <td>6.045970</td>\n",
       "      <td>4.680522</td>\n",
       "      <td>6.229845</td>\n",
       "      <td>87.574172</td>\n",
       "      <td>4.314444</td>\n",
       "      <td>6.967878</td>\n",
       "      <td>19.281892</td>\n",
       "      <td>...</td>\n",
       "      <td>0.105788</td>\n",
       "      <td>0.199682</td>\n",
       "      <td>0.116693</td>\n",
       "      <td>0.569532</td>\n",
       "      <td>0.096252</td>\n",
       "      <td>0.138908</td>\n",
       "      <td>0.072145</td>\n",
       "      <td>2.780203</td>\n",
       "      <td>0.098086</td>\n",
       "      <td>0.453817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>62.250000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>210.000000</td>\n",
       "      <td>132.000000</td>\n",
       "      <td>344.000000</td>\n",
       "      <td>89.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>1898.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>167.000000</td>\n",
       "      <td>223.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 3001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               the           to          ect          and          for  \\\n",
       "count  5172.000000  5172.000000  5172.000000  5172.000000  5172.000000   \n",
       "mean      6.640565     6.188128     5.143852     3.075599     3.124710   \n",
       "std      11.745009     9.534576    14.101142     6.045970     4.680522   \n",
       "min       0.000000     0.000000     1.000000     0.000000     0.000000   \n",
       "25%       0.000000     1.000000     1.000000     0.000000     1.000000   \n",
       "50%       3.000000     3.000000     1.000000     1.000000     2.000000   \n",
       "75%       8.000000     7.000000     4.000000     3.000000     4.000000   \n",
       "max     210.000000   132.000000   344.000000    89.000000    47.000000   \n",
       "\n",
       "                of            a          you          hou           in  \\\n",
       "count  5172.000000  5172.000000  5172.000000  5172.000000  5172.000000   \n",
       "mean      2.627030    55.517401     2.466551     2.024362    10.600155   \n",
       "std       6.229845    87.574172     4.314444     6.967878    19.281892   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000    12.000000     0.000000     0.000000     1.000000   \n",
       "50%       1.000000    28.000000     1.000000     0.000000     5.000000   \n",
       "75%       2.000000    62.250000     3.000000     1.000000    12.000000   \n",
       "max      77.000000  1898.000000    70.000000   167.000000   223.000000   \n",
       "\n",
       "          ...          connevey          jay       valued          lay  \\\n",
       "count     ...       5172.000000  5172.000000  5172.000000  5172.000000   \n",
       "mean      ...          0.005027     0.012568     0.010634     0.098028   \n",
       "std       ...          0.105788     0.199682     0.116693     0.569532   \n",
       "min       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "25%       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "50%       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "75%       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "max       ...          4.000000     7.000000     2.000000    12.000000   \n",
       "\n",
       "       infrastructure     military     allowing           ff          dry  \\\n",
       "count     5172.000000  5172.000000  5172.000000  5172.000000  5172.000000   \n",
       "mean         0.004254     0.006574     0.004060     0.914733     0.006961   \n",
       "std          0.096252     0.138908     0.072145     2.780203     0.098086   \n",
       "min          0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%          0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "50%          0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "75%          0.000000     0.000000     0.000000     1.000000     0.000000   \n",
       "max          3.000000     4.000000     3.000000   114.000000     4.000000   \n",
       "\n",
       "        Prediction  \n",
       "count  5172.000000  \n",
       "mean      0.290023  \n",
       "std       0.453817  \n",
       "min       0.000000  \n",
       "25%       0.000000  \n",
       "50%       0.000000  \n",
       "75%       1.000000  \n",
       "max       1.000000  \n",
       "\n",
       "[8 rows x 3001 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y=data[\"Prediction\"]\n",
    "data.drop(\"Prediction\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,Y_train,Y_test=train_test_split(data,Y,test_size=0.2)\n",
    "print(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaler = MinMaxScaler()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4137, 3000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\mysoftware\\anacondafiles\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\data.py:323: DataConversionWarning: Data with input dtype int64 were all converted to float64 by MinMaxScaler.\n",
      "  return self.partial_fit(X, y)\n"
     ]
    }
   ],
   "source": [
    "min_max_scaler.fit(X_train)\n",
    "X_scaledata=min_max_scaler.transform(X_train)  #scaliing the data\n",
    "X_scaledtest_data=min_max_scaler.transform(X_test)\n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=X_scaledata.T\n",
    "X_Test=X_scaledtest_data.T\n",
    "Y=np.array(Y_train)\n",
    "Y_Test=np.array(Y_test)\n",
    "devX_test=X_Test[:,0:10]\n",
    "devY_test=Y_Test[0:10]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialising parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialise parameters\n",
    "def initialise_parameters(layer_dimensions):\n",
    "    parameters={}\n",
    "    for i in range(1,len(layer_dimensions)):\n",
    "        parameters[\"W\"+str(i)]=np.random.randn(layer_dimensions[i],layer_dimensions[i-1])*np.sqrt(2/layer_dimensions[i-1])\n",
    "        parameters[\"b\"+str(i)]=np.zeros((layer_dimensions[i],1))\n",
    "    return parameters\n",
    "#print(initialise_parameters([4,3,1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helping functions for forward propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(Z):\n",
    "    return np.maximum(0,Z)\n",
    "\n",
    "def sigmoid(Z):\n",
    "    return 1/(1+np.exp(-Z))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_and_activation_relu(A_prev,W,b):\n",
    "    Z=np.dot(W,A_prev)+b\n",
    "    A=relu(Z)\n",
    "    linear_cache=(A_prev,W,b)\n",
    "    Activation_cache=A\n",
    "    cache=(linear_cache,Activation_cache)\n",
    "    return A,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_and_activation_sigmoid(A_prev,W,b):\n",
    "    Z=np.dot(W,A_prev)+b\n",
    "    A=sigmoid(Z)\n",
    "    linear_cache=(A_prev,W,b)\n",
    "    Activation_cache=A\n",
    "    cache=(linear_cache,Activation_cache)\n",
    "    return A,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_layer_cal(A,W,b,activation):\n",
    "    if activation==\"relu\":\n",
    "        A,cache1=linear_and_activation_relu(A,W,b)\n",
    "    elif activation==\"sigmoid\":\n",
    "        A,cache1=linear_and_activation_sigmoid(A,W,b)\n",
    "    return A,cache1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Forward propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    " def forward_module(X,parameters):\n",
    "    caches=[]\n",
    "    cache=()\n",
    "    A=X\n",
    "    layers=int(len(parameters)/2)\n",
    "    \n",
    "    for i in range(1,layers):\n",
    "        A_prev=A\n",
    "        A,cache=forward_layer_cal(A_prev,parameters[\"W\"+str(i)],parameters[\"b\"+str(i)],\"relu\")\n",
    "        caches.append(cache)\n",
    "    AL,cache=forward_layer_cal(A,parameters[\"W\"+str(layers)],parameters[\"b\"+str(layers)],\"sigmoid\")\n",
    "    caches.append(cache)\n",
    "    return AL,caches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_cal(Al,Y,m):\n",
    "    cost=-(1/m)*(np.dot(np.log(Al),Y)+np.dot(np.log(1-Al),(1-Y)))\n",
    "    cost = np.squeeze(cost)\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helping function for back propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_backward(dZ,linear_cache):\n",
    "    A_prev,W,b=linear_cache\n",
    "    m=A_prev.shape[1]\n",
    "    dW = (1/m)*(np.dot(dZ,A_prev.T))\n",
    "    db = (1/m)*np.sum(dZ,axis=1,keepdims=True)\n",
    "    dA_prev = np.dot(W.T,dZ)\n",
    "    return dA_prev,dW,db  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_backward(dA,A):\n",
    "    dZ=dA*A*(1-A)\n",
    "    return dZ\n",
    "def relu_backward(dA,A):\n",
    "    dZ = np.array(dA, copy=True)\n",
    "    dZ[A <=0]=0\n",
    "    return dZ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#finding derivative of parameters with respect to loss\n",
    "def backward_layer_cal(dA,cache,activation):\n",
    "    linear_cache,activation_cache=cache\n",
    "    if activation==\"sigmoid\":\n",
    "        dZ=sigmoid_backward(dA,activation_cache)\n",
    "        dA,dW,db=linear_backward(dZ,linear_cache)\n",
    "    elif activation==\"relu\":\n",
    "        dZ=relu_backward(dA,activation_cache)\n",
    "        dA,dW,db=linear_backward(dZ,linear_cache)\n",
    "    return dA,dW,db"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Back propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_prop_module(cost,AL,cache,m,Y):\n",
    "    grads={}\n",
    "    layers=len(cache)\n",
    "    #print(type(Y))\n",
    "    #print(AL.shape)\n",
    "    Y=Y.reshape(AL.shape)\n",
    "    \n",
    "    dAL= -np.divide(Y,AL)+np.divide(1-Y,1-AL)\n",
    "    #print(dAL.shape)\n",
    "    \n",
    "    sigmoid_cache=cache[layers-1]\n",
    "    \n",
    "    \n",
    "    grads[\"dA\"+str(layers-1)],grads[\"dW\"+str(layers)],grads[\"db\"+str(layers)]=backward_layer_cal(dAL,sigmoid_cache,\"sigmoid\")\n",
    "    #print(grads)\n",
    "    \n",
    "    for l in reversed(range(layers-1)):\n",
    "        #print(l)\n",
    "        current_cache=cache[l]\n",
    "        grads[\"dA\"+str(l)],grads[\"dW\"+str(l+1)],grads[\"db\"+str(l+1)]=backward_layer_cal(grads[\"dA\"+str(l+1)],current_cache,\"relu\")\n",
    "        \n",
    "        \n",
    "    return grads  \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Updating parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_parameters(grads,parameters,learning_rate,layers):\n",
    "    for l in range(1,layers+1):\n",
    "        parameters[\"W\"+str(l)]=parameters[\"W\"+str(l)]-learning_rate*grads[\"dW\"+str(l)]\n",
    "        parameters[\"b\"+str(l)]=parameters[\"b\"+str(l)]-learning_rate*grads[\"db\"+str(l)]\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# L-layer deep learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deep_learning_module(X,Y,learning_rate,layer_dim,iteration):\n",
    "    m=X.shape[1]\n",
    "    layers=len(layer_dim)-1\n",
    "    parameters=initialise_parameters(layer_dim)\n",
    "    for i in range(iteration):\n",
    "        AL,cache=forward_module(X,parameters)\n",
    "        Cost=cost_cal(AL,Y,m)\n",
    "        print(\"cost at\"+str(i)+\" iteration:\"+\" \"+\"{}\".format(Cost))\n",
    "        grads=backward_prop_module(Cost,AL,cache,m,Y)\n",
    "        parameters=update_parameters(grads,parameters,learning_rate,layers)\n",
    "        \n",
    "    return parameters\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X_test,learned_parameters,Y):\n",
    "    AL,cache=forward_module(X_test,learned_parameters)\n",
    "    prediction=(AL>=0.5).astype(int)\n",
    "    prediction=prediction.reshape(Y.shape)\n",
    "    accuracy = accuracy_score(Y,prediction,normalize=True)\n",
    "    return accuracy\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at0 iteration: 0.6873473603698033\n",
      "cost at1 iteration: 0.6768112485287184\n",
      "cost at2 iteration: 0.6669505322332437\n",
      "cost at3 iteration: 0.6579792372819145\n",
      "cost at4 iteration: 0.6498927084920024\n",
      "cost at5 iteration: 0.6426765810491262\n",
      "cost at6 iteration: 0.6362114245922729\n",
      "cost at7 iteration: 0.630410193070876\n",
      "cost at8 iteration: 0.6251962923045394\n",
      "cost at9 iteration: 0.6205101599747562\n",
      "cost at10 iteration: 0.6162813667743346\n",
      "cost at11 iteration: 0.6124572266494267\n",
      "cost at12 iteration: 0.6089905182938651\n",
      "cost at13 iteration: 0.605846557456267\n",
      "cost at14 iteration: 0.6029917503870662\n",
      "cost at15 iteration: 0.6003839315624596\n",
      "cost at16 iteration: 0.5979854278145142\n",
      "cost at17 iteration: 0.595761402074937\n",
      "cost at18 iteration: 0.5936927281948945\n",
      "cost at19 iteration: 0.591760383957588\n",
      "cost at20 iteration: 0.589945587980986\n",
      "cost at21 iteration: 0.5882228366633682\n",
      "cost at22 iteration: 0.5865726282745122\n",
      "cost at23 iteration: 0.5849898355828723\n",
      "cost at24 iteration: 0.5834539853939203\n",
      "cost at25 iteration: 0.581952113581999\n",
      "cost at26 iteration: 0.5804775105368696\n",
      "cost at27 iteration: 0.5790255323047739\n",
      "cost at28 iteration: 0.5775850594584206\n",
      "cost at29 iteration: 0.576152762565627\n",
      "cost at30 iteration: 0.5747202391127308\n",
      "cost at31 iteration: 0.5732725162811813\n",
      "cost at32 iteration: 0.5717935473549096\n",
      "cost at33 iteration: 0.5702873463562034\n",
      "cost at34 iteration: 0.5687409467977077\n",
      "cost at35 iteration: 0.5671418728246138\n",
      "cost at36 iteration: 0.5654879040414227\n",
      "cost at37 iteration: 0.5637739803326807\n",
      "cost at38 iteration: 0.5619758765978861\n",
      "cost at39 iteration: 0.5601117577591059\n",
      "cost at40 iteration: 0.5582678975964918\n",
      "cost at41 iteration: 0.5564556266130422\n",
      "cost at42 iteration: 0.5546637866947994\n",
      "cost at43 iteration: 0.5528803674471348\n",
      "cost at44 iteration: 0.5511092685961331\n",
      "cost at45 iteration: 0.5493346449575792\n",
      "cost at46 iteration: 0.5475589468776231\n",
      "cost at47 iteration: 0.5457790722345057\n",
      "cost at48 iteration: 0.5439881561577038\n",
      "cost at49 iteration: 0.5421841774660471\n",
      "cost at50 iteration: 0.5403663248395003\n",
      "cost at51 iteration: 0.5385258368481187\n",
      "cost at52 iteration: 0.5366529502174081\n",
      "cost at53 iteration: 0.5347628982873733\n",
      "cost at54 iteration: 0.5328586787889829\n",
      "cost at55 iteration: 0.5309368155262628\n",
      "cost at56 iteration: 0.5290005706961008\n",
      "cost at57 iteration: 0.5270520013232802\n",
      "cost at58 iteration: 0.5250844275596901\n",
      "cost at59 iteration: 0.5230993024679331\n",
      "cost at60 iteration: 0.5210991801580581\n",
      "cost at61 iteration: 0.51907420801689\n",
      "cost at62 iteration: 0.5170324757334019\n",
      "cost at63 iteration: 0.5149792238614279\n",
      "cost at64 iteration: 0.5129048131837641\n",
      "cost at65 iteration: 0.5108129220212703\n",
      "cost at66 iteration: 0.5087008339006568\n",
      "cost at67 iteration: 0.5065724271713707\n",
      "cost at68 iteration: 0.5044275007058608\n",
      "cost at69 iteration: 0.5022630155639919\n",
      "cost at70 iteration: 0.5000784663003116\n",
      "cost at71 iteration: 0.4978747374279159\n",
      "cost at72 iteration: 0.49565491014317\n",
      "cost at73 iteration: 0.49341698846825277\n",
      "cost at74 iteration: 0.49116154631009723\n",
      "cost at75 iteration: 0.48888196289773256\n",
      "cost at76 iteration: 0.48658272675547565\n",
      "cost at77 iteration: 0.4842658303131662\n",
      "cost at78 iteration: 0.48193313237061786\n",
      "cost at79 iteration: 0.47958318687183743\n",
      "cost at80 iteration: 0.47721898934885476\n",
      "cost at81 iteration: 0.47483545432938\n",
      "cost at82 iteration: 0.4724373868144318\n",
      "cost at83 iteration: 0.4700259464786372\n",
      "cost at84 iteration: 0.46760094852475764\n",
      "cost at85 iteration: 0.4651624281322544\n",
      "cost at86 iteration: 0.46271031114317523\n",
      "cost at87 iteration: 0.46024693058104243\n",
      "cost at88 iteration: 0.45777376211652443\n",
      "cost at89 iteration: 0.45529299313560395\n",
      "cost at90 iteration: 0.4528004636114637\n",
      "cost at91 iteration: 0.4502976671044769\n",
      "cost at92 iteration: 0.44778537410124764\n",
      "cost at93 iteration: 0.4452650617802658\n",
      "cost at94 iteration: 0.44273883494886407\n",
      "cost at95 iteration: 0.4402061896325307\n",
      "cost at96 iteration: 0.43766806755691356\n",
      "cost at97 iteration: 0.43512465334231737\n",
      "cost at98 iteration: 0.4325769427600222\n",
      "cost at99 iteration: 0.43002468188825776\n",
      "cost at100 iteration: 0.4274652193456942\n",
      "cost at101 iteration: 0.4249023959337004\n",
      "cost at102 iteration: 0.422336429688577\n",
      "cost at103 iteration: 0.4197693617080111\n",
      "cost at104 iteration: 0.417197415485945\n",
      "cost at105 iteration: 0.4146201888777579\n",
      "cost at106 iteration: 0.4120411377997053\n",
      "cost at107 iteration: 0.4094622631785787\n",
      "cost at108 iteration: 0.4068828347646782\n",
      "cost at109 iteration: 0.4043037728830834\n",
      "cost at110 iteration: 0.40172384912818015\n",
      "cost at111 iteration: 0.39914479285269894\n",
      "cost at112 iteration: 0.39656714929161174\n",
      "cost at113 iteration: 0.3939885266318276\n",
      "cost at114 iteration: 0.39141103764469837\n",
      "cost at115 iteration: 0.38883652435600474\n",
      "cost at116 iteration: 0.3862643277784142\n",
      "cost at117 iteration: 0.3836950590627453\n",
      "cost at118 iteration: 0.3811290003493223\n",
      "cost at119 iteration: 0.378568389987614\n",
      "cost at120 iteration: 0.37601308110634707\n",
      "cost at121 iteration: 0.37346284822268944\n",
      "cost at122 iteration: 0.370917710089512\n",
      "cost at123 iteration: 0.3683777340483425\n",
      "cost at124 iteration: 0.3658449813029042\n",
      "cost at125 iteration: 0.3633206490377022\n",
      "cost at126 iteration: 0.36080286731241756\n",
      "cost at127 iteration: 0.35829192848008307\n",
      "cost at128 iteration: 0.35578846120179153\n",
      "cost at129 iteration: 0.35329606173491185\n",
      "cost at130 iteration: 0.3508133831985951\n",
      "cost at131 iteration: 0.3483402629301729\n",
      "cost at132 iteration: 0.3458781696343969\n",
      "cost at133 iteration: 0.34342676873279854\n",
      "cost at134 iteration: 0.3409871486287783\n",
      "cost at135 iteration: 0.3385579105038131\n",
      "cost at136 iteration: 0.3361400759576785\n",
      "cost at137 iteration: 0.3337337834455788\n",
      "cost at138 iteration: 0.33134069953943707\n",
      "cost at139 iteration: 0.32896077070535723\n",
      "cost at140 iteration: 0.326595289162771\n",
      "cost at141 iteration: 0.3242457633242953\n",
      "cost at142 iteration: 0.3219097537623644\n",
      "cost at143 iteration: 0.31958751765144655\n",
      "cost at144 iteration: 0.3172797598862316\n",
      "cost at145 iteration: 0.3149868041660254\n",
      "cost at146 iteration: 0.3127084696909052\n",
      "cost at147 iteration: 0.31044543938605373\n",
      "cost at148 iteration: 0.3081978425275736\n",
      "cost at149 iteration: 0.30596471028006905\n",
      "cost at150 iteration: 0.3037461224351321\n",
      "cost at151 iteration: 0.3015450872485066\n",
      "cost at152 iteration: 0.29935965350585303\n",
      "cost at153 iteration: 0.29718914573131544\n",
      "cost at154 iteration: 0.29503275636604115\n",
      "cost at155 iteration: 0.29289284564332974\n",
      "cost at156 iteration: 0.29076933632104934\n",
      "cost at157 iteration: 0.2886629693812914\n",
      "cost at158 iteration: 0.28657341185869006\n",
      "cost at159 iteration: 0.2845013774517467\n",
      "cost at160 iteration: 0.2824463554067213\n",
      "cost at161 iteration: 0.2804077526392325\n",
      "cost at162 iteration: 0.2783853813211737\n",
      "cost at163 iteration: 0.2763805418866303\n",
      "cost at164 iteration: 0.2743928631942368\n",
      "cost at165 iteration: 0.27242246479220733\n",
      "cost at166 iteration: 0.27047029212062385\n",
      "cost at167 iteration: 0.26853576616220604\n",
      "cost at168 iteration: 0.2666176001512597\n",
      "cost at169 iteration: 0.26471742367386303\n",
      "cost at170 iteration: 0.26283500403113946\n",
      "cost at171 iteration: 0.2609695806573994\n",
      "cost at172 iteration: 0.2591215621837808\n",
      "cost at173 iteration: 0.2572910255168575\n",
      "cost at174 iteration: 0.255477969721148\n",
      "cost at175 iteration: 0.25368183692956997\n",
      "cost at176 iteration: 0.2519022527241725\n",
      "cost at177 iteration: 0.25013896124894675\n",
      "cost at178 iteration: 0.24839179453663401\n",
      "cost at179 iteration: 0.2466616432780734\n",
      "cost at180 iteration: 0.2449483758886735\n",
      "cost at181 iteration: 0.2432518013250168\n",
      "cost at182 iteration: 0.24157224075992736\n",
      "cost at183 iteration: 0.23990920719192405\n",
      "cost at184 iteration: 0.2382632931445435\n",
      "cost at185 iteration: 0.2366341908634327\n",
      "cost at186 iteration: 0.2350219676052487\n",
      "cost at187 iteration: 0.2334259272091648\n",
      "cost at188 iteration: 0.2318456804807117\n",
      "cost at189 iteration: 0.23028144479413185\n",
      "cost at190 iteration: 0.22873327919571357\n",
      "cost at191 iteration: 0.2272013034576828\n",
      "cost at192 iteration: 0.22568511631051216\n",
      "cost at193 iteration: 0.2241841516888457\n",
      "cost at194 iteration: 0.22269857911552515\n",
      "cost at195 iteration: 0.2212278665198636\n",
      "cost at196 iteration: 0.21977291055985254\n",
      "cost at197 iteration: 0.21833341873274673\n",
      "cost at198 iteration: 0.21690893419549068\n",
      "cost at199 iteration: 0.2154992579591505\n",
      "cost at200 iteration: 0.2141043360715369\n",
      "cost at201 iteration: 0.2127239454058688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at202 iteration: 0.21135804754256876\n",
      "cost at203 iteration: 0.21000694040814338\n",
      "cost at204 iteration: 0.2086700986255091\n",
      "cost at205 iteration: 0.20734788278131677\n",
      "cost at206 iteration: 0.20604005275404574\n",
      "cost at207 iteration: 0.2047458549208573\n",
      "cost at208 iteration: 0.2034652541287695\n",
      "cost at209 iteration: 0.20219810897766347\n",
      "cost at210 iteration: 0.20094437229834328\n",
      "cost at211 iteration: 0.1997034977807275\n",
      "cost at212 iteration: 0.1984756630854406\n",
      "cost at213 iteration: 0.1972604729097698\n",
      "cost at214 iteration: 0.19605828520722055\n",
      "cost at215 iteration: 0.19486873227085727\n",
      "cost at216 iteration: 0.19369126120655566\n",
      "cost at217 iteration: 0.19252603260766427\n",
      "cost at218 iteration: 0.1913727461378411\n",
      "cost at219 iteration: 0.19023149546945484\n",
      "cost at220 iteration: 0.18910216673023533\n",
      "cost at221 iteration: 0.18798463427212717\n",
      "cost at222 iteration: 0.18687847763702692\n",
      "cost at223 iteration: 0.1857838391457042\n",
      "cost at224 iteration: 0.18470044098826965\n",
      "cost at225 iteration: 0.1836281907565213\n",
      "cost at226 iteration: 0.1825670132299994\n",
      "cost at227 iteration: 0.1815168674411301\n",
      "cost at228 iteration: 0.18047781447153152\n",
      "cost at229 iteration: 0.17944970834810095\n",
      "cost at230 iteration: 0.17843224585326942\n",
      "cost at231 iteration: 0.17742509981602048\n",
      "cost at232 iteration: 0.17642811080984097\n",
      "cost at233 iteration: 0.17544120748934983\n",
      "cost at234 iteration: 0.17446439443540535\n",
      "cost at235 iteration: 0.17349759948712962\n",
      "cost at236 iteration: 0.1725406000812559\n",
      "cost at237 iteration: 0.1715934289933203\n",
      "cost at238 iteration: 0.17065586092406962\n",
      "cost at239 iteration: 0.16972777151151613\n",
      "cost at240 iteration: 0.16880902828662384\n",
      "cost at241 iteration: 0.16789976004413568\n",
      "cost at242 iteration: 0.16699964295878184\n",
      "cost at243 iteration: 0.16610870231614178\n",
      "cost at244 iteration: 0.16522670803946926\n",
      "cost at245 iteration: 0.16435371676453356\n",
      "cost at246 iteration: 0.16348950279200872\n",
      "cost at247 iteration: 0.16263371597631213\n",
      "cost at248 iteration: 0.1617863201414095\n",
      "cost at249 iteration: 0.16094717691376428\n",
      "cost at250 iteration: 0.16011635446914194\n",
      "cost at251 iteration: 0.15929387775515658\n",
      "cost at252 iteration: 0.15847949696466127\n",
      "cost at253 iteration: 0.15767325945839358\n",
      "cost at254 iteration: 0.15687472983557516\n",
      "cost at255 iteration: 0.1560838936423453\n",
      "cost at256 iteration: 0.15530094576806047\n",
      "cost at257 iteration: 0.15452543508120384\n",
      "cost at258 iteration: 0.15375739382850426\n",
      "cost at259 iteration: 0.15299663112160963\n",
      "cost at260 iteration: 0.15224295904518054\n",
      "cost at261 iteration: 0.1514961686830623\n",
      "cost at262 iteration: 0.15075638304557828\n",
      "cost at263 iteration: 0.1500235358401775\n",
      "cost at264 iteration: 0.14929770328550787\n",
      "cost at265 iteration: 0.14857877017357196\n",
      "cost at266 iteration: 0.147866535883757\n",
      "cost at267 iteration: 0.14716099762625567\n",
      "cost at268 iteration: 0.14646200426217337\n",
      "cost at269 iteration: 0.14576949207407983\n",
      "cost at270 iteration: 0.14508356230497804\n",
      "cost at271 iteration: 0.14440398939283375\n",
      "cost at272 iteration: 0.14373066838754187\n",
      "cost at273 iteration: 0.1430635313373774\n",
      "cost at274 iteration: 0.14240229752404282\n",
      "cost at275 iteration: 0.14174714826336965\n",
      "cost at276 iteration: 0.14109804216212385\n",
      "cost at277 iteration: 0.14045475207362954\n",
      "cost at278 iteration: 0.13981723574192068\n",
      "cost at279 iteration: 0.139185389966885\n",
      "cost at280 iteration: 0.13855902350279795\n",
      "cost at281 iteration: 0.1379383231849914\n",
      "cost at282 iteration: 0.13732309598549786\n",
      "cost at283 iteration: 0.13671332302541545\n",
      "cost at284 iteration: 0.1361089119055856\n",
      "cost at285 iteration: 0.13550979591014983\n",
      "cost at286 iteration: 0.13491582602086452\n",
      "cost at287 iteration: 0.13432700323118768\n",
      "cost at288 iteration: 0.13374325260292377\n",
      "cost at289 iteration: 0.13316454024507282\n",
      "cost at290 iteration: 0.13259096216792426\n",
      "cost at291 iteration: 0.1320224234463734\n",
      "cost at292 iteration: 0.13145876875834345\n",
      "cost at293 iteration: 0.13089994799373006\n",
      "cost at294 iteration: 0.13034595951296066\n",
      "cost at295 iteration: 0.12979666393480654\n",
      "cost at296 iteration: 0.12925203737788218\n",
      "cost at297 iteration: 0.1287119658210955\n",
      "cost at298 iteration: 0.12817648481669855\n",
      "cost at299 iteration: 0.12764553120421743\n",
      "cost at300 iteration: 0.12711910934296264\n",
      "cost at301 iteration: 0.1265970193190799\n",
      "cost at302 iteration: 0.12607924778528973\n",
      "cost at303 iteration: 0.1255656915928377\n",
      "cost at304 iteration: 0.12505644161283377\n",
      "cost at305 iteration: 0.12455141250533167\n",
      "cost at306 iteration: 0.12405049639219463\n",
      "cost at307 iteration: 0.12355378586259931\n",
      "cost at308 iteration: 0.12306110937885552\n",
      "cost at309 iteration: 0.12257246750946267\n",
      "cost at310 iteration: 0.12208775543600749\n",
      "cost at311 iteration: 0.12160694100368115\n",
      "cost at312 iteration: 0.12112990655773619\n",
      "cost at313 iteration: 0.12065666770004217\n",
      "cost at314 iteration: 0.12018721925690054\n",
      "cost at315 iteration: 0.11972147326002669\n",
      "cost at316 iteration: 0.11925943987869098\n",
      "cost at317 iteration: 0.11880108143861305\n",
      "cost at318 iteration: 0.11834641677073078\n",
      "cost at319 iteration: 0.11789536851286199\n",
      "cost at320 iteration: 0.11744796747354953\n",
      "cost at321 iteration: 0.11700403677990741\n",
      "cost at322 iteration: 0.11656355792402351\n",
      "cost at323 iteration: 0.11612651471926073\n",
      "cost at324 iteration: 0.11569283804455947\n",
      "cost at325 iteration: 0.11526256511029019\n",
      "cost at326 iteration: 0.11483552961135592\n",
      "cost at327 iteration: 0.11441174635184775\n",
      "cost at328 iteration: 0.11399120892578384\n",
      "cost at329 iteration: 0.11357379784693465\n",
      "cost at330 iteration: 0.1131595309270762\n",
      "cost at331 iteration: 0.11274840744485216\n",
      "cost at332 iteration: 0.11234034593702566\n",
      "cost at333 iteration: 0.11193535658043166\n",
      "cost at334 iteration: 0.11153340687343091\n",
      "cost at335 iteration: 0.11113444324124305\n",
      "cost at336 iteration: 0.11073846263367591\n",
      "cost at337 iteration: 0.11034540976974938\n",
      "cost at338 iteration: 0.1099552119460691\n",
      "cost at339 iteration: 0.10956790235031874\n",
      "cost at340 iteration: 0.10918341876070482\n",
      "cost at341 iteration: 0.10880176454554358\n",
      "cost at342 iteration: 0.108422899278114\n",
      "cost at343 iteration: 0.1080468114033516\n",
      "cost at344 iteration: 0.10767345428063672\n",
      "cost at345 iteration: 0.10730275910493589\n",
      "cost at346 iteration: 0.10693471890330006\n",
      "cost at347 iteration: 0.10656931267719877\n",
      "cost at348 iteration: 0.10620654204360372\n",
      "cost at349 iteration: 0.1058461213058649\n",
      "cost at350 iteration: 0.10548822735591458\n",
      "cost at351 iteration: 0.1051329061889245\n",
      "cost at352 iteration: 0.10478008578339694\n",
      "cost at353 iteration: 0.10442972360369904\n",
      "cost at354 iteration: 0.10408183853733015\n",
      "cost at355 iteration: 0.10373640413139315\n",
      "cost at356 iteration: 0.10339334042490692\n",
      "cost at357 iteration: 0.10305263561500345\n",
      "cost at358 iteration: 0.10271429937578076\n",
      "cost at359 iteration: 0.10237827000796251\n",
      "cost at360 iteration: 0.10204454870900079\n",
      "cost at361 iteration: 0.10171310835957244\n",
      "cost at362 iteration: 0.10138391154241833\n",
      "cost at363 iteration: 0.10105696807056949\n",
      "cost at364 iteration: 0.10073226769016344\n",
      "cost at365 iteration: 0.10040973772594847\n",
      "cost at366 iteration: 0.100089347704728\n",
      "cost at367 iteration: 0.09977112847850562\n",
      "cost at368 iteration: 0.09945505149018115\n",
      "cost at369 iteration: 0.09914105580998032\n",
      "cost at370 iteration: 0.09882914498371326\n",
      "cost at371 iteration: 0.09851928877932442\n",
      "cost at372 iteration: 0.0982115114668837\n",
      "cost at373 iteration: 0.0979057775999113\n",
      "cost at374 iteration: 0.09760201852987921\n",
      "cost at375 iteration: 0.09730024912435224\n",
      "cost at376 iteration: 0.0970004495165958\n",
      "cost at377 iteration: 0.09670258922311793\n",
      "cost at378 iteration: 0.09640665410772657\n",
      "cost at379 iteration: 0.09611263625182709\n",
      "cost at380 iteration: 0.09582052015966697\n",
      "cost at381 iteration: 0.09553025407081271\n",
      "cost at382 iteration: 0.09524185545715988\n",
      "cost at383 iteration: 0.09495529608433276\n",
      "cost at384 iteration: 0.09467057033817472\n",
      "cost at385 iteration: 0.09438765656675509\n",
      "cost at386 iteration: 0.09410653184320399\n",
      "cost at387 iteration: 0.09382718655429308\n",
      "cost at388 iteration: 0.09354961302879444\n",
      "cost at389 iteration: 0.09327377358361702\n",
      "cost at390 iteration: 0.09299966414547906\n",
      "cost at391 iteration: 0.09272727930071005\n",
      "cost at392 iteration: 0.0924565713312624\n",
      "cost at393 iteration: 0.09218754697033692\n",
      "cost at394 iteration: 0.09192017756623506\n",
      "cost at395 iteration: 0.09165445860749821\n",
      "cost at396 iteration: 0.09139036270848042\n",
      "cost at397 iteration: 0.09112788719918749\n",
      "cost at398 iteration: 0.09086701132190203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at399 iteration: 0.09060774732415892\n",
      "cost at400 iteration: 0.09035004837463126\n",
      "cost at401 iteration: 0.09009386972206464\n",
      "cost at402 iteration: 0.08983924262855303\n",
      "cost at403 iteration: 0.08958610804007147\n",
      "cost at404 iteration: 0.08933453084192607\n",
      "cost at405 iteration: 0.08908448381631767\n",
      "cost at406 iteration: 0.08883591817000622\n",
      "cost at407 iteration: 0.08858878413629775\n",
      "cost at408 iteration: 0.08834311625957571\n",
      "cost at409 iteration: 0.08809889395467786\n",
      "cost at410 iteration: 0.08785610369420378\n",
      "cost at411 iteration: 0.08761471468912373\n",
      "cost at412 iteration: 0.0873747458011862\n",
      "cost at413 iteration: 0.08713616703845899\n",
      "cost at414 iteration: 0.08689897385195805\n",
      "cost at415 iteration: 0.0866631584893171\n",
      "cost at416 iteration: 0.08642870238217948\n",
      "cost at417 iteration: 0.08619559946073416\n",
      "cost at418 iteration: 0.08596383102666597\n",
      "cost at419 iteration: 0.08573337503249137\n",
      "cost at420 iteration: 0.08550423643660235\n",
      "cost at421 iteration: 0.08527638698741193\n",
      "cost at422 iteration: 0.08504983312403741\n",
      "cost at423 iteration: 0.08482456154442705\n",
      "cost at424 iteration: 0.08460056876022215\n",
      "cost at425 iteration: 0.08437783856874416\n",
      "cost at426 iteration: 0.08415635199949363\n",
      "cost at427 iteration: 0.08393607002201245\n",
      "cost at428 iteration: 0.08371703678533327\n",
      "cost at429 iteration: 0.08349923421880967\n",
      "cost at430 iteration: 0.08328259125263178\n",
      "cost at431 iteration: 0.08306718389048783\n",
      "cost at432 iteration: 0.08285294609536\n",
      "cost at433 iteration: 0.08263986566021055\n",
      "cost at434 iteration: 0.08242797986621027\n",
      "cost at435 iteration: 0.08221724417290098\n",
      "cost at436 iteration: 0.08200765347548906\n",
      "cost at437 iteration: 0.08179920495994004\n",
      "cost at438 iteration: 0.08159187344591765\n",
      "cost at439 iteration: 0.08138565804093323\n",
      "cost at440 iteration: 0.08118053993017575\n",
      "cost at441 iteration: 0.08097649624498097\n",
      "cost at442 iteration: 0.0807735589175996\n",
      "cost at443 iteration: 0.08057170032318137\n",
      "cost at444 iteration: 0.08037089262140744\n",
      "cost at445 iteration: 0.08017117585504584\n",
      "cost at446 iteration: 0.07997252083719594\n",
      "cost at447 iteration: 0.07977489851183348\n",
      "cost at448 iteration: 0.0795783150373612\n",
      "cost at449 iteration: 0.07938274261064458\n",
      "cost at450 iteration: 0.07918818041751578\n",
      "cost at451 iteration: 0.07899464821051956\n",
      "cost at452 iteration: 0.07880213054865948\n",
      "cost at453 iteration: 0.07861060251291571\n",
      "cost at454 iteration: 0.0784200668103314\n",
      "cost at455 iteration: 0.07823049716421478\n",
      "cost at456 iteration: 0.07804190725723048\n",
      "cost at457 iteration: 0.07785429974225741\n",
      "cost at458 iteration: 0.07766764002953123\n",
      "cost at459 iteration: 0.07748192830899553\n",
      "cost at460 iteration: 0.07729717256285117\n",
      "cost at461 iteration: 0.07711332202191389\n",
      "cost at462 iteration: 0.07693040628181494\n",
      "cost at463 iteration: 0.0767484290970292\n",
      "cost at464 iteration: 0.07656735706332637\n",
      "cost at465 iteration: 0.07638719241014454\n",
      "cost at466 iteration: 0.07620794976042719\n",
      "cost at467 iteration: 0.07602958721227877\n",
      "cost at468 iteration: 0.07585209893875137\n",
      "cost at469 iteration: 0.07567547122467468\n",
      "cost at470 iteration: 0.0754997138520301\n",
      "cost at471 iteration: 0.0753248338238226\n",
      "cost at472 iteration: 0.07515079659145257\n",
      "cost at473 iteration: 0.07497763372744962\n",
      "cost at474 iteration: 0.07480531440129602\n",
      "cost at475 iteration: 0.07463382252575136\n",
      "cost at476 iteration: 0.0744632014187896\n",
      "cost at477 iteration: 0.07429337481987722\n",
      "cost at478 iteration: 0.07412437906943503\n",
      "cost at479 iteration: 0.07395619707562032\n",
      "cost at480 iteration: 0.07378879153231672\n",
      "cost at481 iteration: 0.07362222032475473\n",
      "cost at482 iteration: 0.07345642715462937\n",
      "cost at483 iteration: 0.07329141092191291\n",
      "cost at484 iteration: 0.07312718279521371\n",
      "cost at485 iteration: 0.07296373360233481\n",
      "cost at486 iteration: 0.07280105669290736\n",
      "cost at487 iteration: 0.07263912411896097\n",
      "cost at488 iteration: 0.07247793984409386\n",
      "cost at489 iteration: 0.07231753011796843\n",
      "cost at490 iteration: 0.0721578453475532\n",
      "cost at491 iteration: 0.0719989288374005\n",
      "cost at492 iteration: 0.07184072966140413\n",
      "cost at493 iteration: 0.07168325854025523\n",
      "cost at494 iteration: 0.07152653447630514\n",
      "cost at495 iteration: 0.071370521927323\n",
      "cost at496 iteration: 0.0712152344643751\n",
      "cost at497 iteration: 0.07106064635058641\n",
      "cost at498 iteration: 0.07090676091291985\n",
      "cost at499 iteration: 0.07075360516214628\n",
      "cost at500 iteration: 0.07060111761379742\n",
      "cost at501 iteration: 0.07044931509996856\n",
      "cost at502 iteration: 0.07029820335745719\n",
      "cost at503 iteration: 0.07014775783881068\n",
      "cost at504 iteration: 0.06999799793903494\n",
      "cost at505 iteration: 0.06984889874554777\n",
      "cost at506 iteration: 0.06970048732884569\n",
      "cost at507 iteration: 0.06955273852420636\n",
      "cost at508 iteration: 0.06940564517882081\n",
      "cost at509 iteration: 0.06925922197205044\n",
      "cost at510 iteration: 0.06911341996887788\n",
      "cost at511 iteration: 0.06896828122272947\n",
      "cost at512 iteration: 0.06882376417011103\n",
      "cost at513 iteration: 0.06867988907784307\n",
      "cost at514 iteration: 0.06853665053845027\n",
      "cost at515 iteration: 0.068394027025194\n",
      "cost at516 iteration: 0.06825205640759142\n",
      "cost at517 iteration: 0.06811067069027939\n",
      "cost at518 iteration: 0.06796990573164936\n",
      "cost at519 iteration: 0.0678297466055119\n",
      "cost at520 iteration: 0.06769018627612042\n",
      "cost at521 iteration: 0.06755122799485368\n",
      "cost at522 iteration: 0.06741283701774167\n",
      "cost at523 iteration: 0.0672750628144764\n",
      "cost at524 iteration: 0.06713785398223876\n",
      "cost at525 iteration: 0.06700123283190386\n",
      "cost at526 iteration: 0.06686518589218343\n",
      "cost at527 iteration: 0.06672970609489434\n",
      "cost at528 iteration: 0.06659480510478927\n",
      "cost at529 iteration: 0.06646046253169079\n",
      "cost at530 iteration: 0.06632670389292351\n",
      "cost at531 iteration: 0.06619349941278503\n",
      "cost at532 iteration: 0.06606086529195274\n",
      "cost at533 iteration: 0.0659287682754815\n",
      "cost at534 iteration: 0.06579722793557985\n",
      "cost at535 iteration: 0.06566623668209803\n",
      "cost at536 iteration: 0.06553578069575214\n",
      "cost at537 iteration: 0.06540586862337586\n",
      "cost at538 iteration: 0.06527648293529235\n",
      "cost at539 iteration: 0.0651476450783992\n",
      "cost at540 iteration: 0.06501930947593429\n",
      "cost at541 iteration: 0.06489152396285562\n",
      "cost at542 iteration: 0.06476423755252668\n",
      "cost at543 iteration: 0.06463746890732301\n",
      "cost at544 iteration: 0.06451121029823105\n",
      "cost at545 iteration: 0.06438545911926902\n",
      "cost at546 iteration: 0.06426021723038065\n",
      "cost at547 iteration: 0.06413547010851366\n",
      "cost at548 iteration: 0.06401121853411225\n",
      "cost at549 iteration: 0.06388745796888536\n",
      "cost at550 iteration: 0.0637642029275656\n",
      "cost at551 iteration: 0.06364142004730748\n",
      "cost at552 iteration: 0.06351914123992006\n",
      "cost at553 iteration: 0.06339732868861672\n",
      "cost at554 iteration: 0.06327599900374933\n",
      "cost at555 iteration: 0.06315515830759323\n",
      "cost at556 iteration: 0.06303477923999279\n",
      "cost at557 iteration: 0.06291489894725769\n",
      "cost at558 iteration: 0.06279546636447941\n",
      "cost at559 iteration: 0.06267651937706974\n",
      "cost at560 iteration: 0.0625580266125912\n",
      "cost at561 iteration: 0.0624400034478973\n",
      "cost at562 iteration: 0.06232245949820269\n",
      "cost at563 iteration: 0.06220535590420521\n",
      "cost at564 iteration: 0.06208872142972338\n",
      "cost at565 iteration: 0.0619725250671025\n",
      "cost at566 iteration: 0.061856762768679484\n",
      "cost at567 iteration: 0.06174147011893443\n",
      "cost at568 iteration: 0.06162658923258909\n",
      "cost at569 iteration: 0.06151216971613241\n",
      "cost at570 iteration: 0.06139817595253541\n",
      "cost at571 iteration: 0.06128459594282224\n",
      "cost at572 iteration: 0.06117146709728496\n",
      "cost at573 iteration: 0.061058739736359745\n",
      "cost at574 iteration: 0.06094644984585519\n",
      "cost at575 iteration: 0.06083456932680118\n",
      "cost at576 iteration: 0.06072310390877589\n",
      "cost at577 iteration: 0.06061207158313083\n",
      "cost at578 iteration: 0.06050143044854536\n",
      "cost at579 iteration: 0.06039120077227679\n",
      "cost at580 iteration: 0.06028139315245817\n",
      "cost at581 iteration: 0.060171985642175276\n",
      "cost at582 iteration: 0.060063011627827775\n",
      "cost at583 iteration: 0.05995440939211736\n",
      "cost at584 iteration: 0.05984621246285863\n",
      "cost at585 iteration: 0.0597384248017975\n",
      "cost at586 iteration: 0.05963101284517603\n",
      "cost at587 iteration: 0.059524009846428576\n",
      "cost at588 iteration: 0.059417381223717446\n",
      "cost at589 iteration: 0.0593111597331116\n",
      "cost at590 iteration: 0.059205324463366585\n",
      "cost at591 iteration: 0.05909986034253775\n",
      "cost at592 iteration: 0.0589947941921865\n",
      "cost at593 iteration: 0.05889009302275527\n",
      "cost at594 iteration: 0.0587857632324758\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at595 iteration: 0.05868183213205361\n",
      "cost at596 iteration: 0.05857824694643204\n",
      "cost at597 iteration: 0.058475038516710355\n",
      "cost at598 iteration: 0.05837219036058143\n",
      "cost at599 iteration: 0.058269702005052305\n",
      "cost at600 iteration: 0.05816760469779933\n",
      "cost at601 iteration: 0.05806584646347794\n",
      "cost at602 iteration: 0.057964452641360555\n",
      "cost at603 iteration: 0.057863418915906725\n",
      "cost at604 iteration: 0.05776273489172592\n",
      "cost at605 iteration: 0.05766243211912429\n",
      "cost at606 iteration: 0.057562454059698794\n",
      "cost at607 iteration: 0.05746284045367401\n",
      "cost at608 iteration: 0.05736357121247951\n",
      "cost at609 iteration: 0.05726463591711217\n",
      "cost at610 iteration: 0.0571660689549631\n",
      "cost at611 iteration: 0.05706783406081316\n",
      "cost at612 iteration: 0.05696995528210246\n",
      "cost at613 iteration: 0.056872412805227264\n",
      "cost at614 iteration: 0.05677519521750484\n",
      "cost at615 iteration: 0.0566783269419627\n",
      "cost at616 iteration: 0.056581767559375724\n",
      "cost at617 iteration: 0.05648555738052818\n",
      "cost at618 iteration: 0.05638966118795952\n",
      "cost at619 iteration: 0.056294093396060466\n",
      "cost at620 iteration: 0.05619885908243543\n",
      "cost at621 iteration: 0.05610392356261185\n",
      "cost at622 iteration: 0.05600932801302041\n",
      "cost at623 iteration: 0.05591503368143644\n",
      "cost at624 iteration: 0.05582107558049875\n",
      "cost at625 iteration: 0.055727417940656006\n",
      "cost at626 iteration: 0.05563407936390905\n",
      "cost at627 iteration: 0.05554106040084678\n",
      "cost at628 iteration: 0.055448329357058936\n",
      "cost at629 iteration: 0.05535593794633774\n",
      "cost at630 iteration: 0.05526383355879018\n",
      "cost at631 iteration: 0.05517203543992534\n",
      "cost at632 iteration: 0.055080544048874434\n",
      "cost at633 iteration: 0.054989340058639834\n",
      "cost at634 iteration: 0.05489846259573684\n",
      "cost at635 iteration: 0.05480785440031505\n",
      "cost at636 iteration: 0.05471756291875913\n",
      "cost at637 iteration: 0.05462755683502427\n",
      "cost at638 iteration: 0.05453784041874071\n",
      "cost at639 iteration: 0.05444842218811554\n",
      "cost at640 iteration: 0.05435928592625709\n",
      "cost at641 iteration: 0.05427045823484743\n",
      "cost at642 iteration: 0.054181891797711844\n",
      "cost at643 iteration: 0.05409363468102388\n",
      "cost at644 iteration: 0.05400564438334318\n",
      "cost at645 iteration: 0.05391793920929576\n",
      "cost at646 iteration: 0.05383052156237627\n",
      "cost at647 iteration: 0.053743376275127076\n",
      "cost at648 iteration: 0.05365652321630768\n",
      "cost at649 iteration: 0.05356994065538187\n",
      "cost at650 iteration: 0.053483656047066924\n",
      "cost at651 iteration: 0.0533976259334188\n",
      "cost at652 iteration: 0.05331187528041125\n",
      "cost at653 iteration: 0.053226405456830175\n",
      "cost at654 iteration: 0.05314118783078005\n",
      "cost at655 iteration: 0.05305625620042876\n",
      "cost at656 iteration: 0.05297157888134241\n",
      "cost at657 iteration: 0.052887185777190426\n",
      "cost at658 iteration: 0.052803041160277925\n",
      "cost at659 iteration: 0.05271918028240799\n",
      "cost at660 iteration: 0.05263556864570311\n",
      "cost at661 iteration: 0.05255221911149405\n",
      "cost at662 iteration: 0.05246915100189741\n",
      "cost at663 iteration: 0.0523863193006517\n",
      "cost at664 iteration: 0.05230376200115046\n",
      "cost at665 iteration: 0.0522214528392913\n",
      "cost at666 iteration: 0.052139407325080156\n",
      "cost at667 iteration: 0.05205761332021609\n",
      "cost at668 iteration: 0.051976085366834734\n",
      "cost at669 iteration: 0.051894783665205724\n",
      "cost at670 iteration: 0.05181375611966118\n",
      "cost at671 iteration: 0.051732962838468524\n",
      "cost at672 iteration: 0.05165242993886257\n",
      "cost at673 iteration: 0.051572131927960445\n",
      "cost at674 iteration: 0.05149209808153162\n",
      "cost at675 iteration: 0.05141228674913424\n",
      "cost at676 iteration: 0.051332741029351696\n",
      "cost at677 iteration: 0.05125342391950923\n",
      "cost at678 iteration: 0.051174368177519886\n",
      "cost at679 iteration: 0.051095533349738424\n",
      "cost at680 iteration: 0.05101695598249124\n",
      "cost at681 iteration: 0.05093859323854394\n",
      "cost at682 iteration: 0.05086049511832592\n",
      "cost at683 iteration: 0.050782610903525024\n",
      "cost at684 iteration: 0.05070497212792075\n",
      "cost at685 iteration: 0.050627560945329615\n",
      "cost at686 iteration: 0.0505504005970617\n",
      "cost at687 iteration: 0.05047344656308228\n",
      "cost at688 iteration: 0.05039674294459838\n",
      "cost at689 iteration: 0.05032026396157544\n",
      "cost at690 iteration: 0.05024401938879553\n",
      "cost at691 iteration: 0.05016800454197075\n",
      "cost at692 iteration: 0.05009221933782849\n",
      "cost at693 iteration: 0.05001665251519708\n",
      "cost at694 iteration: 0.04994132465882532\n",
      "cost at695 iteration: 0.04986621366295764\n",
      "cost at696 iteration: 0.04979131288980923\n",
      "cost at697 iteration: 0.04971665798447165\n",
      "cost at698 iteration: 0.04964221572075877\n",
      "cost at699 iteration: 0.04956799384850604\n",
      "cost at700 iteration: 0.04949399340137086\n",
      "cost at701 iteration: 0.04942021961724215\n",
      "cost at702 iteration: 0.04934665664764882\n",
      "cost at703 iteration: 0.049273328028470675\n",
      "cost at704 iteration: 0.04920019406989503\n",
      "cost at705 iteration: 0.04912728022272951\n",
      "cost at706 iteration: 0.04905458307190023\n",
      "cost at707 iteration: 0.048982101397287384\n",
      "cost at708 iteration: 0.04890980925198076\n",
      "cost at709 iteration: 0.04883776149824151\n",
      "cost at710 iteration: 0.04876589611686952\n",
      "cost at711 iteration: 0.048694256832347356\n",
      "cost at712 iteration: 0.04862281365666382\n",
      "cost at713 iteration: 0.04855158733638244\n",
      "cost at714 iteration: 0.048480552788093365\n",
      "cost at715 iteration: 0.04840973929703918\n",
      "cost at716 iteration: 0.04833910062140685\n",
      "cost at717 iteration: 0.048268704399967416\n",
      "cost at718 iteration: 0.048198482091198626\n",
      "cost at719 iteration: 0.048128471174610096\n",
      "cost at720 iteration: 0.04805866306484558\n",
      "cost at721 iteration: 0.047989043974441835\n",
      "cost at722 iteration: 0.04791964097565485\n",
      "cost at723 iteration: 0.047850423146960384\n",
      "cost at724 iteration: 0.04778141625349737\n",
      "cost at725 iteration: 0.04771259166200763\n",
      "cost at726 iteration: 0.047643976802748504\n",
      "cost at727 iteration: 0.047575540194979594\n",
      "cost at728 iteration: 0.04750731310222248\n",
      "cost at729 iteration: 0.047439265804198286\n",
      "cost at730 iteration: 0.04737140937526134\n",
      "cost at731 iteration: 0.047303760183600746\n",
      "cost at732 iteration: 0.04723627204063268\n",
      "cost at733 iteration: 0.04716900361359062\n",
      "cost at734 iteration: 0.047101889972371636\n",
      "cost at735 iteration: 0.047034992181410044\n",
      "cost at736 iteration: 0.04696826678411412\n",
      "cost at737 iteration: 0.046901723827822046\n",
      "cost at738 iteration: 0.046835372822067754\n",
      "cost at739 iteration: 0.04676920115916117\n",
      "cost at740 iteration: 0.046703218768075\n",
      "cost at741 iteration: 0.04663740805323393\n",
      "cost at742 iteration: 0.04657179107933342\n",
      "cost at743 iteration: 0.0465063422776642\n",
      "cost at744 iteration: 0.046441093832802476\n",
      "cost at745 iteration: 0.04637600826502495\n",
      "cost at746 iteration: 0.046311112057127166\n",
      "cost at747 iteration: 0.04624638432473495\n",
      "cost at748 iteration: 0.04618184161368632\n",
      "cost at749 iteration: 0.04611748176190709\n",
      "cost at750 iteration: 0.04605327345311267\n",
      "cost at751 iteration: 0.04598927479182415\n",
      "cost at752 iteration: 0.04592542343825369\n",
      "cost at753 iteration: 0.045861760396812126\n",
      "cost at754 iteration: 0.045798263071033754\n",
      "cost at755 iteration: 0.04573493353159059\n",
      "cost at756 iteration: 0.04567179536629627\n",
      "cost at757 iteration: 0.04560880604021019\n",
      "cost at758 iteration: 0.04554601856307024\n",
      "cost at759 iteration: 0.04548336793992511\n",
      "cost at760 iteration: 0.04542090607469748\n",
      "cost at761 iteration: 0.04535861023736148\n",
      "cost at762 iteration: 0.04529647527469979\n",
      "cost at763 iteration: 0.04523451973820431\n",
      "cost at764 iteration: 0.04517271367529734\n",
      "cost at765 iteration: 0.04511109470580777\n",
      "cost at766 iteration: 0.045049625056855565\n",
      "cost at767 iteration: 0.04498833345198674\n",
      "cost at768 iteration: 0.04492719384441646\n",
      "cost at769 iteration: 0.04486622815416244\n",
      "cost at770 iteration: 0.044805418878840646\n",
      "cost at771 iteration: 0.04474477720762371\n",
      "cost at772 iteration: 0.04468429823012759\n",
      "cost at773 iteration: 0.04462396538219244\n",
      "cost at774 iteration: 0.04456381911462199\n",
      "cost at775 iteration: 0.04450380536025404\n",
      "cost at776 iteration: 0.04444396625635355\n",
      "cost at777 iteration: 0.044384277140481584\n",
      "cost at778 iteration: 0.044324743142978906\n",
      "cost at779 iteration: 0.044265383349563134\n",
      "cost at780 iteration: 0.04420615138951726\n",
      "cost at781 iteration: 0.044147109483025034\n",
      "cost at782 iteration: 0.04408818741930859\n",
      "cost at783 iteration: 0.044029443356050885\n",
      "cost at784 iteration: 0.04397084890005926\n",
      "cost at785 iteration: 0.04391239417512214\n",
      "cost at786 iteration: 0.04385411929346391\n",
      "cost at787 iteration: 0.04379596490800368\n",
      "cost at788 iteration: 0.043737992002871894\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at789 iteration: 0.04368015390757771\n",
      "cost at790 iteration: 0.04362246144371396\n",
      "cost at791 iteration: 0.04356493344392288\n",
      "cost at792 iteration: 0.04350745505007462\n",
      "cost at793 iteration: 0.04345014644012037\n",
      "cost at794 iteration: 0.043392968329587715\n",
      "cost at795 iteration: 0.04333594633260667\n",
      "cost at796 iteration: 0.04327907650068737\n",
      "cost at797 iteration: 0.04322234263453173\n",
      "cost at798 iteration: 0.043165773677774376\n",
      "cost at799 iteration: 0.04310933751345258\n",
      "cost at800 iteration: 0.04305303975935939\n",
      "cost at801 iteration: 0.04299691163698608\n",
      "cost at802 iteration: 0.04294090640422932\n",
      "cost at803 iteration: 0.04288506524415643\n",
      "cost at804 iteration: 0.04282935695459977\n",
      "cost at805 iteration: 0.042773784320674815\n",
      "cost at806 iteration: 0.04271838474080526\n",
      "cost at807 iteration: 0.04266309069477301\n",
      "cost at808 iteration: 0.04260796850633407\n",
      "cost at809 iteration: 0.04255297415586328\n",
      "cost at810 iteration: 0.04249811145937533\n",
      "cost at811 iteration: 0.042443417409942896\n",
      "cost at812 iteration: 0.04238882932001171\n",
      "cost at813 iteration: 0.04233439970003105\n",
      "cost at814 iteration: 0.042280114210545526\n",
      "cost at815 iteration: 0.04222594690687221\n",
      "cost at816 iteration: 0.042171936119849894\n",
      "cost at817 iteration: 0.042118053311938776\n",
      "cost at818 iteration: 0.042064301886840745\n",
      "cost at819 iteration: 0.04201070181358065\n",
      "cost at820 iteration: 0.0419572273107216\n",
      "cost at821 iteration: 0.041903884329130905\n",
      "cost at822 iteration: 0.04185068843214752\n",
      "cost at823 iteration: 0.04179761084328728\n",
      "cost at824 iteration: 0.04174467907702381\n",
      "cost at825 iteration: 0.041691878108537216\n",
      "cost at826 iteration: 0.04163919086431304\n",
      "cost at827 iteration: 0.04158666644445634\n",
      "cost at828 iteration: 0.04153425178411196\n",
      "cost at829 iteration: 0.041481980076709585\n",
      "cost at830 iteration: 0.04142983599008054\n",
      "cost at831 iteration: 0.04137781300637927\n",
      "cost at832 iteration: 0.04132593900488394\n",
      "cost at833 iteration: 0.04127417320903008\n",
      "cost at834 iteration: 0.04122254485793903\n",
      "cost at835 iteration: 0.04117105191180894\n",
      "cost at836 iteration: 0.04111966944360971\n",
      "cost at837 iteration: 0.041068437415699356\n",
      "cost at838 iteration: 0.04101731428960772\n",
      "cost at839 iteration: 0.04096631559347615\n",
      "cost at840 iteration: 0.0409154648413109\n",
      "cost at841 iteration: 0.04086470868943976\n",
      "cost at842 iteration: 0.04081411215712321\n",
      "cost at843 iteration: 0.04076361141804489\n",
      "cost at844 iteration: 0.04071324212851156\n",
      "cost at845 iteration: 0.04066300113002169\n",
      "cost at846 iteration: 0.04061286973105691\n",
      "cost at847 iteration: 0.04056287920936414\n",
      "cost at848 iteration: 0.0405129906880354\n",
      "cost at849 iteration: 0.04046322487798029\n",
      "cost at850 iteration: 0.04041359850099003\n",
      "cost at851 iteration: 0.0403640779033574\n",
      "cost at852 iteration: 0.04031469023118237\n",
      "cost at853 iteration: 0.04026541505055688\n",
      "cost at854 iteration: 0.04021624869949268\n",
      "cost at855 iteration: 0.04016723083528319\n",
      "cost at856 iteration: 0.040118295475570616\n",
      "cost at857 iteration: 0.04006950831594819\n",
      "cost at858 iteration: 0.0400208165870066\n",
      "cost at859 iteration: 0.039972248337746985\n",
      "cost at860 iteration: 0.0399238090885466\n",
      "cost at861 iteration: 0.03987547162667582\n",
      "cost at862 iteration: 0.03982724017204501\n",
      "cost at863 iteration: 0.039779154903406826\n",
      "cost at864 iteration: 0.03973114874657326\n",
      "cost at865 iteration: 0.03968328725476766\n",
      "cost at866 iteration: 0.03963552002716482\n",
      "cost at867 iteration: 0.03958787190565495\n",
      "cost at868 iteration: 0.03954034494287967\n",
      "cost at869 iteration: 0.0394929205195219\n",
      "cost at870 iteration: 0.039445603080297965\n",
      "cost at871 iteration: 0.03939841594850564\n",
      "cost at872 iteration: 0.0393513257127575\n",
      "cost at873 iteration: 0.03930436131181611\n",
      "cost at874 iteration: 0.03925750375537118\n",
      "cost at875 iteration: 0.03921073742190669\n",
      "cost at876 iteration: 0.03916411980727625\n",
      "cost at877 iteration: 0.03911757700893209\n",
      "cost at878 iteration: 0.03907115670589679\n",
      "cost at879 iteration: 0.03902484792576054\n",
      "cost at880 iteration: 0.03897863660648954\n",
      "cost at881 iteration: 0.03893254864965688\n",
      "cost at882 iteration: 0.03888655578360795\n",
      "cost at883 iteration: 0.03884066562618032\n",
      "cost at884 iteration: 0.03879490086818793\n",
      "cost at885 iteration: 0.03874922339119118\n",
      "cost at886 iteration: 0.03870365506662483\n",
      "cost at887 iteration: 0.038658201482141115\n",
      "cost at888 iteration: 0.038612845042513416\n",
      "cost at889 iteration: 0.03856760456019074\n",
      "cost at890 iteration: 0.03852246582442684\n",
      "cost at891 iteration: 0.038477411284797695\n",
      "cost at892 iteration: 0.03843249943852691\n",
      "cost at893 iteration: 0.03838765825184078\n",
      "cost at894 iteration: 0.03834294430206438\n",
      "cost at895 iteration: 0.03829832624561144\n",
      "cost at896 iteration: 0.03825379444866721\n",
      "cost at897 iteration: 0.03820939921982242\n",
      "cost at898 iteration: 0.03816506511018446\n",
      "cost at899 iteration: 0.03812087242508737\n",
      "cost at900 iteration: 0.038076751557436866\n",
      "cost at901 iteration: 0.03803274168115779\n",
      "cost at902 iteration: 0.03798884181759159\n",
      "cost at903 iteration: 0.03794501874889237\n",
      "cost at904 iteration: 0.03790132689541938\n",
      "cost at905 iteration: 0.03785770500149759\n",
      "cost at906 iteration: 0.03781419088069086\n",
      "cost at907 iteration: 0.03777078458107735\n",
      "cost at908 iteration: 0.037727454984669284\n",
      "cost at909 iteration: 0.03768425344037753\n",
      "cost at910 iteration: 0.03764112841210892\n",
      "cost at911 iteration: 0.03759810272336797\n",
      "cost at912 iteration: 0.03755518997547656\n",
      "cost at913 iteration: 0.03751235005021624\n",
      "cost at914 iteration: 0.03746963939024046\n",
      "cost at915 iteration: 0.03742700164692835\n",
      "cost at916 iteration: 0.03738446117259393\n",
      "cost at917 iteration: 0.03734202466644859\n",
      "cost at918 iteration: 0.03729967065007922\n",
      "cost at919 iteration: 0.03725742843467365\n",
      "cost at920 iteration: 0.037215261164626416\n",
      "cost at921 iteration: 0.03717319584510358\n",
      "cost at922 iteration: 0.03713122767371807\n",
      "cost at923 iteration: 0.03708934464735663\n",
      "cost at924 iteration: 0.03704756994413038\n",
      "cost at925 iteration: 0.03700587760346099\n",
      "cost at926 iteration: 0.03696427556053754\n",
      "cost at927 iteration: 0.03692277667783515\n",
      "cost at928 iteration: 0.03688135374039054\n",
      "cost at929 iteration: 0.03684003631228976\n",
      "cost at930 iteration: 0.036798803896662666\n",
      "cost at931 iteration: 0.036757654357910625\n",
      "cost at932 iteration: 0.03671661588912686\n",
      "cost at933 iteration: 0.03667564014391613\n",
      "cost at934 iteration: 0.03663477893856902\n",
      "cost at935 iteration: 0.03659399547110098\n",
      "cost at936 iteration: 0.03655329196765534\n",
      "cost at937 iteration: 0.03651270108134136\n",
      "cost at938 iteration: 0.036472170395324806\n",
      "cost at939 iteration: 0.03643175846381368\n",
      "cost at940 iteration: 0.03639141036365887\n",
      "cost at941 iteration: 0.03635115387196934\n",
      "cost at942 iteration: 0.03631100300387565\n",
      "cost at943 iteration: 0.036270908730838235\n",
      "cost at944 iteration: 0.036230928455642894\n",
      "cost at945 iteration: 0.03619102169235348\n",
      "cost at946 iteration: 0.03615119245458095\n",
      "cost at947 iteration: 0.03611147659736283\n",
      "cost at948 iteration: 0.036071814245894125\n",
      "cost at949 iteration: 0.03603227314327325\n",
      "cost at950 iteration: 0.03599278871456635\n",
      "cost at951 iteration: 0.03595339482556844\n",
      "cost at952 iteration: 0.0359140987351669\n",
      "cost at953 iteration: 0.03587486722045538\n",
      "cost at954 iteration: 0.035835747426522256\n",
      "cost at955 iteration: 0.035796685442697325\n",
      "cost at956 iteration: 0.03575771170442042\n",
      "cost at957 iteration: 0.0357188372066004\n",
      "cost at958 iteration: 0.03568001998231043\n",
      "cost at959 iteration: 0.03564131413399408\n",
      "cost at960 iteration: 0.03560266560364257\n",
      "cost at961 iteration: 0.035564106296586184\n",
      "cost at962 iteration: 0.03552564123909194\n",
      "cost at963 iteration: 0.03548723216100442\n",
      "cost at964 iteration: 0.03544893245462978\n",
      "cost at965 iteration: 0.035410690755872165\n",
      "cost at966 iteration: 0.03537253668845081\n",
      "cost at967 iteration: 0.03533447735996521\n",
      "cost at968 iteration: 0.035296472964444055\n",
      "cost at969 iteration: 0.03525857539974801\n",
      "cost at970 iteration: 0.03522073290698487\n",
      "cost at971 iteration: 0.03518298170144258\n",
      "cost at972 iteration: 0.035145313983324634\n",
      "cost at973 iteration: 0.03510770835830814\n",
      "cost at974 iteration: 0.035070201837585205\n",
      "cost at975 iteration: 0.03503275178972567\n",
      "cost at976 iteration: 0.03499539348037194\n",
      "cost at977 iteration: 0.034958119881636024\n",
      "cost at978 iteration: 0.0349208983153844\n",
      "cost at979 iteration: 0.0348837778704187\n",
      "cost at980 iteration: 0.034846719918621746\n",
      "cost at981 iteration: 0.03480973936888671\n",
      "cost at982 iteration: 0.034772849824250614\n",
      "cost at983 iteration: 0.034736006498966285\n",
      "cost at984 iteration: 0.03469927652052109\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at985 iteration: 0.03466259173341753\n",
      "cost at986 iteration: 0.034625993493535394\n",
      "cost at987 iteration: 0.03458947907497944\n",
      "cost at988 iteration: 0.03455301756060867\n",
      "cost at989 iteration: 0.0345166651772382\n",
      "cost at990 iteration: 0.03448035168870019\n",
      "cost at991 iteration: 0.03444413200594409\n",
      "cost at992 iteration: 0.034407989904118086\n",
      "cost at993 iteration: 0.03437190228519188\n",
      "cost at994 iteration: 0.0343359185618292\n",
      "cost at995 iteration: 0.034299977921798425\n",
      "cost at996 iteration: 0.034264125810103505\n",
      "cost at997 iteration: 0.0342283506679571\n",
      "cost at998 iteration: 0.03419262580512298\n",
      "cost at999 iteration: 0.034157003237040115\n",
      "cost at1000 iteration: 0.0341214246259512\n",
      "cost at1001 iteration: 0.03408593561318233\n",
      "cost at1002 iteration: 0.034050513159571534\n",
      "cost at1003 iteration: 0.03401514628095431\n",
      "cost at1004 iteration: 0.03397988101266478\n",
      "cost at1005 iteration: 0.033944654324489204\n",
      "cost at1006 iteration: 0.03390952691643082\n",
      "cost at1007 iteration: 0.03387444654889718\n",
      "cost at1008 iteration: 0.03383944193954811\n",
      "cost at1009 iteration: 0.03380452495376108\n",
      "cost at1010 iteration: 0.03376965067466831\n",
      "cost at1011 iteration: 0.03373487631334552\n",
      "cost at1012 iteration: 0.03370013747314826\n",
      "cost at1013 iteration: 0.03366549773959044\n",
      "cost at1014 iteration: 0.033630910032608416\n",
      "cost at1015 iteration: 0.033596385781878665\n",
      "cost at1016 iteration: 0.033561949676644165\n",
      "cost at1017 iteration: 0.03352756039962318\n",
      "cost at1018 iteration: 0.033493260818902255\n",
      "cost at1019 iteration: 0.03345900867799006\n",
      "cost at1020 iteration: 0.03342483710152766\n",
      "cost at1021 iteration: 0.03339073092931679\n",
      "cost at1022 iteration: 0.03335668501832461\n",
      "cost at1023 iteration: 0.03332271804952576\n",
      "cost at1024 iteration: 0.03328880035559418\n",
      "cost at1025 iteration: 0.0332549745978636\n",
      "cost at1026 iteration: 0.033221191890620144\n",
      "cost at1027 iteration: 0.03318749284588194\n",
      "cost at1028 iteration: 0.03315384664253423\n",
      "cost at1029 iteration: 0.033120274174758726\n",
      "cost at1030 iteration: 0.03308676703379094\n",
      "cost at1031 iteration: 0.033053309680195465\n",
      "cost at1032 iteration: 0.033019944065595906\n",
      "cost at1033 iteration: 0.03298662466268863\n",
      "cost at1034 iteration: 0.03295337489546047\n",
      "cost at1035 iteration: 0.03292019218103103\n",
      "cost at1036 iteration: 0.03288706592944027\n",
      "cost at1037 iteration: 0.03285401415201223\n",
      "cost at1038 iteration: 0.032821021027557344\n",
      "cost at1039 iteration: 0.0327880821415062\n",
      "cost at1040 iteration: 0.03275523201987029\n",
      "cost at1041 iteration: 0.03272241688797771\n",
      "cost at1042 iteration: 0.03268968693728408\n",
      "cost at1043 iteration: 0.03265701157537837\n",
      "cost at1044 iteration: 0.03262438966666459\n",
      "cost at1045 iteration: 0.0325918542496885\n",
      "cost at1046 iteration: 0.03255935265723833\n",
      "cost at1047 iteration: 0.03252693904127249\n",
      "cost at1048 iteration: 0.03249457625633801\n",
      "cost at1049 iteration: 0.032462266045966304\n",
      "cost at1050 iteration: 0.03243004369600675\n",
      "cost at1051 iteration: 0.03239784999496269\n",
      "cost at1052 iteration: 0.03236574661262744\n",
      "cost at1053 iteration: 0.03233368538585761\n",
      "cost at1054 iteration: 0.03230168388126474\n",
      "cost at1055 iteration: 0.03226976483271923\n",
      "cost at1056 iteration: 0.03223787068699638\n",
      "cost at1057 iteration: 0.032206070178483634\n",
      "cost at1058 iteration: 0.03217431544999766\n",
      "cost at1059 iteration: 0.03214261152566616\n",
      "cost at1060 iteration: 0.032110988037129896\n",
      "cost at1061 iteration: 0.03207940262981878\n",
      "cost at1062 iteration: 0.03204789862977109\n",
      "cost at1063 iteration: 0.032016437569739986\n",
      "cost at1064 iteration: 0.03198503734941821\n",
      "cost at1065 iteration: 0.03195371147610142\n",
      "cost at1066 iteration: 0.03192242019093278\n",
      "cost at1067 iteration: 0.031891213007490954\n",
      "cost at1068 iteration: 0.031860037856577886\n",
      "cost at1069 iteration: 0.031828943685182666\n",
      "cost at1070 iteration: 0.031797901810445714\n",
      "cost at1071 iteration: 0.031766909876853655\n",
      "cost at1072 iteration: 0.0317359861966647\n",
      "cost at1073 iteration: 0.03170510631686073\n",
      "cost at1074 iteration: 0.031674298033537523\n",
      "cost at1075 iteration: 0.031643538751513385\n",
      "cost at1076 iteration: 0.03161283478727205\n",
      "cost at1077 iteration: 0.03158219485975046\n",
      "cost at1078 iteration: 0.0315515982505791\n",
      "cost at1079 iteration: 0.03152108249901042\n",
      "cost at1080 iteration: 0.031490593304560846\n",
      "cost at1081 iteration: 0.03146018606300549\n",
      "cost at1082 iteration: 0.03142981913598053\n",
      "cost at1083 iteration: 0.031399512037431215\n",
      "cost at1084 iteration: 0.03136926614625756\n",
      "cost at1085 iteration: 0.031339066364809105\n",
      "cost at1086 iteration: 0.031308945745261905\n",
      "cost at1087 iteration: 0.03127883765547246\n",
      "cost at1088 iteration: 0.031248825183624245\n",
      "cost at1089 iteration: 0.031218845764736724\n",
      "cost at1090 iteration: 0.031188925306564613\n",
      "cost at1091 iteration: 0.031159066939449413\n",
      "cost at1092 iteration: 0.031129240712127988\n",
      "cost at1093 iteration: 0.031099508090959502\n",
      "cost at1094 iteration: 0.031069788978076128\n",
      "cost at1095 iteration: 0.031040153483407974\n",
      "cost at1096 iteration: 0.031010557101859505\n",
      "cost at1097 iteration: 0.03098100852979679\n",
      "cost at1098 iteration: 0.03095153520607557\n",
      "cost at1099 iteration: 0.030922091339368362\n",
      "cost at1100 iteration: 0.03089272410181502\n",
      "cost at1101 iteration: 0.030863392430298456\n",
      "cost at1102 iteration: 0.030834108193558033\n",
      "cost at1103 iteration: 0.030804902433996112\n",
      "cost at1104 iteration: 0.03077571751280727\n",
      "cost at1105 iteration: 0.0307466108233766\n",
      "cost at1106 iteration: 0.03071753746337334\n",
      "cost at1107 iteration: 0.030688528619404624\n",
      "cost at1108 iteration: 0.03065956694745606\n",
      "cost at1109 iteration: 0.03063065260564855\n",
      "cost at1110 iteration: 0.030601804321032298\n",
      "cost at1111 iteration: 0.03057298799100647\n",
      "cost at1112 iteration: 0.03054423892629245\n",
      "cost at1113 iteration: 0.03051553517244404\n",
      "cost at1114 iteration: 0.030486878452721196\n",
      "cost at1115 iteration: 0.0304582786402396\n",
      "cost at1116 iteration: 0.030429729953738754\n",
      "cost at1117 iteration: 0.030401239601760714\n",
      "cost at1118 iteration: 0.030372775609661166\n",
      "cost at1119 iteration: 0.03034439231650159\n",
      "cost at1120 iteration: 0.030316031093816243\n",
      "cost at1121 iteration: 0.030287735811466352\n",
      "cost at1122 iteration: 0.030259496113094017\n",
      "cost at1123 iteration: 0.03023128317937882\n",
      "cost at1124 iteration: 0.030203154547512052\n",
      "cost at1125 iteration: 0.03017503963388004\n",
      "cost at1126 iteration: 0.030147001641546992\n",
      "cost at1127 iteration: 0.03011900428823578\n",
      "cost at1128 iteration: 0.030091049233967628\n",
      "cost at1129 iteration: 0.0300631635781387\n",
      "cost at1130 iteration: 0.03003529839971065\n",
      "cost at1131 iteration: 0.030007505321515443\n",
      "cost at1132 iteration: 0.029979742123937052\n",
      "cost at1133 iteration: 0.029952047541453926\n",
      "cost at1134 iteration: 0.029924375292191663\n",
      "cost at1135 iteration: 0.029896774585163714\n",
      "cost at1136 iteration: 0.02986921683212065\n",
      "cost at1137 iteration: 0.029841691430149534\n",
      "cost at1138 iteration: 0.02981424464149478\n",
      "cost at1139 iteration: 0.029786809012526668\n",
      "cost at1140 iteration: 0.029759453874662677\n",
      "cost at1141 iteration: 0.029732130612543685\n",
      "cost at1142 iteration: 0.02970485418461879\n",
      "cost at1143 iteration: 0.029677639358114037\n",
      "cost at1144 iteration: 0.02965044803961599\n",
      "cost at1145 iteration: 0.029623328173162537\n",
      "cost at1146 iteration: 0.029596232063582806\n",
      "cost at1147 iteration: 0.029569208901857755\n",
      "cost at1148 iteration: 0.0295422009625417\n",
      "cost at1149 iteration: 0.029515274104169858\n",
      "cost at1150 iteration: 0.02948836589865825\n",
      "cost at1151 iteration: 0.02946151454979183\n",
      "cost at1152 iteration: 0.02943471801584087\n",
      "cost at1153 iteration: 0.029407952410448763\n",
      "cost at1154 iteration: 0.0293812506674194\n",
      "cost at1155 iteration: 0.029354575503881736\n",
      "cost at1156 iteration: 0.02932796249235375\n",
      "cost at1157 iteration: 0.02930137752586677\n",
      "cost at1158 iteration: 0.02927485276386607\n",
      "cost at1159 iteration: 0.029248370065235403\n",
      "cost at1160 iteration: 0.029221921363100447\n",
      "cost at1161 iteration: 0.029195546314024965\n",
      "cost at1162 iteration: 0.029169179768323075\n",
      "cost at1163 iteration: 0.02914289462101974\n",
      "cost at1164 iteration: 0.029116625286376145\n",
      "cost at1165 iteration: 0.029090423655695478\n",
      "cost at1166 iteration: 0.029064236767850412\n",
      "cost at1167 iteration: 0.029038134987701553\n",
      "cost at1168 iteration: 0.02901202957915908\n",
      "cost at1169 iteration: 0.028986013560606504\n",
      "cost at1170 iteration: 0.028960013120102344\n",
      "cost at1171 iteration: 0.028934059092609772\n",
      "cost at1172 iteration: 0.028908166498229096\n",
      "cost at1173 iteration: 0.028882297413899274\n",
      "cost at1174 iteration: 0.028856492382481325\n",
      "cost at1175 iteration: 0.02883070727356396\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at1176 iteration: 0.02880499091807854\n",
      "cost at1177 iteration: 0.02877929524517109\n",
      "cost at1178 iteration: 0.028753658569588596\n",
      "cost at1179 iteration: 0.028728065860039773\n",
      "cost at1180 iteration: 0.028702495175472378\n",
      "cost at1181 iteration: 0.028677001470671485\n",
      "cost at1182 iteration: 0.028651515681654405\n",
      "cost at1183 iteration: 0.02862609767471289\n",
      "cost at1184 iteration: 0.028600719507957686\n",
      "cost at1185 iteration: 0.028575366980331876\n",
      "cost at1186 iteration: 0.028550071049933935\n",
      "cost at1187 iteration: 0.028524805961354083\n",
      "cost at1188 iteration: 0.028499600361431524\n",
      "cost at1189 iteration: 0.0284744109805602\n",
      "cost at1190 iteration: 0.028449291735307414\n",
      "cost at1191 iteration: 0.028424183738515584\n",
      "cost at1192 iteration: 0.028399146413065544\n",
      "cost at1193 iteration: 0.028374138421537128\n",
      "cost at1194 iteration: 0.02834917537577356\n",
      "cost at1195 iteration: 0.02832423294052164\n",
      "cost at1196 iteration: 0.028299369059598213\n",
      "cost at1197 iteration: 0.02827450978098837\n",
      "cost at1198 iteration: 0.02824971050964656\n",
      "cost at1199 iteration: 0.028224957250022185\n",
      "cost at1200 iteration: 0.028200223738258128\n",
      "cost at1201 iteration: 0.028175561031639904\n",
      "cost at1202 iteration: 0.028150916201754753\n",
      "cost at1203 iteration: 0.028126321582817332\n",
      "cost at1204 iteration: 0.028101755498971186\n",
      "cost at1205 iteration: 0.028077249865168\n",
      "cost at1206 iteration: 0.02805276125647022\n",
      "cost at1207 iteration: 0.028028328351847438\n",
      "cost at1208 iteration: 0.028003934279178074\n",
      "cost at1209 iteration: 0.027979564576200662\n",
      "cost at1210 iteration: 0.027955267886602103\n",
      "cost at1211 iteration: 0.02793097276525377\n",
      "cost at1212 iteration: 0.027906744944939506\n",
      "cost at1213 iteration: 0.02788253875561566\n",
      "cost at1214 iteration: 0.02785838664728444\n",
      "cost at1215 iteration: 0.027834246645481037\n",
      "cost at1216 iteration: 0.027810183261234928\n",
      "cost at1217 iteration: 0.027786125222373375\n",
      "cost at1218 iteration: 0.027762121954146183\n",
      "cost at1219 iteration: 0.027738164917036786\n",
      "cost at1220 iteration: 0.02771422045101618\n",
      "cost at1221 iteration: 0.027690356213448987\n",
      "cost at1222 iteration: 0.027666492552662635\n",
      "cost at1223 iteration: 0.02764269238096527\n",
      "cost at1224 iteration: 0.027618906510357154\n",
      "cost at1225 iteration: 0.027595183372798194\n",
      "cost at1226 iteration: 0.027571469395452738\n",
      "cost at1227 iteration: 0.027547825621925665\n",
      "cost at1228 iteration: 0.027524200604779833\n",
      "cost at1229 iteration: 0.027500612912900006\n",
      "cost at1230 iteration: 0.02747708286590137\n",
      "cost at1231 iteration: 0.02745356239066513\n",
      "cost at1232 iteration: 0.02743010355940037\n",
      "cost at1233 iteration: 0.027406661555155785\n",
      "cost at1234 iteration: 0.02738327333592377\n",
      "cost at1235 iteration: 0.02735990023883606\n",
      "cost at1236 iteration: 0.02733660041475008\n",
      "cost at1237 iteration: 0.027313300745049257\n",
      "cost at1238 iteration: 0.027290066309460095\n",
      "cost at1239 iteration: 0.02726684746928706\n",
      "cost at1240 iteration: 0.02724368501843887\n",
      "cost at1241 iteration: 0.027220536529722766\n",
      "cost at1242 iteration: 0.027197443695227478\n",
      "cost at1243 iteration: 0.027174367974278035\n",
      "cost at1244 iteration: 0.027151357926625874\n",
      "cost at1245 iteration: 0.02712835599687777\n",
      "cost at1246 iteration: 0.027105411914124397\n",
      "cost at1247 iteration: 0.027082480776066124\n",
      "cost at1248 iteration: 0.02705961527725656\n",
      "cost at1249 iteration: 0.02703675107043915\n",
      "cost at1250 iteration: 0.027013953842819\n",
      "cost at1251 iteration: 0.026991165733815225\n",
      "cost at1252 iteration: 0.026968439052698245\n",
      "cost at1253 iteration: 0.02694573699447042\n",
      "cost at1254 iteration: 0.026923067591461198\n",
      "cost at1255 iteration: 0.026900437912487557\n",
      "cost at1256 iteration: 0.026877836329482718\n",
      "cost at1257 iteration: 0.026855291046727615\n",
      "cost at1258 iteration: 0.026832748657838162\n",
      "cost at1259 iteration: 0.026810272639422136\n",
      "cost at1260 iteration: 0.026787811172473128\n",
      "cost at1261 iteration: 0.026765399793208938\n",
      "cost at1262 iteration: 0.026743014207873294\n",
      "cost at1263 iteration: 0.026720660753534395\n",
      "cost at1264 iteration: 0.026698350368190144\n",
      "cost at1265 iteration: 0.026676064621062468\n",
      "cost at1266 iteration: 0.026653834009302425\n",
      "cost at1267 iteration: 0.026631613653058116\n",
      "cost at1268 iteration: 0.026609451973101786\n",
      "cost at1269 iteration: 0.02658729229421705\n",
      "cost at1270 iteration: 0.02656519740674516\n",
      "cost at1271 iteration: 0.026543117691292306\n",
      "cost at1272 iteration: 0.02652108418999467\n",
      "cost at1273 iteration: 0.026499067503042387\n",
      "cost at1274 iteration: 0.026477115726737034\n",
      "cost at1275 iteration: 0.026455169535523442\n",
      "cost at1276 iteration: 0.026433260946396274\n",
      "cost at1277 iteration: 0.026411400978258393\n",
      "cost at1278 iteration: 0.026389549302701532\n",
      "cost at1279 iteration: 0.02636776169680516\n",
      "cost at1280 iteration: 0.026345985309088344\n",
      "cost at1281 iteration: 0.026324262027727464\n",
      "cost at1282 iteration: 0.0263025476184194\n",
      "cost at1283 iteration: 0.026280890054312275\n",
      "cost at1284 iteration: 0.026259241877838138\n",
      "cost at1285 iteration: 0.026237656681568723\n",
      "cost at1286 iteration: 0.02621607641508976\n",
      "cost at1287 iteration: 0.026194555473291282\n",
      "cost at1288 iteration: 0.026173042471784775\n",
      "cost at1289 iteration: 0.02615158458340213\n",
      "cost at1290 iteration: 0.02613013788356291\n",
      "cost at1291 iteration: 0.026108744128848938\n",
      "cost at1292 iteration: 0.026087365287120294\n",
      "cost at1293 iteration: 0.026066032692968102\n",
      "cost at1294 iteration: 0.02604473177855119\n",
      "cost at1295 iteration: 0.02602345200674147\n",
      "cost at1296 iteration: 0.02600222919867843\n",
      "cost at1297 iteration: 0.025981004331667147\n",
      "cost at1298 iteration: 0.025959846810147305\n",
      "cost at1299 iteration: 0.02593869321203027\n",
      "cost at1300 iteration: 0.025917589173033852\n",
      "cost at1301 iteration: 0.025896511973447736\n",
      "cost at1302 iteration: 0.02587547504546376\n",
      "cost at1303 iteration: 0.025854442674903604\n",
      "cost at1304 iteration: 0.02583347257815891\n",
      "cost at1305 iteration: 0.02581250616949474\n",
      "cost at1306 iteration: 0.0257915968353595\n",
      "cost at1307 iteration: 0.02577070827713508\n",
      "cost at1308 iteration: 0.025749845648520855\n",
      "cost at1309 iteration: 0.02572902677317331\n",
      "cost at1310 iteration: 0.025708224708129244\n",
      "cost at1311 iteration: 0.025687466651207743\n",
      "cost at1312 iteration: 0.025666724760354725\n",
      "cost at1313 iteration: 0.02564603253096756\n",
      "cost at1314 iteration: 0.025625356014479617\n",
      "cost at1315 iteration: 0.025604726877216304\n",
      "cost at1316 iteration: 0.02558410594967334\n",
      "cost at1317 iteration: 0.025563536422566036\n",
      "cost at1318 iteration: 0.025542975055138323\n",
      "cost at1319 iteration: 0.025522474730931624\n",
      "cost at1320 iteration: 0.025501981056014908\n",
      "cost at1321 iteration: 0.025481533529953367\n",
      "cost at1322 iteration: 0.0254610972235764\n",
      "cost at1323 iteration: 0.025440719413471775\n",
      "cost at1324 iteration: 0.02542033699661661\n",
      "cost at1325 iteration: 0.025400010936285203\n",
      "cost at1326 iteration: 0.025379703432975412\n",
      "cost at1327 iteration: 0.025359436984690988\n",
      "cost at1328 iteration: 0.02533918518621984\n",
      "cost at1329 iteration: 0.025318978928129423\n",
      "cost at1330 iteration: 0.025298778182135856\n",
      "cost at1331 iteration: 0.025278639344950192\n",
      "cost at1332 iteration: 0.02525850906796996\n",
      "cost at1333 iteration: 0.025238418887791762\n",
      "cost at1334 iteration: 0.025218351762401437\n",
      "cost at1335 iteration: 0.02519832320346901\n",
      "cost at1336 iteration: 0.025178301759838945\n",
      "cost at1337 iteration: 0.025158341018706724\n",
      "cost at1338 iteration: 0.02513838416123915\n",
      "cost at1339 iteration: 0.025118478954356382\n",
      "cost at1340 iteration: 0.025098584546883927\n",
      "cost at1341 iteration: 0.02507872268492447\n",
      "cost at1342 iteration: 0.025058882096161254\n",
      "cost at1343 iteration: 0.025039091260279568\n",
      "cost at1344 iteration: 0.025019302221323966\n",
      "cost at1345 iteration: 0.024999576282420805\n",
      "cost at1346 iteration: 0.02497983995461374\n",
      "cost at1347 iteration: 0.024960162179744134\n",
      "cost at1348 iteration: 0.02494049477778575\n",
      "cost at1349 iteration: 0.024920864494079074\n",
      "cost at1350 iteration: 0.024901258258305574\n",
      "cost at1351 iteration: 0.024881692699453367\n",
      "cost at1352 iteration: 0.02486213418797315\n",
      "cost at1353 iteration: 0.024842626033116284\n",
      "cost at1354 iteration: 0.024823117322248775\n",
      "cost at1355 iteration: 0.024803670628797774\n",
      "cost at1356 iteration: 0.024784227533025165\n",
      "cost at1357 iteration: 0.024764827725263033\n",
      "cost at1358 iteration: 0.024745439333935568\n",
      "cost at1359 iteration: 0.024726099945501036\n",
      "cost at1360 iteration: 0.024706760623851354\n",
      "cost at1361 iteration: 0.024687477580532198\n",
      "cost at1362 iteration: 0.024668201947126267\n",
      "cost at1363 iteration: 0.024648975267400317\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost at1364 iteration: 0.024629753707503005\n",
      "cost at1365 iteration: 0.02461057609747408\n",
      "cost at1366 iteration: 0.02459140262772298\n",
      "cost at1367 iteration: 0.024572286936126116\n",
      "cost at1368 iteration: 0.024553176343351676\n",
      "cost at1369 iteration: 0.024534107586727524\n",
      "cost at1370 iteration: 0.024515050925911393\n",
      "cost at1371 iteration: 0.024496040429472354\n",
      "cost at1372 iteration: 0.024477028247499418\n",
      "cost at1373 iteration: 0.024458079674063212\n",
      "cost at1374 iteration: 0.024439134821149507\n",
      "cost at1375 iteration: 0.02442022613070852\n",
      "cost at1376 iteration: 0.02440134292285237\n",
      "cost at1377 iteration: 0.024382483160618678\n",
      "cost at1378 iteration: 0.02436364732802329\n",
      "cost at1379 iteration: 0.024344857201581235\n",
      "cost at1380 iteration: 0.024326064309285182\n",
      "cost at1381 iteration: 0.02430733115173284\n",
      "cost at1382 iteration: 0.024288593268194124\n",
      "cost at1383 iteration: 0.024269902920427005\n",
      "cost at1384 iteration: 0.02425122036617649\n",
      "cost at1385 iteration: 0.024232594783841203\n",
      "cost at1386 iteration: 0.024213961922779116\n",
      "cost at1387 iteration: 0.02419538061806492\n",
      "cost at1388 iteration: 0.024176806359604664\n",
      "cost at1389 iteration: 0.024158267659555168\n",
      "cost at1390 iteration: 0.024139751120426173\n",
      "cost at1391 iteration: 0.024121280168750904\n",
      "cost at1392 iteration: 0.024102804569645458\n",
      "cost at1393 iteration: 0.024084380122546357\n",
      "cost at1394 iteration: 0.024065956648939447\n",
      "cost at1395 iteration: 0.024047579022826487\n",
      "cost at1396 iteration: 0.024029224896021457\n",
      "cost at1397 iteration: 0.024010899109130568\n",
      "cost at1398 iteration: 0.023992572161569085\n",
      "cost at1399 iteration: 0.023974306275604852\n",
      "cost at1400 iteration: 0.023956039363485466\n",
      "cost at1401 iteration: 0.023937815437260288\n",
      "cost at1402 iteration: 0.023919614882266182\n",
      "cost at1403 iteration: 0.023901432433047536\n",
      "cost at1404 iteration: 0.023883270094297606\n",
      "cost at1405 iteration: 0.023865153952678037\n",
      "cost at1406 iteration: 0.02384704064459578\n",
      "cost at1407 iteration: 0.023828969976332333\n",
      "cost at1408 iteration: 0.02381090769975781\n",
      "cost at1409 iteration: 0.02379288844814307\n",
      "cost at1410 iteration: 0.02377487315435817\n",
      "cost at1411 iteration: 0.023756906856701055\n",
      "cost at1412 iteration: 0.023738944656010702\n",
      "cost at1413 iteration: 0.023721022611397843\n",
      "cost at1414 iteration: 0.023703107299439863\n",
      "cost at1415 iteration: 0.02368523220774839\n",
      "cost at1416 iteration: 0.02366738042238858\n",
      "cost at1417 iteration: 0.023649553892835753\n",
      "cost at1418 iteration: 0.02363174505049932\n",
      "cost at1419 iteration: 0.023613966289772238\n",
      "cost at1420 iteration: 0.02359619535360334\n",
      "cost at1421 iteration: 0.023578478304120028\n",
      "cost at1422 iteration: 0.023560755897422705\n",
      "cost at1423 iteration: 0.02354308845699273\n",
      "cost at1424 iteration: 0.02352541274036998\n",
      "cost at1425 iteration: 0.02350778125470202\n",
      "cost at1426 iteration: 0.02349015913176705\n",
      "cost at1427 iteration: 0.02347258351795851\n",
      "cost at1428 iteration: 0.02345501157731391\n",
      "cost at1429 iteration: 0.023437480682498648\n",
      "cost at1430 iteration: 0.023419955446826627\n",
      "cost at1431 iteration: 0.02340246671166267\n",
      "cost at1432 iteration: 0.023384990955998095\n",
      "cost at1433 iteration: 0.023367554159540737\n",
      "cost at1434 iteration: 0.023350126353871157\n",
      "cost at1435 iteration: 0.023332738660640256\n",
      "cost at1436 iteration: 0.023315352068630397\n",
      "cost at1437 iteration: 0.023298016953543985\n",
      "cost at1438 iteration: 0.02328067981856277\n",
      "cost at1439 iteration: 0.023263389973905507\n",
      "cost at1440 iteration: 0.02324610032709628\n",
      "cost at1441 iteration: 0.02322884710074191\n",
      "cost at1442 iteration: 0.023211613237008915\n",
      "cost at1443 iteration: 0.0231944110019955\n",
      "cost at1444 iteration: 0.02317721822440572\n",
      "cost at1445 iteration: 0.023160063444279573\n",
      "cost at1446 iteration: 0.023142912774284357\n",
      "cost at1447 iteration: 0.023125801544623764\n",
      "cost at1448 iteration: 0.02310870467448342\n",
      "cost at1449 iteration: 0.023091639105209408\n",
      "cost at1450 iteration: 0.023074584690073705\n",
      "cost at1451 iteration: 0.02305756204969443\n",
      "cost at1452 iteration: 0.023040551151536186\n",
      "cost at1453 iteration: 0.023023579007368664\n",
      "cost at1454 iteration: 0.02300660445190279\n",
      "cost at1455 iteration: 0.022989688069788093\n",
      "cost at1456 iteration: 0.02297275527041805\n",
      "cost at1457 iteration: 0.0229558611956047\n",
      "cost at1458 iteration: 0.022938978642792575\n",
      "cost at1459 iteration: 0.022922127747746043\n",
      "cost at1460 iteration: 0.02290529031299834\n",
      "cost at1461 iteration: 0.02288848347361313\n",
      "cost at1462 iteration: 0.022871695463566214\n",
      "cost at1463 iteration: 0.022854933117869597\n",
      "cost at1464 iteration: 0.02283818530675625\n",
      "cost at1465 iteration: 0.022821480647825503\n",
      "cost at1466 iteration: 0.022804769374798733\n",
      "cost at1467 iteration: 0.02278810425348268\n",
      "cost at1468 iteration: 0.02277144261086623\n",
      "cost at1469 iteration: 0.02275482709151231\n",
      "cost at1470 iteration: 0.022738212232234045\n",
      "cost at1471 iteration: 0.022721627622479885\n",
      "cost at1472 iteration: 0.022705057997692336\n",
      "cost at1473 iteration: 0.02268852202852428\n",
      "cost at1474 iteration: 0.02267199906544409\n",
      "cost at1475 iteration: 0.022655505422166454\n",
      "cost at1476 iteration: 0.022639012481432356\n",
      "cost at1477 iteration: 0.022622564175254663\n",
      "cost at1478 iteration: 0.0226061248767128\n",
      "cost at1479 iteration: 0.02258971925006475\n",
      "cost at1480 iteration: 0.022573333746721658\n",
      "cost at1481 iteration: 0.02255696522207234\n",
      "cost at1482 iteration: 0.02254061406169136\n",
      "cost at1483 iteration: 0.022524306655800092\n",
      "cost at1484 iteration: 0.022507997297735446\n",
      "cost at1485 iteration: 0.022491728809919724\n",
      "cost at1486 iteration: 0.02247545538218854\n",
      "cost at1487 iteration: 0.02245923195915146\n",
      "cost at1488 iteration: 0.022443014563736575\n",
      "cost at1489 iteration: 0.022426818389654404\n",
      "cost at1490 iteration: 0.022410642574884194\n",
      "cost at1491 iteration: 0.02239449098706689\n",
      "cost at1492 iteration: 0.02237835458530451\n",
      "cost at1493 iteration: 0.022362247563839546\n",
      "cost at1494 iteration: 0.02234614721560061\n",
      "cost at1495 iteration: 0.022330082749362507\n",
      "cost at1496 iteration: 0.022314015060518364\n",
      "cost at1497 iteration: 0.022298007726630507\n",
      "cost at1498 iteration: 0.02228199998360127\n",
      "cost at1499 iteration: 0.02226602849998063\n",
      "train_set accuracy is=0.9963741841914431\n",
      "test_set accuracy is=1.0\n"
     ]
    }
   ],
   "source": [
    "learned_parameters=deep_learning_module(X,Y,0.1,[3000,100,10,1],1500)\n",
    "print(\"train_set accuracy is={}\".format(predict(X,learned_parameters,Y)))\n",
    "\n",
    "print(\"test_set accuracy is={}\".format(predict(devX_test,learned_parameters,devY_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}