08.3 - XOR with Wide and Deep Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Lab 8-3: XOR with Wide and Deep Neural Network\r\n",
    "\r\n",
    "Edited By Steve Ive\r\n",
    "\r\n",
    "Here, we are going to learn about how the wide and deep neural network can change the process of learning and the reducing cost of XOR problem.\r\n",
    "\r\n",
    "Reference From\r\n",
    "\r\n",
    "https://github.com/deeplearningzerotoall/PyTorch/blob/master/lab-08_3_xor_nn_wide_deep.ipynb"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Imports"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "import torch\r\n",
    "import torch.nn as nn\r\n",
    "import torch.nn.functional as F\r\n",
    "import torch.optim as optim"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\r\n",
    "\r\n",
    "torch.manual_seed(1)\r\n",
    "if device == 'cuda':\r\n",
    "    torch.cuda.manual_seed_all(1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Define XOR Problem"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "X = torch.FloatTensor([[0, 0], [0, 1], [1, 0], [1, 1]]).to(device)\r\n",
    "Y = torch.FloatTensor([[0], [1], [1], [0]]).to(device)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create the Wide and Deep Model"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "# nn layers\r\n",
    "linear1 = nn.Linear(2, 10)\r\n",
    "linear2 = nn.Linear(10, 10)\r\n",
    "linear3 = nn.Linear(10, 10)\r\n",
    "linear4 = nn.Linear(10, 1)\r\n",
    "sigmoid = nn.Sigmoid()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "model = nn.Sequential(linear1, sigmoid, linear2, sigmoid, linear3, sigmoid, linear4, sigmoid).to(device)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "criterion = nn.BCELoss().to(device)\r\n",
    "optimizer = optim.SGD(model.parameters(), lr =1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "for step in range(10001):\r\n",
    "\r\n",
    "    #prediction\r\n",
    "    pred = model(X)\r\n",
    "\r\n",
    "    #cost\r\n",
    "    cost = criterion(pred, Y)\r\n",
    "\r\n",
    "    #Reduce the cost\r\n",
    "    optimizer.zero_grad()\r\n",
    "    cost.backward()\r\n",
    "    optimizer.step()\r\n",
    "\r\n",
    "    if step % 1000 == 0:\r\n",
    "        result = sigmoid(pred).squeeze().detach().cpu().numpy()\r\n",
    "        print('Epoch: {:4d}/10000, Result: {}, Cost: {:.6f}'.format(step, result, cost.item()))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch:    0/10000, Result: [0.6539865 0.6540312 0.6539376 0.6539956], Cost: 0.731973\n",
      "Epoch: 1000/10000, Result: [0.62243706 0.62250036 0.6224063  0.62246627], Cost: 0.693140\n",
      "Epoch: 2000/10000, Result: [0.62244695 0.6225462  0.6224133  0.6224938 ], Cost: 0.693107\n",
      "Epoch: 3000/10000, Result: [0.6224198  0.6225931  0.62239367 0.62253016], Cost: 0.693069\n",
      "Epoch: 4000/10000, Result: [0.62223357 0.6227078  0.62229025 0.6226171 ], Cost: 0.692835\n",
      "Epoch: 5000/10000, Result: [0.58411795 0.6377132  0.62946147 0.6384127 ], Cost: 0.615173\n",
      "Epoch: 6000/10000, Result: [0.50020653 0.7308674  0.7308163  0.5003069 ], Cost: 0.001065\n",
      "Epoch: 7000/10000, Result: [0.5000864  0.7309806  0.73095864 0.5001244 ], Cost: 0.000437\n",
      "Epoch: 8000/10000, Result: [0.5000536  0.7310111  0.73099756 0.500076  ], Cost: 0.000268\n",
      "Epoch: 9000/10000, Result: [0.5000386  0.7310245  0.7310147  0.50005406], Cost: 0.000192\n",
      "Epoch: 10000/10000, Result: [0.50003004 0.73103225 0.73102444 0.5000416 ], Cost: 0.000149\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "source": [
    "# Accuracy computation\r\n",
    "# True if hypothesis > 0.5 else False\r\n",
    "with torch.no_grad():\r\n",
    "    pred = model(X)\r\n",
    "    predicted = (pred > 0.5).float()\r\n",
    "    accuracy = (predicted == Y).float().mean() *100\r\n",
    "    print('Prediction: {}, \\nPredicted: {}, \\nAccuracy: {}'.format(pred.detach().cpu().numpy(), predicted.detach().cpu().numpy(), accuracy))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Prediction: [[1.19966135e-04]\n",
      " [9.99865890e-01]\n",
      " [9.99826610e-01]\n",
      " [1.66115104e-04]], \n",
      "Predicted: [[0.]\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]], \n",
      "Accuracy: 100.0\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## High-Level Implementation with ```nn.Module```"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "source": [
    "class XOR_Wide_Deep(nn.Module):\r\n",
    "    def __init__(self):\r\n",
    "        super().__init__()\r\n",
    "        self.sq = nn.Sequential(\r\n",
    "            nn.Linear(2, 10),\r\n",
    "            nn.Sigmoid(),\r\n",
    "            nn.Linear(10, 10),\r\n",
    "            nn.Sigmoid(),\r\n",
    "            nn.Linear(10, 10),\r\n",
    "            nn.Sigmoid(),\r\n",
    "            nn.Linear(10, 1),\r\n",
    "            nn.Sigmoid()\r\n",
    "        )\r\n",
    "    \r\n",
    "    def forward(self, x):\r\n",
    "        return self.sq(x)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "source": [
    "model = XOR_Wide_Deep().to(device)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "source": [
    "nb_epochs = 10001\r\n",
    "\r\n",
    "for epoch in range(nb_epochs):\r\n",
    "\r\n",
    "    #prediction\r\n",
    "    pred = model(X)\r\n",
    "\r\n",
    "    #cost\r\n",
    "    cost = F.binary_cross_entropy(pred, Y).to(device)\r\n",
    "\r\n",
    "    #Reduce cost\r\n",
    "    optimizer.zero_grad()\r\n",
    "    cost.backward()\r\n",
    "    optimizer.step()\r\n",
    "\r\n",
    "    if epoch % 1000 == 0:\r\n",
    "        result = torch.sigmoid(pred).squeeze().detach().cpu().numpy()\r\n",
    "        print('Epoch: {:2d}/10000, result: {}, cost: {:.6f}'.format(epoch, result, cost.item()))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch:  0/10000, result: [0.60209876 0.60213494 0.6020612  0.60210985], cost: 0.708109\n",
      "Epoch: 1000/10000, result: [0.6224386  0.6224769  0.62242305 0.6224517 ], cost: 0.693127\n",
      "Epoch: 2000/10000, result: [0.6224384  0.62247753 0.62241894 0.62245524], cost: 0.693141\n",
      "Epoch: 3000/10000, result: [0.62243104 0.6224865  0.6224224  0.6224708 ], cost: 0.693133\n",
      "Epoch: 4000/10000, result: [0.6224201  0.62248063 0.62240714 0.622475  ], cost: 0.693163\n",
      "Epoch: 5000/10000, result: [0.6224164 0.6225073 0.6224153 0.6224915], cost: 0.693116\n",
      "Epoch: 6000/10000, result: [0.6223871 0.6225118 0.6223942 0.622507 ], cost: 0.693122\n",
      "Epoch: 7000/10000, result: [0.6223548  0.62254095 0.62237835 0.6225446 ], cost: 0.693105\n",
      "Epoch: 8000/10000, result: [0.62221515 0.62258583 0.6223145  0.6226085 ], cost: 0.692985\n",
      "Epoch: 9000/10000, result: [0.6205708  0.62311643 0.62223154 0.62352747], cost: 0.690540\n",
      "Epoch: 10000/10000, result: [0.5002793  0.7306931  0.73068523 0.5004934 ], cost: 0.001713\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "source": [
    "# Accuracy computation\r\n",
    "# True if hypothesis > 0.5 else False\r\n",
    "with torch.no_grad():\r\n",
    "    pred = model(X)\r\n",
    "    predicted = (pred > 0.5).float()\r\n",
    "    accuracy = (predicted == Y).float().mean() *100\r\n",
    "    print('Prediction: {}, \\nPredicted: {}, \\nAccuracy: {}'.format(pred.detach().cpu().numpy(), predicted.detach().cpu().numpy(), accuracy))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Prediction: [[0.00111703]\n",
      " [0.99814284]\n",
      " [0.9980995 ]\n",
      " [0.00196971]], \n",
      "Predicted: [[0.]\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]], \n",
      "Accuracy: 100.0\n"
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('buddhalight': conda)"
  },
  "interpreter": {
   "hash": "38ed4d61829b01de31b0fe0651719916120d9f7e023a62cbbfea93b7d24a50a0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}