1_Intro_Pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a399f488-91a8-4484-9849-bed543db6a67",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# The jupyter notebook is launched from your $HOME directory.\n",
    "# Change the working directory to the C-Debugging directory\n",
    "# which was created in your username directory under /scratch/vp91\n",
    "os.chdir(os.path.expandvars(\"/scratch/vp91/$USER/AAPP-Pytorch\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8812b75b-5639-4201-abf2-a5de9b9d6c0d",
   "metadata": {},
   "source": [
    "# Intro to AI in Pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9d836f5-fc47-4d7b-b8ef-add093c48153",
   "metadata": {},
   "source": [
    "In this notebook we will explore the basics of doing classification with PyTorch."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3a66306-1b78-4967-97a5-4e224710d91c",
   "metadata": {},
   "source": [
    "First thing we want to demonstrate is that PyTorch can make performing GPU computations very easy. If the GPU is available, using the GPU for matrix multiplication is very easy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d63b36e8-03b7-429c-9e04-6101eb1036bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "N = int(1e4)\n",
    "A_cpu = torch.rand(N, N)\n",
    "\n",
    "# start_time = datetime.now()\n",
    "# A_cpu @ A_cpu\n",
    "# print(f\"Total time: {datetime.now() - start_time}\")\n",
    "\n",
    "# A_gpu = A_cpu.to(device)\n",
    "# start_time = datetime.now()\n",
    "# A_gpu @ A_gpu\n",
    "# print(f\"Total time: {datetime.now() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1d48c13-2371-43d4-8c7e-9126550457d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf877252-d887-4ea1-9d02-157450fab988",
   "metadata": {},
   "source": [
    "We now turn to deep learning. We first create a classification problem. In this dataset, each spiral arm is of a different class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c31cbe1a-b4b8-4983-bdaa-1b054704ecce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import math\n",
    "from IPython import display\n",
    "from matplotlib import pyplot as plt\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader, TensorDataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8079e74c-79e6-4fd0-91dd-fd942e839b40",
   "metadata": {},
   "source": [
    "## Linear Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "923c91a6-cab7-426e-847b-6949f2f3512a",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1000  # num_samples_per_class\n",
    "D = 1  # dimensions\n",
    " \n",
    "tensor = torch.ones((N,D), dtype=torch.float32)\n",
    "k = tensor.new_full((N,D),random.random())\n",
    "b = 0.4 * torch.randn((N,D),dtype=torch.float32)\n",
    "x = torch.randn((N,D),dtype=torch.float32)\n",
    "\n",
    "y = k*x+b\n",
    "\n",
    "\n",
    "print(tuple(y.size()), tuple(b.size()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06a190db-3f25-4ba0-b510-18a4183ef9bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "548d77d8-de11-4670-a8d5-71c0b5e19a95",
   "metadata": {},
   "outputs": [],
   "source": [
    "batchsize = 100\n",
    "dataset = TensorDataset(x, y)\n",
    "train_loader = DataLoader(dataset,batch_size=batchsize)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3041948-acba-4508-9621-2b466869d08d",
   "metadata": {},
   "source": [
    "## Linear Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9674b6b6-ed5d-43ce-b3c9-8c986f49f0d2",
   "metadata": {},
   "source": [
    "The below function encapsulates the training procedure for an epoch (single run over the entire training dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10435a81-12db-4e32-a2d4-587a21b86417",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearRegression(torch.nn.Module):\n",
    "\n",
    "    def __init__(self): \n",
    "        super(LinearRegression, self).__init__() \n",
    "        self.linear = torch.nn.Linear(D,D)\n",
    "      \n",
    "    def forward(self, x): \n",
    "        predict_y = self.linear(x) \n",
    "        return predict_y "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f61d7d0e-c3c5-47b9-a734-5fc3e67f949c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, model, criteria, train_loader, optimizer, device):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # send to device\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "\n",
    "        # Optimize\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criteria(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    if epoch % 10 == 0:\n",
    "        loss_l.append(loss.item())\n",
    "        print(f'Train Epoch: {epoch} \\tLoss: {loss.item():.6f}')\n",
    "    return loss_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fc29890-fdc2-46d4-86a8-b6c38cdc0df3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "linear_model = LinearRegression().to(device)\n",
    "define_criterion = torch.nn.MSELoss()\n",
    "SGD_optimizer = torch.optim.SGD(linear_model.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c99b2377-f1ac-4ad2-9b00-035201a78c47",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_l = []\n",
    "for epoch in range(200):\n",
    "    loss_l = train(epoch, linear_model, define_criterion, train_loader, SGD_optimizer, device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18fb9c8e-9069-4ff7-91e6-f1893a7d21ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(range(20),loss_l)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e63a6053-7d26-4695-a7bf-f20113aa6e49",
   "metadata": {},
   "source": [
    "## Use the Bottle Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1272cf03-cf5c-4718-8eb6-3e0d52c8c23e",
   "metadata": {},
   "source": [
    "### Reading Data  \n",
    "Read and explore the data as much as you want. Here we want to select suitable input for linear regression model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b965ca9b-e5be-45af-9dd3-6ad770bef65f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe_raw = pd.read_csv('/scratch/vp91/AAPP2023/Data/CalCOFI_bottle.csv')\n",
    "dataframe_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6728472-3a95-4b0c-a729-be3992fdfcd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe_raw.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89b9cd91-5550-4263-8cb8-343108bd949d",
   "metadata": {},
   "source": [
    "**TODO**   \n",
    "Change the columns below to select suitable columns for Simple Linear Regression or Multiple Linear Regression. Remember the input variables need to be independent to each other for linear regression to work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8491e142-ed55-4de2-a9e4-9366f5020ee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = dataframe_raw.filter(items=['T_degC', 'Depthm', 'Salnty', 'O2ml_L', 'STheta', 'O2Sat', 'Oxy_µmol/Kg'])\n",
    "dataframe.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60666077-3375-4c3c-bc9c-6ecc26e8a7c2",
   "metadata": {},
   "source": [
    "**TODO**  \n",
    "How do you want to handle the Null values here? Modify the function to your needs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83c07022-78e4-473d-9fd9-545163069962",
   "metadata": {},
   "outputs": [],
   "source": [
    "def customize_dataset(data):\n",
    "    dataframe = data.copy(deep=True)\n",
    "    #fill na values\n",
    "    dataframe = dataframe.groupby(dataframe.columns, axis = 1).transform(lambda x: x.fillna(x.mean()))\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c131f81-04c9-4048-bfeb-cab9a4762a01",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = customize_dataset(dataframe)\n",
    "dataframe.isnull().sum() "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62be6e43-b63f-42e6-9492-b453874fb63f",
   "metadata": {},
   "source": [
    "**TODO**  \n",
    "Change the dataframe selected to pytorch tensor. Hint: pandas has a function DataFrame.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d103599a-47b3-4637-94e4-ef1fe1d1eaac",
   "metadata": {},
   "outputs": [],
   "source": [
    "input = dataframe[[\"T_degC\",\"Depthm\"]].to_numpy()\n",
    "x = torch.from_numpy(input)\n",
    "input.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2a46046-d93f-4912-8ffb-d9f29f623921",
   "metadata": {},
   "source": [
    "**TODO**  \n",
    "Modify below cells from above example to build and train your own model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba93ed89-192b-421c-8722-37f5daa9e582",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearRegression(torch.nn.Module):\n",
    "\n",
    "    def __init__(self): \n",
    "        super(LinearRegression, self).__init__() \n",
    "        self.linear = torch.nn.Linear(D,D)\n",
    "      \n",
    "    def forward(self, x): \n",
    "        predict_y = self.linear(x) \n",
    "        return predict_y "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3adc67ae-a05b-4d6a-a18d-cc6de7cc7381",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, model, criteria, train_loader, optimizer, device):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # send to device\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "\n",
    "        # Optimize\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criteria(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    if epoch % 10 == 0:\n",
    "        loss_l.append(loss.item())\n",
    "        print(f'Train Epoch: {epoch} \\tLoss: {loss.item():.6f}')\n",
    "    return loss_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e162144b-95e7-44b0-ae2d-10e9d6fb21f8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "linear_model = LinearRegression().to(device)\n",
    "define_criterion = torch.nn.MSELoss()\n",
    "SGD_optimizer = torch.optim.SGD(linear_model.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb9c8d34-b427-46fb-abbf-be5b10c73ea6",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_l = []\n",
    "for epoch in range(200):\n",
    "    loss_l = train(epoch, linear_model, define_criterion, train_loader, SGD_optimizer, device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5c4d75f-60b6-4f98-aaad-61a16110e4e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(range(20),loss_l)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b95ab9cc-8ad2-43a4-ae68-43e7ed9bdadc",
   "metadata": {},
   "source": [
    "- How is your model performance?  \n",
    "- What could be improved?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "118ef8b8-621b-4e67-9da5-a7fa6dea2dc2",
   "metadata": {},
   "source": [
    "# Unsupervised Learning - Clustering  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d644ed32-c33c-4d24-aeee-6c34add51ffd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from time import time\n",
    "from kmeans_pytorch import kmeans, kmeans_predict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73ce9bd3-6a3e-4e4b-a276-0cb4a5ea03c5",
   "metadata": {},
   "source": [
    "## K-means Clustering  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f5be94c-876d-4baa-9e7b-db47f051f3fc",
   "metadata": {},
   "source": [
    "1. Set K – number of clusters\n",
    "2. Randomly assign k points as the centroid of the clusers\n",
    "3. Measure distance between point a and the k clusters\n",
    "4. Assign point a to the cluster with the minimum distance\n",
    "5. Repeat 3-4 for all data points\n",
    "6. Recalculate the cluster centroid\n",
    "7. Repeat 5-6 until the clusters don’t change anymore\n",
    "8. Calculate total clusters variance\n",
    "9. Repeat 2-8 N times, result is the clustering with the minimum variance\n",
    "ariance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4ee0323-dd53-4245-8924-9f40f7198250",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set random seed\n",
    "np.random.seed(123)\n",
    "# data\n",
    "data_size, dims, num_clusters = 1000, 2, 3\n",
    "x = np.random.randn(data_size, dims) /6\n",
    "x = torch.from_numpy(x)\n",
    "# more data\n",
    "y = np.random.randn(5, dims) /6\n",
    "y = torch.from_numpy(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95ad2dd2-09a4-4ff9-94ef-b19e54743109",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set device\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda:0')\n",
    "else:\n",
    "    device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5330d79a-047c-4e86-b6df-df416aeb595c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# k-means\n",
    "cluster_ids_x, cluster_centers = kmeans(\n",
    "    X=x, num_clusters=num_clusters, distance='euclidean', device=device\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "759824a0-31b5-409b-b130-d547dfd8238e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict cluster ids for y\n",
    "cluster_ids_y = kmeans_predict(\n",
    "    y, cluster_centers, 'euclidean', device=device\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93f1ad7b-a219-4b64-8274-42f227d3c16d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "plt.figure(figsize=(4, 3), dpi=160)\n",
    "plt.scatter(x[:, 0], x[:, 1], c=cluster_ids_x, cmap='cool')\n",
    "plt.scatter(y[:, 0], y[:, 1], c=cluster_ids_y, cmap='cool', marker='X')\n",
    "plt.scatter(\n",
    "    cluster_centers[:, 0], cluster_centers[:, 1],\n",
    "    c='white',\n",
    "    alpha=0.6,\n",
    "    edgecolors='black',\n",
    "    linewidths=2\n",
    ")\n",
    "plt.axis([-1, 1, -1, 1])\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c565412-e043-459e-983e-1f3ba755abdd",
   "metadata": {},
   "source": [
    "## K-means CPU vs GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c41457be-91a2-40cc-b0bc-3860be7bcc9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dimensions, num clusters\n",
    "dims, num_clusters = 2, 3\n",
    "\n",
    "# data sizes\n",
    "data_sizes = [100000, 1000000, 5000000, 10000000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0a4e059-9a59-4f0f-907b-5492917cca44",
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_times = []\n",
    "cpu_times = []\n",
    "\n",
    "for data_size in data_sizes:\n",
    "    print(f'\\ndata size: {data_size}')\n",
    "\n",
    "    # data\n",
    "    x = np.random.randn(data_size, dims) / 6\n",
    "    x = torch.from_numpy(x)\n",
    "\n",
    "    # gpu\n",
    "    start_gpu = time()\n",
    "    kmeans_gpu = kmeans(X=x, num_clusters=num_clusters, device=torch.device('cuda:0'))\n",
    "    gpu_time = time() - start_gpu\n",
    "    gpu_times.append(gpu_time)\n",
    "    print(f'gpu time: {gpu_time}')\n",
    "    \n",
    "    # cpu\n",
    "    start_cpu = time()\n",
    "    kmeans_cpu = kmeans(X=x, num_clusters=num_clusters, device=torch.device('cpu'))\n",
    "    cpu_time = time() - start_cpu\n",
    "    cpu_times.append(cpu_time)\n",
    "    print(f'cpu time: {cpu_time}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd69a0ec-8756-4119-8b83-2474106f6d51",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "plt.figure(figsize=(6, 3), dpi=160)\n",
    "plt.plot(data_sizes, gpu_times, marker='o', label='gpu', color='xkcd:vermillion')\n",
    "plt.plot(data_sizes, cpu_times, marker='o', label='cpu', color='xkcd:neon blue')\n",
    "plt.xticks(data_sizes)\n",
    "plt.legend(fontsize=12)\n",
    "plt.grid(alpha=0.2)\n",
    "plt.xlabel('data size', fontsize=14)\n",
    "plt.ylabel('time (s)', fontsize=14)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}