CNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "BVwrbJVYUxP0",
        "colab_type": "code",
        "outputId": "daf88744-ec3b-4bd2-d273-fbfcaf542929",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "######################################\n",
        "############ 1. load data ############\n",
        "######################################\n",
        "\n",
        "# CIFAR 10 documentation: 50 000 training instances\n",
        "# 32 pixels x 32 pixels x 3 channels, 10 classes, 10 000 test instances\n",
        "\n",
        "# TUTORIAL: https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html?fbclid=IwAR3VHznvftP-KPTgY0Ffu6wNLD7Lx0iYVhNzJqlrt1Ef3frcZrDqc_moru4\n",
        "\n",
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import numpy as np\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import torch.optim as optim\n",
        "import pandas as pd\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import math\n",
        "\n",
        "### optional data augmentation ###\n",
        "augmented_transform = transforms.Compose(\n",
        "    [transforms.RandomChoice([transforms.RandomHorizontalFlip(p=0.5), transforms.RandomRotation(degrees=30), transforms.RandomVerticalFlip(p=0.5)]),\n",
        "     transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
        "\n",
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
        "\n",
        "# batch size: how many instances do you run through before moving in the direction of the gradient (minibatch-SGD)\n",
        "batch = 20\n",
        "\n",
        "# training data\n",
        "traindata = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n",
        "\n",
        "# optional training data augmentation\n",
        "#traindata = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=augmented_transform)\n",
        "\n",
        "# use 20% of training data as a validation set\n",
        "trainset, valset = torch.utils.data.random_split(traindata, [40000, 10000])\n",
        "# load validation set\n",
        "valloader = torch.utils.data.DataLoader(valset, batch_size=batch, shuffle=True, num_workers=2)\n",
        "\n",
        "# load test set\n",
        "testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=batch, shuffle=False, num_workers=2)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-l6Ym6aGafqW",
        "colab_type": "text"
      },
      "source": [
        "possible hyper-parameters:\n",
        "\n",
        "\n",
        "*   **f**: filter size (3, 4 or 5)\n",
        "*   **max_channels**: number of channels output by last convolution (16 or 18)\n",
        "*   **convs**: number of convolutional layers (2, 3 or 4)\n",
        "*   **fcs**: number of fully connected layers (3, 4 or 5)\n",
        "*   **pool**: different pooling options\n",
        "        1: max pool LAST layer only\n",
        "        2: avg pool LAST layer only\n",
        "        3: max pool LAST two layers\n",
        "        4: avg pool LAST two layers\n",
        "        5: max pool before-last layer, avg pool last layer\n",
        "        6: avg pool before-last layer, max pool last layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sn2Da6Y3U7Q-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##################################################\n",
        "### 2. functions to generate customizable CNNs ###\n",
        "##################################################\n",
        "\n",
        "def define_stuff(f,max_channels,convs,fcs,pool):\n",
        "    def pool_dim(s):\n",
        "        s = math.floor(((s-2)/2)+1)\n",
        "        return s\n",
        "    def conv_dim(s, f):\n",
        "        s = s - f + 1\n",
        "        return s\n",
        "\n",
        "    # array with numbers of channels output by each convolution\n",
        "    channel_options = [[6,16],[6,18],[6,10,16],[6,12,18],[5,10,15,18]]\n",
        "\n",
        "    # 2 conv layers:\n",
        "    if (max_channels*convs == 32 or max_channels*convs == 36):\n",
        "        if (max_channels*convs == 32): channels=channel_options[0]\n",
        "        else: channels=channel_options[1]\n",
        "        last_channel = channels[1]\n",
        "        if (pool == 1 or pool == 2): s = pool_dim(conv_dim(conv_dim(32, f),f))\n",
        "        if (pool != 1 and pool != 2): s = pool_dim(conv_dim(pool_dim(conv_dim(32, f)),f))\n",
        "    # 3 conv layers:\n",
        "    if (max_channels*convs == 48 or max_channels*convs == 54):\n",
        "        if (max_channels*convs == 48): channels=channel_options[2]\n",
        "        else: channels=channel_options[3]\n",
        "        last_channel = channels[2]\n",
        "        if (pool == 1 or pool == 2): s = pool_dim(conv_dim(conv_dim(conv_dim(32, f),f),f))\n",
        "        if (pool != 1 and pool != 2):s = pool_dim(conv_dim(pool_dim(conv_dim(conv_dim(32, f),f)),f))\n",
        "    # 4 conv layers:\n",
        "    if (max_channels*convs == 72): \n",
        "        channels=channel_options[4]\n",
        "        last_channel = channels[3]\n",
        "        if (pool == 1 or pool == 2): s = pool_dim(conv_dim(conv_dim(conv_dim(conv_dim(32, f),f),f),f))\n",
        "        if (pool != 1 and pool != 2): s = pool_dim(conv_dim(pool_dim(conv_dim(conv_dim(conv_dim(32, f),f),f)),f))\n",
        "    return channels, last_channel, s\n",
        "class Net(nn.Module):\n",
        "    def __init__(self,f,max_channels,convs,fcs,pool):\n",
        "        super(Net, self).__init__()\n",
        "        channels, last_channel, s = define_stuff(f,max_channels,convs,fcs,pool)\n",
        "        self.conv1 = nn.Conv2d(3, channels[0],f)\n",
        "        self.conv2 = nn.Conv2d(channels[0], channels[1], f)\n",
        "        if len(channels) >= 3:\n",
        "            self.conv3 = nn.Conv2d(channels[1], channels[2], f)\n",
        "            if len(channels) >= 4:\n",
        "                self.conv4 = nn.Conv2d(channels[2], channels[3], f)\n",
        "        self.max_pool = nn.MaxPool2d(2, 2)\n",
        "        self.avg_pool = nn.AvgPool2d(2, 2)\n",
        "        if (fcs == 3):\n",
        "            self.fc1 = nn.Linear(last_channel * s * s, 120)\n",
        "            self.fc2 = nn.Linear(120, 84)\n",
        "            self.fc3 = nn.Linear(84, 10)\n",
        "        if (fcs == 4):   \n",
        "            self.fc1 = nn.Linear(last_channel * s * s, 120)\n",
        "            self.fc2 = nn.Linear(120, 84)\n",
        "            self.fc3 = nn.Linear(84, 54)\n",
        "            self.fc4 = nn.Linear(54, 10)\n",
        "        if (fcs == 5):   \n",
        "            self.fc1 = nn.Linear(last_channel * s * s, 120)\n",
        "            self.fc2 = nn.Linear(120, 90)\n",
        "            self.fc3 = nn.Linear(90, 80)       \n",
        "            self.fc4 = nn.Linear(80, 64)\n",
        "            self.fc5 = nn.Linear(64, 10)      \n",
        "    def forward(self,x,f,max_channels,convs,fcs,pool):\n",
        "        channels, last_channel, s = define_stuff(f,max_channels,convs,fcs,pool)\n",
        "        if len(channels) == 2:\n",
        "            if (pool == 1):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = self.max_pool(F.relu(self.conv2(x)))\n",
        "            if (pool == 2):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = self.avg_pool(F.relu(self.conv2(x)))\n",
        "            if (pool == 3):\n",
        "                x = self.max_pool(F.relu(self.conv1(x)))\n",
        "                x = self.max_pool(F.relu(self.conv2(x)))\n",
        "            if (pool == 4):\n",
        "                x = self.avg_pool(F.relu(self.conv1(x)))\n",
        "                x = self.avg_pool(F.relu(self.conv2(x)))\n",
        "            if (pool == 5):\n",
        "                x = self.max_pool(F.relu(self.conv1(x)))\n",
        "                x = self.avg_pool(F.relu(self.conv2(x)))\n",
        "            if (pool == 6):\n",
        "                x = self.avg_pool(F.relu(self.conv1(x)))\n",
        "                x = self.max_pool(F.relu(self.conv2(x)))\n",
        "        if len(channels) == 3:\n",
        "            if (pool == 1):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = self.max_pool(F.relu(self.conv3(x)))\n",
        "            if (pool == 2):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = self.avg_pool(F.relu(self.conv3(x)))\n",
        "            if (pool == 3):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = self.max_pool(F.relu(self.conv2(x)))\n",
        "                x = self.max_pool(F.relu(self.conv3(x)))\n",
        "            if (pool == 4):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = self.avg_pool(F.relu(self.conv2(x)))\n",
        "                x = self.avg_pool(F.relu(self.conv3(x)))\n",
        "            if (pool == 5):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = self.max_pool(F.relu(self.conv2(x)))\n",
        "                x = self.avg_pool(F.relu(self.conv3(x)))\n",
        "            if (pool == 6):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = self.avg_pool(F.relu(self.conv2(x)))\n",
        "                x = self.max_pool(F.relu(self.conv3(x)))\n",
        "        if len(channels) == 4:\n",
        "            if (pool == 1):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = F.relu(self.conv3(x))\n",
        "                x = self.max_pool(F.relu(self.conv4(x)))\n",
        "            if (pool == 2):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = F.relu(self.conv3(x))\n",
        "                x = self.avg_pool(F.relu(self.conv4(x)))\n",
        "            if (pool == 3):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = self.max_pool(F.relu(self.conv3(x)))\n",
        "                x = self.max_pool(F.relu(self.conv4(x)))\n",
        "            if (pool == 4):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = self.avg_pool(F.relu(self.conv3(x)))\n",
        "                x = self.avg_pool(F.relu(self.conv4(x)))\n",
        "            if (pool == 5):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = self.max_pool(F.relu(self.conv3(x)))\n",
        "                x = self.avg_pool(F.relu(self.conv4(x)))\n",
        "            if (pool == 6):\n",
        "                x = F.relu(self.conv1(x))\n",
        "                x = F.relu(self.conv2(x))\n",
        "                x = self.avg_pool(F.relu(self.conv3(x)))\n",
        "                x = self.max_pool(F.relu(self.conv4(x)))\n",
        "        x = x.view(-1, last_channel * s * s) \n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        if (fcs == 3):\n",
        "            x = self.fc3(x)\n",
        "            return x\n",
        "        else:\n",
        "            x = F.relu(self.fc3(x))\n",
        "            if (fcs == 4):\n",
        "                x = self.fc4(x)\n",
        "                return x\n",
        "            else:\n",
        "                x = F.relu(self.fc4(x))\n",
        "                x = self.fc5(x)\n",
        "                return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PO54hcMbpGKQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "############################################\n",
        "### 3. function to train cutomizable CNN ###\n",
        "############################################\n",
        "\n",
        "def train_model(num_epochs,f,max_channels,convs,fcs,pool, opt,batch):\n",
        "    stop = 0 # variable that will be used for early stopping\n",
        "\n",
        "    epoch_accuracy = [] # stores validation accuracy per epoch\n",
        "    train_accuracy = [] # stores training accuracy per epoch\n",
        "    test_accuracy = [] # stores test accuracy per epoch\n",
        "\n",
        "    losses_tuples = [] # stores average loss per batch of 2000 instances\n",
        "    net = Net(f,max_channels,convs,fcs,pool) # generate net\n",
        "    criterion = nn.CrossEntropyLoss() # loss function: cross-entropy loss\n",
        "    \n",
        "    # define optimier:\n",
        "    if opt == 1: optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
        "    if opt == 2: optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.5)\n",
        "    if opt == 3: optimizer = optim.Adam(net.parameters(),lr=0.001)\n",
        "    if opt == 4: optimizer = optim.Adagrad(net.parameters(), lr=0.01, lr_decay=0, weight_decay=0, initial_accumulator_value=0, eps=1e-10)\n",
        "    if opt == 5: optimizer = optim.Adadelta(net.parameters(), lr=1.0, rho=0.9, eps=1e-06, weight_decay=0)\n",
        "    if opt == 6: optimizer = optim.AdamW(net.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.01, amsgrad=False)\n",
        "    if opt == 8: optimizer = optim.Adamax(net.parameters(), lr=0.002, betas=(0.9, 0.999), eps=1e-08, weight_decay=0)\n",
        "    if opt == 9: optimizer = optim.ASGD(net.parameters(), lr=0.01, lambd=0.0001, alpha=0.75, t0=1000000.0, weight_decay=0)\n",
        "    if opt == 10: optimizer = optim.RMSprop(net.parameters(), lr=0.01, alpha=0.99, eps=1e-08, weight_decay=0, momentum=0, centered=False)\n",
        "    if opt == 11: optimizer = optim.Rprop(net.parameters(), lr=0.01, etas=(0.5, 1.2), step_sizes=(1e-06, 50))\n",
        "    \n",
        "    # variables to keep track of for early stopping\n",
        "    curr = 0\n",
        "    prev = 0\n",
        "    prev_prev = 0\n",
        "\n",
        "    # loop over the dataset multiple times\n",
        "    for epoch in range(num_epochs): \n",
        "        # shuffle training data with each iteration\n",
        "        trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch, shuffle=True, num_workers=2)\n",
        "        running_loss = 0.0\n",
        "        losses = []\n",
        "        for i, data in enumerate(trainloader, 0):\n",
        "            # get the inputs; data is a list of [inputs, labels]\n",
        "            inputs, labels = data\n",
        "\n",
        "            # zero the parameter gradients\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # forward + backward + optimize\n",
        "            outputs = net(inputs,f,max_channels,convs,fcs,pool)\n",
        "            loss = criterion(outputs, labels)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # print statistics\n",
        "            running_loss += loss.item()\n",
        "            if i % 2000 == 1999:    # print every 2000 mini-batches\n",
        "                print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000))\n",
        "                losses_tuples.append((epoch + 1, i + 1, running_loss/ 2000))\n",
        "                running_loss = 0.0\n",
        "\n",
        "        # training data accuracy\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        with torch.no_grad():\n",
        "            for j, data in enumerate(trainloader, 0): \n",
        "                images, labels = data\n",
        "                outputs = net(images,f,max_channels,convs,fcs,pool)\n",
        "                _, predicted = torch.max(outputs.data, 1)\n",
        "                total += labels.size(0)\n",
        "                correct += (predicted == labels).sum().item()\n",
        "            train_accuracy.append((epoch + 1, 100 * correct / total))\n",
        "\n",
        "        # test data accuracy\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        with torch.no_grad():\n",
        "            for j, data in enumerate(testloader, 0): \n",
        "                images, labels = data\n",
        "                outputs = net(images,f,max_channels,convs,fcs,pool)\n",
        "                _, predicted = torch.max(outputs.data, 1)\n",
        "                total += labels.size(0)\n",
        "                correct += (predicted == labels).sum().item()\n",
        "            test_accuracy.append((epoch + 1, 100 * correct / total))\n",
        "\n",
        "        # validation data accuracy (and implementation of early stopping!)\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        with torch.no_grad():\n",
        "            for i, data in enumerate(valloader, 0): \n",
        "                images, labels = data\n",
        "                outputs = net(images,f,max_channels,convs,fcs,pool)\n",
        "                _, predicted = torch.max(outputs.data, 1)\n",
        "                total += labels.size(0)\n",
        "                correct += (predicted == labels).sum().item()\n",
        "            epoch_accuracy.append((epoch + 1, 100 * correct / total))\n",
        "            print('Accuracy of the network on validation set: %d %%' % (100 * correct / total))\n",
        "\n",
        "        # implementation of early stopping:\n",
        "            curr = 100 * correct / total\n",
        "            if (epoch == 0): prev_prev = curr\n",
        "            if (epoch == 1): prev = curr\n",
        "            if (epoch >= 2):\n",
        "                if (prev_prev == prev == curr):\n",
        "                    stop += 1\n",
        "                if (prev_prev >= curr):\n",
        "                    stop += 1\n",
        "                prev_prev = prev\n",
        "                prev = curr\n",
        "        if stop == 1: break\n",
        "    \n",
        "    # save outputs:\n",
        "    pd.DataFrame(losses_tuples, columns = ['epoch','iteration','loss']).to_csv(f'{num_epochs}epochs_{batch}batch_{f}filters_{max_channels}channels_{convs}convs_{fcs}fcs_{pool}pool_{opt}optim_loss_aug.csv')\n",
        "    pd.DataFrame(epoch_accuracy, columns = ['epoch','iteration']).to_csv(f'{num_epochs}epochs_{batch}batch_{f}filters_{max_channels}channels_{convs}convs_{fcs}fcs_{pool}pool__{opt}optim_validation_accuracy_aug.csv')\n",
        "    pd.DataFrame(train_accuracy, columns = ['epoch','iteration']).to_csv(f'{num_epochs}epochs_{batch}batch_{f}filters_{max_channels}channels_{convs}convs_{fcs}fcs_{pool}pool__{opt}optim_train_accuracy_aug.csv')\n",
        "    pd.DataFrame(test_accuracy, columns = ['epoch','iteration']).to_csv(f'{num_epochs}epochs_{batch}batch_{f}filters_{max_channels}channels_{convs}convs_{fcs}fcs_{pool}pool__{opt}optim_test_accuracy_aug.csv')\n",
        "    print('Finished Training')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-srv_nmexIUS",
        "colab_type": "code",
        "outputId": "178ca9cf-0810-412d-c936-a7ad3cac3f85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "#############################\n",
        "### 4. Running the models ###\n",
        "#############################\n",
        "\n",
        "# Final model parameters:\n",
        "num_epochs=40\n",
        "f=3\n",
        "max_channels=18\n",
        "convs=3\n",
        "fcs=3\n",
        "pool=3\n",
        "opt=8\n",
        "batch=20 \n",
        "\n",
        "train_model(num_epochs,f,max_channels,convs,fcs,pool,opt,batch)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1,  2000] loss: 1.783\n",
            "Accuracy of the network on validation set: 41 %\n",
            "[2,  2000] loss: 1.512\n",
            "Accuracy of the network on validation set: 45 %\n",
            "[3,  2000] loss: 1.401\n",
            "Accuracy of the network on validation set: 51 %\n",
            "[4,  2000] loss: 1.311\n",
            "Accuracy of the network on validation set: 52 %\n",
            "[5,  2000] loss: 1.248\n",
            "Accuracy of the network on validation set: 55 %\n",
            "[6,  2000] loss: 1.196\n",
            "Accuracy of the network on validation set: 57 %\n",
            "[7,  2000] loss: 1.154\n",
            "Accuracy of the network on validation set: 58 %\n",
            "[8,  2000] loss: 1.121\n",
            "Accuracy of the network on validation set: 58 %\n",
            "[9,  2000] loss: 1.091\n",
            "Accuracy of the network on validation set: 60 %\n",
            "[10,  2000] loss: 1.061\n",
            "Accuracy of the network on validation set: 61 %\n",
            "[11,  2000] loss: 1.036\n",
            "Accuracy of the network on validation set: 62 %\n",
            "[12,  2000] loss: 1.013\n",
            "Accuracy of the network on validation set: 62 %\n",
            "[13,  2000] loss: 0.999\n",
            "Accuracy of the network on validation set: 63 %\n",
            "[14,  2000] loss: 0.978\n",
            "Accuracy of the network on validation set: 63 %\n",
            "[15,  2000] loss: 0.967\n",
            "Accuracy of the network on validation set: 62 %\n",
            "Finished Training\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RxdGgqQIJDLo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##################################################\n",
        "### 5. download each .csv file to save outputs ###\n",
        "##################################################\n",
        "\n",
        "import os \n",
        "from google.colab import files\n",
        "import re\n",
        "\n",
        "for item in os.listdir():\n",
        "    if (re.search(\".csv\", item)):\n",
        "        files.download(item)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAW5pw1-uMeM",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SRY8Dh79wWij",
        "colab_type": "text"
      },
      "source": [
        "# New Section"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oc-0JOwCJL2y",
        "colab_type": "text"
      },
      "source": [
        "# New Section"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AUOucMnJvRIF",
        "colab_type": "text"
      },
      "source": [
        "# New Section"
      ]
    }
  ]
}