pipeline_on_board_4layer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8fd0b6e4-8b53-49c9-b03a-cc71a053b442",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "\n",
    "import pprint\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import torch.backends.cudnn as cudnn\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import matplotlib.pyplot as plt\n",
    "from torchvision import datasets, models, transforms\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "cudnn.benchmark = True\n",
    "plt.ion() \n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "32aca3eb-3a9f-4b08-9e55-355b4673b078",
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate_fn(batch):\n",
    "    return [\n",
    "      torch.stack([x[0] for x in batch]),\n",
    "      torch.tensor(np.array([x[1] for x in batch]))\n",
    "      #torch.stack([x[audio] for x in batch]),\n",
    "      #torch.tensor([x[labels] for x in batch])\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3f2f2ea1-f25a-4377-ab78-047b4c7ed05f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/mnt/g/onlinelessons/deep learning/dataset/train_full_malware/ByteToImage/on_board/\"\n",
    "\n",
    "class image_custom_dataset(Dataset):\n",
    "    \"\"\"Face Landmarks dataset.\"\"\"\n",
    "\n",
    "    def __init__(self, csv_file, transform=None):\n",
    "        \"\"\"\n",
    "        Arguments:\n",
    "            csv_file (string): Path to the csv file with labels.\n",
    "            root_dir (string): Directory with all the images.\n",
    "            transform (callable, optional): Optional transform to be applied\n",
    "                on a sample.\n",
    "        \"\"\"\n",
    "        \n",
    "        self.image_csv = pd.read_csv(csv_file)\n",
    "        self.transform = transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.ToTensor(),\n",
    "    ])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.image_csv)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "\n",
    "        image_name = self.image_csv.iloc[idx, 1]      \n",
    "        image = Image.open(image_name)\n",
    "\n",
    "        \n",
    "        label = self.image_csv.iloc[idx, 3:12]\n",
    "        label = np.array(label, dtype=np.float16)\n",
    "        \n",
    "        if self.transform:\n",
    "            image = self.transform(image)\n",
    "            \n",
    "        if image.shape[0] == 1:\n",
    "            image = torch.cat((image, image, image), dim=0)\n",
    "            \n",
    "        image = transforms.Compose([transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])(image)\n",
    "                \n",
    "        # Add a batch dimension to the tensor\n",
    "        # image = image.unsqueeze(0)\n",
    "        \n",
    "        return_tuple = tuple([image , label])\n",
    "        \n",
    "\n",
    "        return return_tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "18bdfe4a-837a-4e7d-b9db-ad2069f75a8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "test_dir = \"/mnt/g/onlinelessons/deep learning/dataset/train_full_malware/ByteToImage/on_board/test/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "86395261-79e9-4826-8dba-8010889767ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_datasets = {x.split(\"_\")[0]: image_custom_dataset(os.path.join(data_dir, x))\n",
    "                      for x in [\"test_data_encoded.csv\"]}\n",
    "batch_size = 4\n",
    "\n",
    "dataloaders = {\"test\": torch.utils.data.DataLoader(image_datasets[\"test\"], batch_size=batch_size,\n",
    "                                             shuffle=True, num_workers=8, collate_fn=collate_fn)}\n",
    "\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in [\"test\"]}\n",
    "#class_names = audio_datasets['train'].classes\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "print(dataset_sizes[\"test\"])                  \n",
    "len(dataloaders[\"test\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "433bce63-0a49-4571-9d5f-13fac97e926b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "50704931-4752-4f33-8fd3-4718131a972e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pipeline_module(model_list, confidence_factor=0.8):\n",
    "    print(f\"Confidence factor: {confidence_factor}\")\n",
    "    \n",
    "    was_training =[]\n",
    "    \n",
    "    for models in model_list:\n",
    "        was_training.append(models.training)\n",
    "        models.eval()\n",
    "    \n",
    "    model = model_list[0]\n",
    "    \n",
    "    pred_in_each_step = [0, 0, 0, 0]\n",
    "\n",
    "    total = 0\n",
    "    \n",
    "    model = model.to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        since = time.time()\n",
    "        for i, (inputs, _) in enumerate(dataloaders['test']):\n",
    "            inputs = inputs.to(device)\n",
    "\n",
    "            outputs = model(inputs)\n",
    "            preds = outputs\n",
    "            # print('\\033[30m' + f'label:     {chosen_label}' + '\\033[0m')       if you uncomment this line you can see original labels of each epoch\n",
    "            \n",
    "            _, predicted_labels = torch.max(preds, dim=1)\n",
    "            predicted_labels = predicted_labels.clone().detach()\n",
    "            # print(predicted_labels)\n",
    "            # print(chosen_label)\n",
    "            # print(inputs.shape)\n",
    "            for j , batch in enumerate(preds):\n",
    "                # print(predicted_labels[1][j])\n",
    "                if (float(batch[int(predicted_labels[j])]) < confidence_factor):\n",
    "                    stronger_model = model_list[1]\n",
    "                    stronger_model = stronger_model.to(device)\n",
    "                    strong_inputs = inputs[j]\n",
    "                    strong_inputs = strong_inputs.unsqueeze(0)\n",
    "                    # print(strong_inputs.shape)\n",
    "                    strong_outputs = stronger_model(strong_inputs)\n",
    "                    strong_preds = strong_outputs\n",
    "                    _, strong_labels = torch.tensor(torch.max(strong_preds, dim=1))\n",
    "                    # predicted_labels[j] = strong_labels\n",
    "                    # pred_in_each_step[1] += 1\n",
    "                    \n",
    "                    # print(strong_labels)\n",
    "        \n",
    "                    if (float(strong_preds[0][int(strong_labels)])< confidence_factor):\n",
    "                        stronger_model = model_list[2]\n",
    "                        stronger_model = stronger_model.to(device)\n",
    "                        # strong_inputs = inputs[j]\n",
    "                        # strong_inputs = strong_inputs.unsqueeze(0)\n",
    "                        # print(strong_inputs.shape)\n",
    "                        strong_outputs = stronger_model(strong_inputs)\n",
    "                        strong_preds = strong_outputs\n",
    "                        _, strong_labels = torch.tensor(torch.max(strong_preds, dim=1))\n",
    "                        # predicted_labels[j] = strong_labels\n",
    "                        # pred_in_each_step[2] += 1\n",
    "                        if (float(strong_preds[0][int(strong_labels)])< confidence_factor):\n",
    "                            stronger_model = model_list[3]\n",
    "                            stronger_model = stronger_model.to(device)\n",
    "                            # strong_inputs = inputs[j]\n",
    "                            # strong_inputs = strong_inputs.unsqueeze(0)\n",
    "                            # print(strong_inputs.shape)\n",
    "                            strong_outputs = stronger_model(strong_inputs)\n",
    "                            strong_preds = strong_outputs\n",
    "                            _, strong_labels = torch.tensor(torch.max(strong_preds, dim=1))\n",
    "                            \n",
    "                            pred_in_each_step[3] += 1\n",
    "                            predicted_labels[j] = strong_labels\n",
    "\n",
    "                        else:\n",
    "                            predicted_labels[j] = strong_labels\n",
    "                            pred_in_each_step[2] += 1\n",
    "\n",
    "                    else:\n",
    "                        predicted_labels[j] = strong_labels\n",
    "                        pred_in_each_step[1] += 1\n",
    "\n",
    "                else:\n",
    "                    pred_in_each_step[0] += 1\n",
    "                    \n",
    "            time_elapsed = time.time() - since   \n",
    "        \n",
    "            # print('\\033[32m' + f'corrects:: {correct}' + '\\033[0m')            if you uncomment this line you can see totall corrects on that epoch\n",
    "\n",
    "            total = total + inputs.size(0)\n",
    "            # print('\\033[34m' + f'total: {total}' + '\\033[0m')                  if you uncomment this line you can see total number of data processed in each epoch \n",
    "        \n",
    "        for k, steps_power in enumerate(pred_in_each_step):\n",
    "            print('\\033[36m' + f\"{pred_in_each_step[k]} predicted in {k}th layer\" + '\\033[0m')\n",
    "            \n",
    "        print('\\033[34m' + f'run-time(per input in second) = {time_elapsed/20}' + '\\033[0m')\n",
    "        print(f\"datas handled by each layer = {pred_in_each_step}\")\n",
    "        \n",
    "        for i, train_mode in enumerate(was_training):\n",
    "            model_list[i].train(mode=train_mode)\n",
    "\n",
    "    \n",
    "    return {\"run-time\": time_elapsed,\n",
    "           }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9144b25a-c0a1-4534-9d81-37f2a18162ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "class FireModule(nn.Module):\n",
    "    def __init__(self, in_channels, s1x1, e1x1, e3x3):\n",
    "        super(FireModule, self).__init__()\n",
    "        self.squeeze = nn.Conv2d(in_channels, s1x1, kernel_size=1)\n",
    "        self.expand_1x1 = nn.Conv2d(s1x1, e1x1, kernel_size=1)\n",
    "        self.expand_3x3 = nn.Conv2d(s1x1, e3x3, kernel_size=3, padding=1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.squeeze(x))\n",
    "        return torch.cat([\n",
    "            F.relu(self.expand_1x1(x)),\n",
    "            F.relu(self.expand_3x3(x))\n",
    "        ], dim=1)\n",
    "\n",
    "\n",
    "class SqueezeNet(nn.Module):\n",
    "    def __init__(self, num_classes=9):\n",
    "        super(SqueezeNet, self).__init__()\n",
    "        self.num_classes = 9\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 96, kernel_size=11, stride=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n",
    "            FireModule(96, 64, 128, 128),\n",
    "            # FireModule(64, 16, 64, 64),\n",
    "            # FireModule(128, 16, 64, 64),\n",
    "            # nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n",
    "            # FireModule(256, 32, 128, 128),\n",
    "            # FireModule(256, 48, 192, 192),\n",
    "            # FireModule(384, 48, 192, 192),\n",
    "            # FireModule(384, 64, 256, 256),\n",
    "        #     nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n",
    "        #     # FireModule(512, 64, 256, 256)\n",
    "        )\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(p=0.5),\n",
    "            nn.Conv2d(256,self.num_classes, kernel_size=3),\n",
    "            nn.ReLU(inplace=True),\n",
    "            # nn.Dropout(p=0.5),\n",
    "            # nn.Conv2d(128, self.num_classes, kernel_size=1),\n",
    "            # nn.ReLU(inplace=True),\n",
    "            nn.AdaptiveAvgPool2d((1, 1)),\n",
    "            # torch.nn.Softmax(dim = 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.classifier(x)\n",
    "        return x.view(-1, self.num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "455be7e1-1e00-4904-a2ff-6d51cf6fa7e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SqueezeNet(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d(3, 96, kernel_size=(11, 11), stride=(2, 2))\n",
       "    (1): ReLU(inplace=True)\n",
       "    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=True)\n",
       "    (3): FireModule(\n",
       "      (squeeze): Conv2d(96, 64, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand_1x1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n",
       "      (expand_3x3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    )\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Dropout(p=0.5, inplace=False)\n",
       "    (1): Conv2d(256, 9, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n",
    "\n",
    "model1 = SqueezeNet()\n",
    "model1 = model1.to(device)\n",
    "\n",
    "model1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4a1c0ac6-bc64-4e73-9255-ebf1e6a82f72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 1000])\n",
      "5636720\n",
      "torch.Size([5, 1000])\n",
      "2382944\n",
      "torch.Size([5, 1000])\n",
      "1331472\n"
     ]
    }
   ],
   "source": [
    "from einops import rearrange\n",
    "\n",
    "\n",
    "def conv_1x1_bn(inp, oup):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(inp, oup, 1, 1, 0, bias=False),\n",
    "        nn.BatchNorm2d(oup),\n",
    "        nn.SiLU()\n",
    "    )\n",
    "\n",
    "\n",
    "def conv_nxn_bn(inp, oup, kernal_size=3, stride=1):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(inp, oup, kernal_size, stride, 1, bias=False),\n",
    "        nn.BatchNorm2d(oup),\n",
    "        nn.SiLU()\n",
    "    )\n",
    "\n",
    "\n",
    "class PreNorm(nn.Module):\n",
    "    def __init__(self, dim, fn):\n",
    "        super().__init__()\n",
    "        self.norm = nn.LayerNorm(dim)\n",
    "        self.fn = fn\n",
    "    \n",
    "    def forward(self, x, **kwargs):\n",
    "        return self.fn(self.norm(x), **kwargs)\n",
    "\n",
    "\n",
    "class FeedForward(nn.Module):\n",
    "    def __init__(self, dim, hidden_dim, dropout=0.):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(dim, hidden_dim),\n",
    "            nn.SiLU(),\n",
    "            nn.Dropout(dropout),\n",
    "            nn.Linear(hidden_dim, dim),\n",
    "            nn.Dropout(dropout)\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.net(x)\n",
    "\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, dim, heads=8, dim_head=64, dropout=0.):\n",
    "        super().__init__()\n",
    "        inner_dim = dim_head *  heads\n",
    "        project_out = not (heads == 1 and dim_head == dim)\n",
    "\n",
    "        self.heads = heads\n",
    "        self.scale = dim_head ** -0.5\n",
    "\n",
    "        self.attend = nn.Softmax(dim = -1)\n",
    "        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n",
    "\n",
    "        self.to_out = nn.Sequential(\n",
    "            nn.Linear(inner_dim, dim),\n",
    "            nn.Dropout(dropout)\n",
    "        ) if project_out else nn.Identity()\n",
    "\n",
    "    def forward(self, x):\n",
    "        qkv = self.to_qkv(x).chunk(3, dim=-1)\n",
    "        q, k, v = map(lambda t: rearrange(t, 'b p n (h d) -> b p h n d', h = self.heads), qkv)\n",
    "\n",
    "        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n",
    "        attn = self.attend(dots)\n",
    "        out = torch.matmul(attn, v)\n",
    "        out = rearrange(out, 'b p h n d -> b p n (h d)')\n",
    "        return self.to_out(out)\n",
    "\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([])\n",
    "        for _ in range(depth):\n",
    "            self.layers.append(nn.ModuleList([\n",
    "                PreNorm(dim, Attention(dim, heads, dim_head, dropout)),\n",
    "                PreNorm(dim, FeedForward(dim, mlp_dim, dropout))\n",
    "            ]))\n",
    "    \n",
    "    def forward(self, x):\n",
    "        for attn, ff in self.layers:\n",
    "            x = attn(x) + x\n",
    "            x = ff(x) + x\n",
    "        return x\n",
    "\n",
    "\n",
    "class MV2Block(nn.Module):\n",
    "    def __init__(self, inp, oup, stride=1, expansion=4):\n",
    "        super().__init__()\n",
    "        self.stride = stride\n",
    "        assert stride in [1, 2]\n",
    "\n",
    "        hidden_dim = int(inp * expansion)\n",
    "        self.use_res_connect = self.stride == 1 and inp == oup\n",
    "\n",
    "        if expansion == 1:\n",
    "            self.conv = nn.Sequential(\n",
    "                # dw\n",
    "                nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),\n",
    "                nn.BatchNorm2d(hidden_dim),\n",
    "                nn.SiLU(),\n",
    "                # pw-linear\n",
    "                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n",
    "                nn.BatchNorm2d(oup),\n",
    "            )\n",
    "        else:\n",
    "            self.conv = nn.Sequential(\n",
    "                # pw\n",
    "                nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False),\n",
    "                nn.BatchNorm2d(hidden_dim),\n",
    "                nn.SiLU(),\n",
    "                # dw\n",
    "                nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),\n",
    "                nn.BatchNorm2d(hidden_dim),\n",
    "                nn.SiLU(),\n",
    "                # pw-linear\n",
    "                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n",
    "                nn.BatchNorm2d(oup),\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.use_res_connect:\n",
    "            return x + self.conv(x)\n",
    "        else:\n",
    "            return self.conv(x)\n",
    "\n",
    "\n",
    "class MobileViTBlock(nn.Module):\n",
    "    def __init__(self, dim, depth, channel, kernel_size, patch_size, mlp_dim, dropout=0.):\n",
    "        super().__init__()\n",
    "        self.ph, self.pw = patch_size\n",
    "\n",
    "        self.conv1 = conv_nxn_bn(channel, channel, kernel_size)\n",
    "        self.conv2 = conv_1x1_bn(channel, dim)\n",
    "\n",
    "        self.transformer = Transformer(dim, depth, 4, 8, mlp_dim, dropout)\n",
    "\n",
    "        self.conv3 = conv_1x1_bn(dim, channel)\n",
    "        self.conv4 = conv_nxn_bn(2 * channel, channel, kernel_size)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        y = x.clone()\n",
    "\n",
    "        # Local representations\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        \n",
    "        # Global representations\n",
    "        _, _, h, w = x.shape\n",
    "        x = rearrange(x, 'b d (h ph) (w pw) -> b (ph pw) (h w) d', ph=self.ph, pw=self.pw)\n",
    "        x = self.transformer(x)\n",
    "        x = rearrange(x, 'b (ph pw) (h w) d -> b d (h ph) (w pw)', h=h//self.ph, w=w//self.pw, ph=self.ph, pw=self.pw)\n",
    "\n",
    "        # Fusion\n",
    "        x = self.conv3(x)\n",
    "        x = torch.cat((x, y), 1)\n",
    "        x = self.conv4(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class MobileViT(nn.Module):\n",
    "    def __init__(self, image_size, dims, channels, num_classes, expansion=4, kernel_size=3, patch_size=(2, 2)):\n",
    "        super().__init__()\n",
    "        ih, iw = image_size\n",
    "        ph, pw = patch_size\n",
    "        assert ih % ph == 0 and iw % pw == 0\n",
    "\n",
    "        L = [2, 4, 3]\n",
    "\n",
    "        self.conv1 = conv_nxn_bn(3, channels[0], stride=2)\n",
    "\n",
    "        self.mv2 = nn.ModuleList([])\n",
    "        self.mv2.append(MV2Block(channels[0], channels[1], 1, expansion))\n",
    "        self.mv2.append(MV2Block(channels[1], channels[2], 2, expansion))\n",
    "        self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion))\n",
    "        self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion))   # Repeat\n",
    "        self.mv2.append(MV2Block(channels[3], channels[4], 2, expansion))\n",
    "        self.mv2.append(MV2Block(channels[5], channels[6], 2, expansion))\n",
    "        self.mv2.append(MV2Block(channels[7], channels[8], 2, expansion))\n",
    "        \n",
    "        self.mvit = nn.ModuleList([])\n",
    "        self.mvit.append(MobileViTBlock(dims[0], L[0], channels[5], kernel_size, patch_size, int(dims[0]*2)))\n",
    "        self.mvit.append(MobileViTBlock(dims[1], L[1], channels[7], kernel_size, patch_size, int(dims[1]*4)))\n",
    "        self.mvit.append(MobileViTBlock(dims[2], L[2], channels[9], kernel_size, patch_size, int(dims[2]*4)))\n",
    "\n",
    "        self.conv2 = conv_1x1_bn(channels[-2], channels[-1])\n",
    "\n",
    "        self.pool = nn.AvgPool2d(ih//32, 1)\n",
    "        self.fc = nn.Linear(channels[-1], num_classes, bias=False)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.mv2[0](x)\n",
    "\n",
    "        x = self.mv2[1](x)\n",
    "        x = self.mv2[2](x)\n",
    "        x = self.mv2[3](x)      # Repeat\n",
    "\n",
    "        x = self.mv2[4](x)\n",
    "        x = self.mvit[0](x)\n",
    "\n",
    "        x = self.mv2[5](x)\n",
    "        x = self.mvit[1](x)\n",
    "\n",
    "        x = self.mv2[6](x)\n",
    "        x = self.mvit[2](x)\n",
    "        x = self.conv2(x)\n",
    "\n",
    "        x = self.pool(x).view(-1, x.shape[1])\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "def mobilevit_xxs():\n",
    "    dims = [64, 80, 96]\n",
    "    channels = [16, 16, 24, 24, 48, 48, 64, 64, 80, 80, 320]\n",
    "    return MobileViT((256, 256), dims, channels, num_classes=1000, expansion=2)\n",
    "\n",
    "\n",
    "def mobilevit_xs():\n",
    "    dims = [96, 120, 144]\n",
    "    channels = [16, 32, 48, 48, 64, 64, 80, 80, 96, 96, 384]\n",
    "    return MobileViT((256, 256), dims, channels, num_classes=1000)\n",
    "\n",
    "\n",
    "def mobilevit_s():\n",
    "    dims = [144, 192, 240]\n",
    "    channels = [16, 32, 64, 64, 96, 96, 128, 128, 160, 160, 640]\n",
    "    return MobileViT((256, 256), dims, channels, num_classes=1000)\n",
    "\n",
    "\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    img = torch.randn(5, 3, 256, 256)\n",
    "    \n",
    "    vit = mobilevit_s()\n",
    "    out = vit(img)\n",
    "    print(out.shape)\n",
    "    print(count_parameters(vit))\n",
    "\n",
    "    vit = mobilevit_xs()\n",
    "    out = vit(img)\n",
    "    print(out.shape)\n",
    "    print(count_parameters(vit))\n",
    "    \n",
    "    vit = mobilevit_xxs()\n",
    "    out = vit(img)\n",
    "    print(out.shape)\n",
    "    print(count_parameters(vit))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6df14a18-a3e8-4b2f-bb3e-49f3ce207002",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model:\n",
      "Linear_layers_fc(\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class Linear_layers_fc(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Linear_layers_fc, self).__init__()\n",
    "\n",
    "        # self.linear1 = torch.nn.Linear(1000, 256)\n",
    "        # self.bachNoem1 = nn.BatchNorm1d(256)\n",
    "        # self.activation1 = torch.nn.ReLU()\n",
    "        # self.dropout1 = nn.Dropout(p=0.2)\n",
    "\n",
    "        # self.linear5 = torch.nn.Linear(32, 9)\n",
    "\n",
    "        self.softmax = torch.nn.Softmax(dim = 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        # x = self.linear5(x)\n",
    "\n",
    "        x = self.softmax(x)\n",
    "        return x\n",
    "\n",
    "linear_layers_ViT= Linear_layers_fc()\n",
    "\n",
    "print('The model:')\n",
    "print(linear_layers_ViT)\n",
    "\n",
    "# print('\\n\\nModel params:')\n",
    "# for param in MLP_model.parameters():\n",
    "#     print(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "834f7c40-950e-4f15-84d6-6dcc5fa4e7eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model:\n",
      "CNN_last(\n",
      "  (conv): Conv2d(80, 9, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "  (bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (activation): ReLU6()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class CNN_last(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(CNN_last, self).__init__()\n",
    "\n",
    "        # self.linear1 = torch.nn.Linear(1000, 256)\n",
    "        # self.bachNoem1 = nn.BatchNorm1d(256)\n",
    "        # self.activation1 = torch.nn.ReLU()\n",
    "        # self.dropout1 = nn.Dropout(p=0.2)\n",
    "\n",
    "        self.conv = nn.Conv2d(80, 9, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
    "        self.bn = nn.BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
    "        self.activation = nn.ReLU6()\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = self.conv(x)\n",
    "        x = self.bn(x)\n",
    "        x = self.activation(x)\n",
    "        \n",
    "        return x\n",
    "\n",
    "CNN_last_layer_ViT= CNN_last()\n",
    "\n",
    "print('The model:')\n",
    "print(CNN_last_layer_ViT)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1b3f2f2a-29ad-44ab-837b-0c31ef67689f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MobileViT(\n",
       "  (conv1): Sequential(\n",
       "    (0): Conv2d(3, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): SiLU()\n",
       "  )\n",
       "  (mv2): ModuleList(\n",
       "    (0): MV2Block(\n",
       "      (conv): Sequential(\n",
       "        (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "        (3): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)\n",
       "        (4): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (5): SiLU()\n",
       "        (6): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (7): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): MV2Block(\n",
       "      (conv): Sequential(\n",
       "        (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "        (3): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=32, bias=False)\n",
       "        (4): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (5): SiLU()\n",
       "        (6): Conv2d(32, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (7): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (2-3): 2 x MV2Block(\n",
       "      (conv): Sequential(\n",
       "        (0): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "        (3): Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=48, bias=False)\n",
       "        (4): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (5): SiLU()\n",
       "        (6): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (7): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (4): MV2Block(\n",
       "      (conv): Sequential(\n",
       "        (0): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "        (3): Conv2d(48, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=48, bias=False)\n",
       "        (4): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (5): SiLU()\n",
       "        (6): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (7): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (5): MV2Block(\n",
       "      (conv): Sequential(\n",
       "        (0): Conv2d(48, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "        (3): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96, bias=False)\n",
       "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (5): SiLU()\n",
       "        (6): Conv2d(96, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (6): MV2Block(\n",
       "      (conv): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "        (3): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=128, bias=False)\n",
       "        (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (5): SiLU()\n",
       "        (6): Conv2d(128, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (7): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (mvit): ModuleList(\n",
       "    (0): MobileViTBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2d(48, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (transformer): Transformer(\n",
       "        (layers): ModuleList(\n",
       "          (0-1): 2 x ModuleList(\n",
       "            (0): PreNorm(\n",
       "              (norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "              (fn): Attention(\n",
       "                (attend): Softmax(dim=-1)\n",
       "                (to_qkv): Linear(in_features=64, out_features=96, bias=False)\n",
       "                (to_out): Sequential(\n",
       "                  (0): Linear(in_features=32, out_features=64, bias=True)\n",
       "                  (1): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "            (1): PreNorm(\n",
       "              (norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "              (fn): FeedForward(\n",
       "                (net): Sequential(\n",
       "                  (0): Linear(in_features=64, out_features=128, bias=True)\n",
       "                  (1): SiLU()\n",
       "                  (2): Dropout(p=0.0, inplace=False)\n",
       "                  (3): Linear(in_features=128, out_features=64, bias=True)\n",
       "                  (4): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (conv3): Sequential(\n",
       "        (0): Conv2d(64, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (conv4): Sequential(\n",
       "        (0): Conv2d(96, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "    )\n",
       "    (1): MobileViTBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2d(64, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (transformer): Transformer(\n",
       "        (layers): ModuleList(\n",
       "          (0-3): 4 x ModuleList(\n",
       "            (0): PreNorm(\n",
       "              (norm): LayerNorm((80,), eps=1e-05, elementwise_affine=True)\n",
       "              (fn): Attention(\n",
       "                (attend): Softmax(dim=-1)\n",
       "                (to_qkv): Linear(in_features=80, out_features=96, bias=False)\n",
       "                (to_out): Sequential(\n",
       "                  (0): Linear(in_features=32, out_features=80, bias=True)\n",
       "                  (1): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "            (1): PreNorm(\n",
       "              (norm): LayerNorm((80,), eps=1e-05, elementwise_affine=True)\n",
       "              (fn): FeedForward(\n",
       "                (net): Sequential(\n",
       "                  (0): Linear(in_features=80, out_features=320, bias=True)\n",
       "                  (1): SiLU()\n",
       "                  (2): Dropout(p=0.0, inplace=False)\n",
       "                  (3): Linear(in_features=320, out_features=80, bias=True)\n",
       "                  (4): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (conv3): Sequential(\n",
       "        (0): Conv2d(80, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (conv4): Sequential(\n",
       "        (0): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "    )\n",
       "    (2): MobileViTBlock(\n",
       "      (conv1): Sequential(\n",
       "        (0): Conv2d(80, 80, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (conv2): Sequential(\n",
       "        (0): Conv2d(80, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (transformer): Transformer(\n",
       "        (layers): ModuleList(\n",
       "          (0-2): 3 x ModuleList(\n",
       "            (0): PreNorm(\n",
       "              (norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)\n",
       "              (fn): Attention(\n",
       "                (attend): Softmax(dim=-1)\n",
       "                (to_qkv): Linear(in_features=96, out_features=96, bias=False)\n",
       "                (to_out): Sequential(\n",
       "                  (0): Linear(in_features=32, out_features=96, bias=True)\n",
       "                  (1): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "            (1): PreNorm(\n",
       "              (norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)\n",
       "              (fn): FeedForward(\n",
       "                (net): Sequential(\n",
       "                  (0): Linear(in_features=96, out_features=384, bias=True)\n",
       "                  (1): SiLU()\n",
       "                  (2): Dropout(p=0.0, inplace=False)\n",
       "                  (3): Linear(in_features=384, out_features=96, bias=True)\n",
       "                  (4): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (conv3): Sequential(\n",
       "        (0): Conv2d(96, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "      (conv4): Sequential(\n",
       "        (0): Conv2d(160, 80, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (2): SiLU()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (conv2): CNN_last(\n",
       "    (conv): Conv2d(80, 9, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "    (bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (activation): ReLU6()\n",
       "  )\n",
       "  (pool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  (fc): Linear_layers_fc(\n",
       "    (softmax): Softmax(dim=1)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from PIL import Image\n",
    "import torch\n",
    "import timm\n",
    "import requests\n",
    "import torchvision.transforms as transforms\n",
    "from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD\n",
    "\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n",
    "\n",
    "model2 = mobilevit_xxs()\n",
    "\n",
    "model2.conv2 = CNN_last_layer_ViT\n",
    "model2.fc = linear_layers_ViT\n",
    "model2.pool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "\n",
    "model2 = model2.to(device)\n",
    "\n",
    "model2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c469132d-a029-4458-b867-f8c771da2ba1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model:\n",
      "Linear_layers_fc(\n",
      "  (linear2): Linear(in_features=1280, out_features=9, bias=True)\n",
      "  (dropout2): Dropout(p=0.2, inplace=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class Linear_layers_fc(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Linear_layers_fc, self).__init__()\n",
    "\n",
    "\n",
    "        # self.linear1 = torch.nn.Linear(1280, 128)\n",
    "        # self.bachNoem1 = nn.BatchNorm1d(128)\n",
    "        # self.activation1 = torch.nn.ReLU()\n",
    "        # self.dropout1 = nn.Dropout(p=0.2)\n",
    "\n",
    "        self.linear2 = torch.nn.Linear(1280, 9)\n",
    "        # self.bachNoem2 = nn.BatchNorm1d(64)\n",
    "        # self.activation2 = torch.nn.ReLU()\n",
    "        self.dropout2 = nn.Dropout(p=0.2)\n",
    "\n",
    "        # self.linear5 = torch.nn.Linear(256, 9)\n",
    "\n",
    "        self.softmax = torch.nn.Softmax(dim = 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x = self.dropout1(x)\n",
    "        # x = self.linear1(x)\n",
    "        # x = self.bachNoem1(x)\n",
    "        # x = self.activation1(x)\n",
    "\n",
    "        x = self.dropout2(x)\n",
    "        x = self.linear2(x)\n",
    "        # x = self.bachNoem2(x)\n",
    "        # x = self.activation2(x)\n",
    "\n",
    "        x = self.softmax(x)\n",
    "        return x\n",
    "\n",
    "linear_layers_mobilenet= Linear_layers_fc()\n",
    "\n",
    "print('The model:')\n",
    "print(linear_layers_mobilenet)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "eae8e64c-7fc8-4f01-8e71-3a68f62c1f4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/ghostoftime111/.cache/torch/hub/pytorch_vision_v0.10.0\n",
      "/home/ghostoftime111/anaconda3/envs/DL/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "/home/ghostoftime111/anaconda3/envs/DL/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=MobileNet_V2_Weights.IMAGENET1K_V1`. You can also use `weights=MobileNet_V2_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model3 = torch.hub.load('pytorch/vision:v0.10.0', 'mobilenet_v2', pretrained=True)\n",
    "\n",
    "num_classes = 9\n",
    "model3.classifier = linear_layers_mobilenet\n",
    "\n",
    "model3 = model3.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5f72a1cf-f74b-449b-a597-461fac9d54be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ghostoftime111/anaconda3/envs/DL/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ResNet(\n",
       "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu): ReLU(inplace=True)\n",
       "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "  (layer1): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  (fc): Linear(in_features=512, out_features=9, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n",
    "\n",
    "class_num = 9\n",
    "\n",
    "model4 = models.resnet18(pretrained=True)\n",
    "num_ftrs = model4.fc.in_features\n",
    "model4.fc = nn.Linear(num_ftrs, class_num)\n",
    "\n",
    "model4 = model4.to(device)\n",
    "model4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4fd4aad5-722e-4302-8cb5-054b25823162",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_1_dir = \"./squeeznet_combined_best_256_better.pth\"\n",
    "model_2_dir = \"./vit-mobile-combined.pth\"\n",
    "model_3_dir = \"./mobilenet_v2_9_finetuned_combined_256.pth\"\n",
    "model_4_dir = \"./resnet18_finetuned-smote-256_new_better2.pth\"\n",
    "\n",
    "# model1 = torch.load(model_1_dir)\n",
    "# model2 = torch.load(model_2_dir)\n",
    "# model3 = torch.load(model_3_dir)\n",
    "# model4 = torch.load(model_4_dir)\n",
    "\n",
    "model1.load_state_dict(torch.load(model_1_dir))\n",
    "model1 = nn.Sequential(model1, nn.Softmax())\n",
    "model2.load_state_dict(torch.load(model_2_dir))\n",
    "model3.load_state_dict(torch.load(model_3_dir))\n",
    "model4.load_state_dict(torch.load(model_4_dir))\n",
    "\n",
    "models_list = []\n",
    "models_list.append(model1)\n",
    "models_list.append(model2)\n",
    "models_list.append(model3)\n",
    "models_list.append(model4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cdabd13c-24b7-4335-b81e-18ea5132e585",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[36m5 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m14 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.1646178960800171\u001b[0m\n",
      "datas handled by each layer = [5, 0, 14, 1]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results = pipeline_module(models_list, confidence_factor=0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c87b82b1-d735-4ed4-93ea-aaca187a22c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Confidence factor: 0.4\n",
      "\u001b[36m20 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.060647571086883546\u001b[0m\n",
      "datas handled by each layer = [20, 0, 0, 0]\n",
      "\n",
      "Confidence factor: 0.6\n",
      "\u001b[36m20 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.03837306499481201\u001b[0m\n",
      "datas handled by each layer = [20, 0, 0, 0]\n",
      "\n",
      "Confidence factor: 0.8\n",
      "\u001b[36m18 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.03335083723068237\u001b[0m\n",
      "datas handled by each layer = [18, 1, 1, 0]\n",
      "\n",
      "Confidence factor: 0.9\n",
      "\u001b[36m18 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.029581952095031738\u001b[0m\n",
      "datas handled by each layer = [18, 1, 1, 0]\n",
      "\n",
      "Confidence factor: 0.91\n",
      "\u001b[36m18 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.031081390380859376\u001b[0m\n",
      "datas handled by each layer = [18, 1, 1, 0]\n",
      "\n",
      "Confidence factor: 0.92\n",
      "\u001b[36m17 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m2 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.03374029397964477\u001b[0m\n",
      "datas handled by each layer = [17, 2, 1, 0]\n",
      "\n",
      "Confidence factor: 0.93\n",
      "\u001b[36m17 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m2 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.0385834813117981\u001b[0m\n",
      "datas handled by each layer = [17, 2, 1, 0]\n",
      "\n",
      "Confidence factor: 0.94\n",
      "\u001b[36m17 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m2 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.03227407932281494\u001b[0m\n",
      "datas handled by each layer = [17, 2, 1, 0]\n",
      "\n",
      "Confidence factor: 0.95\n",
      "\u001b[36m14 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m5 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.03832001686096191\u001b[0m\n",
      "datas handled by each layer = [14, 5, 1, 0]\n",
      "\n",
      "Confidence factor: 0.96\n",
      "\u001b[36m14 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m5 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.0400506854057312\u001b[0m\n",
      "datas handled by each layer = [14, 5, 1, 0]\n",
      "\n",
      "Confidence factor: 0.97\n",
      "\u001b[36m13 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m5 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m2 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.04380578994750976\u001b[0m\n",
      "datas handled by each layer = [13, 5, 2, 0]\n",
      "\n",
      "Confidence factor: 0.98\n",
      "\u001b[36m9 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m10 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.08081238269805908\u001b[0m\n",
      "datas handled by each layer = [9, 0, 10, 1]\n",
      "\n",
      "Confidence factor: 0.99\n",
      "\u001b[36m5 predicted in 0th layer\u001b[0m\n",
      "\u001b[36m0 predicted in 1th layer\u001b[0m\n",
      "\u001b[36m14 predicted in 2th layer\u001b[0m\n",
      "\u001b[36m1 predicted in 3th layer\u001b[0m\n",
      "\u001b[34mrun-time(per input in second) = 0.11941339969635009\u001b[0m\n",
      "datas handled by each layer = [5, 0, 14, 1]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "row = 2\n",
    "conf_list = [0.4, 0.6, 0.8, 0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99]\n",
    "for i in range(13):\n",
    "    print()\n",
    "    results = pipeline_module(models_list, confidence_factor=conf_list[i])\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "756e40a3-4703-4dfe-9564-fe19f9d0474d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d830a715-f56d-4366-a5f3-aa41d2d1f3cf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7775fce-055b-424b-9e9f-bd46cbcbc09b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f561a80e-5e58-4146-a522-57125f789972",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d5b12b5-8073-4709-86f1-5199f1fd4110",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21d3345c-83cc-4a30-b78f-7973a988bc17",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "509dd4ee-9a4a-4bdf-8c64-810afe40f77f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5f6ace2-fe7a-4c3d-942d-f103498a150d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da1fe35c-ad8b-4b1c-a662-1267e3e4d18b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64e390df-2c71-4e02-ac55-d7638fccc5d7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9442fda-e5ab-48be-a90c-6f2134f7298c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "858f097f-6700-4c30-b0e0-8035e850bd59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08775a39-acc1-4539-b826-7a7d5cc0cabf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "296cdc21-1be4-44e5-b1a5-c835ad9c78a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80eef16c-e33f-4f39-a9ab-38ced04a25ce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa93ba3-71f0-45a9-8ddf-6739be425c4e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2a2736d-bd4e-4590-adf1-e31bfb5c2b0c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1d24b9a-e4ea-4550-a813-1acba7eeb6e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc2a9b1a-d72f-4f50-aaa8-ae058493c2d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}