text_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import joblib\n",
    "import torch\n",
    "from sklearn import preprocessing\n",
    "from word2vec_util import TextCorpusProcess\n",
    "from skip_gram_neg import SkipGramNeg\n",
    "from text_cnn_model import TextClassification\n",
    "import time\n",
    "import copy\n",
    "from torch import Tensor, nn\n",
    "import datetime\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import os\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/notebooks/data/val_txt_descs.pkl']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save train label data for easy load\n",
    "train_pklname = os.path.join('/notebooks/data/', 'img_prepared_train.pkl')\n",
    "train_data = joblib.load(train_pklname)\n",
    "train_text_label = train_data['label']\n",
    "joblib.dump(train_text_label, os.path.join(\n",
    "    '/notebooks/data/', 'train_txt_labels.pkl'))\n",
    "\n",
    "# save val label data for easy load\n",
    "val_pklname = os.path.join('/notebooks/data/', 'img_prepared_val.pkl')\n",
    "val_data = joblib.load(val_pklname)\n",
    "val_text_label = val_data['label']\n",
    "joblib.dump(val_text_label, os.path.join(\n",
    "    '/notebooks/data/', 'val_txt_labels.pkl'))\n",
    "\n",
    "# save val desc data for easy load\n",
    "val_text_descs = val_data['desc']\n",
    "joblib.dump(val_text_descs, os.path.join(\n",
    "    '/notebooks/data/', 'val_txt_descs.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load saved train and val labels\n",
    "train_pklname = os.path.join('/notebooks/data', 'train_txt_labels.pkl')\n",
    "train_txt_labels = joblib.load(train_pklname)\n",
    "\n",
    "val_pklname = os.path.join('/notebooks/data', 'val_txt_labels.pkl')\n",
    "val_txt_labels = joblib.load(val_pklname)\n",
    "\n",
    "# load the saved processed text data\n",
    "text_path = os.path.join('/notebooks/data', 'text_processed.pkl')\n",
    "traintextCorpus = joblib.load(text_path)\n",
    "train_descs_idx = traintextCorpus.prod_word_idx_lists\n",
    "\n",
    "# # save train_descs_idx\n",
    "# joblib.dump(train_descs_idx, os.path.join(\n",
    "#     '/notebooks/data/' + 'train_descs_idx.pkl'))\n",
    "\n",
    "# load saved val description data\n",
    "val_text_descs = joblib.load(os.path.join(\n",
    "    '/notebooks/data/', 'val_txt_descs.pkl'))\n",
    "valtextCorpus = TextCorpusProcess(prod_descs=val_text_descs)\n",
    "val_descs = valtextCorpus.prod_word_lists\n",
    "\n",
    "# load the word2vec model\n",
    "# hyperparams\n",
    "EMBED_DIM = 128\n",
    "BATCH_SIZE = 16\n",
    "NEG_SAMPLE_SIZE = 5\n",
    "\n",
    "# instantiating the model\n",
    "model = SkipGramNeg(\n",
    "    vocab_size=traintextCorpus.vocab_size,\n",
    "    embed_dim=EMBED_DIM,\n",
    "    neg_sample_size=NEG_SAMPLE_SIZE,\n",
    "    batch_size=BATCH_SIZE,\n",
    ")\n",
    "# load saved word2vec model\n",
    "embedding_file = os.path.join(\n",
    "    '/notebooks/models', 'best_word2vec_ebedding_model.pt')\n",
    "model.load_state_dict(torch.load(embedding_file))\n",
    "embeddings = model.in_embed\n",
    "embeddings.requires_grad_(False)\n",
    "\n",
    "# prepare embedded train data\n",
    "train_words_emb = []\n",
    "for desc_idxs in train_descs_idx:\n",
    "    words_idx = torch.from_numpy(np.array(desc_idxs).reshape(-1, 1))\n",
    "    words_emb = embeddings(words_idx).squeeze()\n",
    "    words_emb = torch.concat([emb for emb in words_emb])\n",
    "    train_words_emb.append(words_emb)\n",
    "\n",
    "# # save embedded train text data\n",
    "# joblib.dump(train_words_emb, os.path.join(\n",
    "#     '/notebooks/data/' + 'train_words_emb.pkl'))\n",
    "\n",
    "# # load embedded train text data\n",
    "# train_words_emb = joblib.load(os.path.join(\n",
    "#     '/notebooks/data/', 'train_words_emb.pkl'))\n",
    "\n",
    "# prepare embedded val data\n",
    "# ingnores words not in training data\n",
    "val_descs_idx = []\n",
    "for desc in val_descs:\n",
    "    desc_idxs = []\n",
    "    for word in desc:\n",
    "        word_idx = traintextCorpus.word_to_idx.get(word)\n",
    "        if not word_idx is None:\n",
    "            desc_idxs.append(word_idx)\n",
    "    if not desc_idxs:\n",
    "        raise ValueError('Empty description!')\n",
    "    val_descs_idx.append(desc_idxs)\n",
    "\n",
    "# joblib.dump(val_descs_idx, os.path.join(\n",
    "#     '/notebooks/data/' + 'val_descs_idx.pkl'))\n",
    "\n",
    "# prepare embedded val data\n",
    "val_words_emb = []\n",
    "for desc_idxs in val_descs_idx:\n",
    "    words_idx = torch.from_numpy(np.array(desc_idxs).reshape(-1, 1))\n",
    "    words_emb = embeddings(words_idx).squeeze()\n",
    "    words_emb = torch.concat([emb for emb in words_emb])\n",
    "    val_words_emb.append(words_emb)\n",
    "\n",
    "# # save embedded val text data\n",
    "# joblib.dump(val_words_emb, os.path.join(\n",
    "#     '/notebooks/data/' + 'val_words_emb.pkl'))\n",
    "\n",
    "# # load embedded val text data\n",
    "# val_words_emb = joblib.load(os.path.join(\n",
    "#     '/notebooks/data/', 'val_words_emb.pkl'))\n",
    "\n",
    "# create ordinal encoded train target\n",
    "label_train = np.array(train_txt_labels).reshape(-1, 1)\n",
    "target_enc = preprocessing.OrdinalEncoder()\n",
    "target_enc.fit(label_train)\n",
    "target_train = target_enc.transform(label_train)\n",
    "target_train = target_train.reshape(-1)\n",
    "\n",
    "# create ordinal encoded val target\n",
    "label_val = np.array(val_txt_labels).reshape(-1, 1)\n",
    "target_val = target_enc.transform(label_val)\n",
    "target_val = target_val.reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/notebooks/models/word_to_idx.pkl']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(traintextCorpus.word_to_idx, os.path.join(\n",
    "    '/notebooks/models/', 'word_to_idx.pkl'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_wts = copy.deepcopy(embeddings.state_dict())\n",
    "torch.save(embeddings_wts, os.path.join(\n",
    "    '/notebooks', 'models', 'embeddings_wts.pt'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextEmbedDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, inputs, labels, max_word_len, embd_dim):\n",
    "        self.embd_dim = embd_dim\n",
    "        # expects word embeddings\n",
    "        self.inputs = inputs\n",
    "        # expects ordinal encoded classes\n",
    "        self.labels = labels\n",
    "        self.max_word_len = max_word_len\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        words = self.inputs[index]\n",
    "        pad_len = self.max_word_len * self.embd_dim - len(words)\n",
    "        assert(pad_len >= 0)\n",
    "        pad_len_r = pad_len // 2\n",
    "        pad_len_l = pad_len - pad_len_r\n",
    "        pad_r = torch.zeros(pad_len_r)\n",
    "        pad_l = torch.zeros(pad_len_l)\n",
    "        words_padded = torch.concat([pad_l, words, pad_r])\n",
    "        # words_padded = torch.transpose(words_padded,0, 1)\n",
    "        label = self.labels[index]\n",
    "        return words_padded, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "def train_model(model, criterion, optimizer, scheduler, writer, num_epochs=50, save_path=None):\n",
    "    since = time.time()\n",
    "\n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print(f'Epoch {epoch}/{num_epochs - 1}')\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()  # Set model to training mode\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.unsqueeze(1)\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.type(torch.LongTensor)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    loss = criterion(outputs, labels)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "            if phase == 'train':\n",
    "                scheduler.step()\n",
    "\n",
    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
    "            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
    "\n",
    "            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n",
    "\n",
    "            writer.add_scalar(f'Loss/{phase}', epoch_loss, epoch)\n",
    "            writer.add_scalar(f'Accuracy/{phase}', epoch_acc, epoch)\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "                # torch.save(best_model_wts, os.path.join(\n",
    "                #     save_path, 'best_text_cnn_model.pt'))\n",
    "\n",
    "        print()\n",
    "        writer.flush()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print(\n",
    "        f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n",
    "    print(f'Best val Acc: {best_acc:4f}')\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define train and val dataset\n",
    "\n",
    "# find max description word length\n",
    "max_train_desc_len = max(map(len, train_descs_idx))\n",
    "max_val_desc_len = max(map(len, val_descs_idx))\n",
    "max_desc_len = max_train_desc_len\n",
    "if max_val_desc_len > max_train_desc_len:\n",
    "    max_desc_len = max_val_desc_len\n",
    "\n",
    "trainEmbedDataset = TextEmbedDataset(\n",
    "    train_words_emb, target_train, max_desc_len, embd_dim=EMBED_DIM)\n",
    "valEmbedDataset = TextEmbedDataset(\n",
    "    val_words_emb, target_val, max_desc_len, embd_dim=EMBED_DIM)\n",
    "\n",
    "text_datasets = {'train': trainEmbedDataset, 'val': valEmbedDataset}\n",
    "dataset_sizes = {x: len(text_datasets[x]) for x in ['train', 'val']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/notebooks/models/max_desc_len.pkl']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(max_desc_len, '/notebooks/models/max_desc_len.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir_path = os.path.join('/notebooks', 'runs')\n",
    "save_dir_path = os.path.join('/notebooks', 'models')\n",
    "try:\n",
    "    os.makedirs(log_dir_path)\n",
    "except:\n",
    "    pass\n",
    "try:\n",
    "    os.makedirs(save_dir_path)\n",
    "except:\n",
    "    pass\n",
    "# %load_ext tensorboard\n",
    "# %tensorboard --logdir {log_dir_path}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/49\n",
      "----------\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n",
      "torch.Size([16, 63408])\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb Cell 11'\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000010?line=23'>24</a>\u001b[0m dataloaders \u001b[39m=\u001b[39m {x: torch\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mdata\u001b[39m.\u001b[39mDataLoader(text_datasets[x], batch_size\u001b[39m=\u001b[39mBATCH_SIZE,\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000010?line=24'>25</a>\u001b[0m                                               shuffle\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m, num_workers\u001b[39m=\u001b[39m\u001b[39m4\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000010?line=25'>26</a>\u001b[0m                \u001b[39mfor\u001b[39;00m x \u001b[39min\u001b[39;00m [\u001b[39m'\u001b[39m\u001b[39mtrain\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mval\u001b[39m\u001b[39m'\u001b[39m]}\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000010?line=27'>28</a>\u001b[0m \u001b[39m# train the model\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000010?line=28'>29</a>\u001b[0m model_ft \u001b[39m=\u001b[39m train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, writer,\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000010?line=29'>30</a>\u001b[0m                        num_epochs\u001b[39m=\u001b[39;49m\u001b[39m50\u001b[39;49m, save_path\u001b[39m=\u001b[39;49msave_dir_path)\n",
      "\u001b[1;32m/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb Cell 7'\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(model, criterion, optimizer, scheduler, writer, num_epochs, save_path)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000006?line=55'>56</a>\u001b[0m \u001b[39m# forward\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000006?line=56'>57</a>\u001b[0m \u001b[39m# track history if only in train\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000006?line=57'>58</a>\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mset_grad_enabled(phase \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mtrain\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000006?line=58'>59</a>\u001b[0m     outputs \u001b[39m=\u001b[39m model(inputs)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000006?line=59'>60</a>\u001b[0m     _, preds \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mmax(outputs, \u001b[39m1\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/shbz/Dropbox/Workspace/Aicore/VSworkspace/fb_marketplace_reco/text_classification.ipynb#ch0000006?line=60'>61</a>\u001b[0m     loss \u001b[39m=\u001b[39m criterion(outputs, labels)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1106\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1107\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1108\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1109\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1111\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m/notebooks/text_cnn_model.py:33\u001b[0m, in \u001b[0;36mTextClassification.forward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, inputs):\n\u001b[0;32m---> 33\u001b[0m     x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcnn_layers(inputs)\n\u001b[1;32m     34\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlinear_layer(x)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1106\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1107\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1108\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1109\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1111\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py:141\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    139\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m):\n\u001b[1;32m    140\u001b[0m     \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m:\n\u001b[0;32m--> 141\u001b[0m         \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m module(\u001b[39minput\u001b[39;49m)\n\u001b[1;32m    142\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39minput\u001b[39m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1106\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1107\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1108\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1109\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1111\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1112\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py:302\u001b[0m, in \u001b[0;36mConv1d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    301\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 302\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_conv_forward(\u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py:298\u001b[0m, in \u001b[0;36mConv1d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m    294\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpadding_mode \u001b[39m!=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mzeros\u001b[39m\u001b[39m'\u001b[39m:\n\u001b[1;32m    295\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39mconv1d(F\u001b[39m.\u001b[39mpad(\u001b[39minput\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpadding_mode),\n\u001b[1;32m    296\u001b[0m                     weight, bias, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstride,\n\u001b[1;32m    297\u001b[0m                     _single(\u001b[39m0\u001b[39m), \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdilation, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgroups)\n\u001b[0;32m--> 298\u001b[0m \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49mconv1d(\u001b[39minput\u001b[39;49m, weight, bias, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mstride,\n\u001b[1;32m    299\u001b[0m                 \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpadding, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdilation, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgroups)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# log dir for storing tensorboard files\n",
    "log_dir = os.path.join(\n",
    "    log_dir_path, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))\n",
    "writer = SummaryWriter(log_dir=log_dir)\n",
    "\n",
    "model_ft = TextClassification(\n",
    "    word_embd_dim=EMBED_DIM, num_classes=13, word_kernel_size=3)\n",
    "model_ft = model_ft.to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# SGD is the best optimizer\n",
    "optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "# Decay LR by a factor of 0.1 every 7 epochs\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n",
    "\n",
    "# 16 is the best batch size\n",
    "BATCH_SIZE = 16\n",
    "\n",
    "# create a dict of train and val dataloaders\n",
    "dataloaders = {x: torch.utils.data.DataLoader(text_datasets[x], batch_size=BATCH_SIZE,\n",
    "                                              shuffle=True, num_workers=4)\n",
    "               for x in ['train', 'val']}\n",
    "\n",
    "# train the model\n",
    "model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, writer,\n",
    "                       num_epochs=50, save_path=save_dir_path)\n",
    "\n",
    "# experiments\n",
    "# 2022-06-19_18-44-01: \n",
    "# nn.Conv1d(in_channels=1,\n",
    "#                       out_channels=8,\n",
    "#                       kernel_size=self.word_kernel_size * EMBED_DIM,\n",
    "#                       stride=EMBED_DIM,\n",
    "#                       padding=(self.word_kernel_size - 1) * EMBED_DIM),\n",
    "#             nn.ReLU(),\n",
    "#             nn.Flatten()\n",
    "#         )\n",
    "# nn.Linear(in_features=10560, out_features=13)\n",
    "# optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n",
    "# exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n",
    "# BATCH_SIZE = 16\n",
    "\n",
    "# 2022-06-19_19-18-03:\n",
    "# + nn.MaxPool1d(3, 3),\n",
    "\n",
    "#2022-06-19_19-42-43:\n",
    "# - nn.MaxPool1d(3, 3),\n",
    "# + out_channels=16\n",
    "\n",
    "#2022-06-19_19-49-02:\n",
    "# + out_channels=32\n",
    "\n",
    "#2022-06-19_20-04-31:\n",
    "# + word_kernel_size = 3\n",
    "\n",
    "#2022-06-19_20-10-39:\n",
    "# + nn.Conv1d(in_channels=32,\n",
    "        # out_channels = 8,\n",
    "        # kernel_size = 2,\n",
    "        # stride = 1,\n",
    "        # padding = 1),\n",
    "    # nn.ReLU(),\n",
    "\n",
    "#2022-06-19_20-18-00:\n",
    "# - nn.Conv1d(in_channels=32,\n",
    "# out_channels = 8,\n",
    "# kernel_size = 2,\n",
    "# stride = 1,\n",
    "# padding = 1),\n",
    "# nn.ReLU(),\n",
    "# - nn.ReLU(), (first CNN layer)\n",
    "# + nn.Tanh(), (first CNN layer)\n",
    "\n",
    "#2022-06-20_08-40-53:\n",
    "#  + nn.Sequential(\n",
    "#     nn.Linear(in_features=63408, out_features=256),\n",
    "#     nn.ReLU(),\n",
    "#     nn.Linear(in_features=256, out_features=len(\n",
    "#         target_enc.categories_[0])),\n",
    "#     )\n",
    "# + out_channels=48, (first CNN layer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d1a014fee8dab19e381f5c0a59544b00a460ed8d6f43556103265470604014b1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}