07_blurr_hf_fastai_jigsaw_toxicity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook uses blurr, integration library between fastai and huggingface to train a multilabel classification model to identify the toxic comments from the dataset.\n",
    "\n",
    "DataSet: [Jigsaw Toxicity Prediction](https://huggingface.co/datasets/jigsaw_toxicity_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25l\r\u001b[K     |█▊                              | 10kB 25.2MB/s eta 0:00:01\r\u001b[K     |███▍                            | 20kB 32.3MB/s eta 0:00:01\r\u001b[K     |█████                           | 30kB 22.5MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 40kB 17.5MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 51kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████                      | 61kB 7.9MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 71kB 8.9MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 81kB 9.8MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 92kB 10.3MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 102kB 7.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 112kB 7.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 122kB 7.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 133kB 7.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 143kB 7.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 153kB 7.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 163kB 7.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 174kB 7.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 184kB 7.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 194kB 7.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 204kB 7.2MB/s \n",
      "\u001b[K     |████████████████████████████████| 61kB 8.8MB/s \n",
      "\u001b[K     |████████████████████████████████| 51kB 5.2MB/s \n",
      "\u001b[K     |████████████████████████████████| 61kB 8.1MB/s \n",
      "\u001b[K     |████████████████████████████████| 61kB 4.8MB/s \n",
      "\u001b[K     |████████████████████████████████| 245kB 12.8MB/s \n",
      "\u001b[K     |████████████████████████████████| 51kB 7.4MB/s \n",
      "\u001b[K     |████████████████████████████████| 2.3MB 12.9MB/s \n",
      "\u001b[K     |████████████████████████████████| 1.2MB 52.3MB/s \n",
      "\u001b[K     |████████████████████████████████| 122kB 58.7MB/s \n",
      "\u001b[K     |████████████████████████████████| 245kB 51.7MB/s \n",
      "\u001b[K     |████████████████████████████████| 3.3MB 54.0MB/s \n",
      "\u001b[K     |████████████████████████████████| 901kB 45.9MB/s \n",
      "\u001b[?25h  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "\u001b[31mERROR: transformers 4.6.1 has requirement huggingface-hub==0.0.8, but you'll have huggingface-hub 0.0.10 which is incompatible.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install fastai -q --upgrade\n",
    "!pip install nbdev -q --upgrade\n",
    "!pip install -q huggingface-hub>0.0.10\n",
    "!pip install ohmeow-blurr -q --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "%pip install datasets -q --upgrade #datasets & evaluation metrics for nlp from HF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from datasets import load_dataset\n",
    "\n",
    "from transformers import *\n",
    "from fastai.text.all import * \n",
    "\n",
    "from blurr.data.all import *\n",
    "from blurr.modeling.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import show_doc\n",
    "\n",
    "from fastai import __version__ as fastai_version\n",
    "from torch import __version__ as torch_version\n",
    "from transformers import __version__ as transformers_version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pytorch: 1.8.1+cu101 Fastai: 2.3.1 HF Transformers: 4.6.1\n"
     ]
    }
   ],
   "source": [
    "print(f'Pytorch: {torch_version} Fastai: {fastai_version} HF Transformers: {transformers_version}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU #0: Tesla T4\n"
     ]
    }
   ],
   "source": [
    "#cuda\n",
    "#torch.cuda.set_device(1)\n",
    "print(f'Using GPU #{torch.cuda.current_device()}: {torch.cuda.get_device_name()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get your data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%mkdir data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d7963f9331d4e93ad58b308b808384d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2414.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7477b48f71c44408d440af3d48e360f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1377.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Downloading and preparing dataset civil_comments/default (download: 395.73 MiB, generated: 630.60 MiB, post-processed: Unknown size, total: 1.00 GiB) to /root/.cache/huggingface/datasets/civil_comments/default/0.9.0/e7a3aacd2ab7d135fa958e7209d10b1fa03807d44c486e3c34897aa08ea8ffab...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db5967db0d9b4fd3b89649a8ca8dff84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=414947977.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c42971e1ab14e96ae4f338e9ddc3e8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27189f3fa6fa42efa27faa73cb9ad683",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8de57d390633488abbb730171dcee66d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\rDataset civil_comments downloaded and prepared to /root/.cache/huggingface/datasets/civil_comments/default/0.9.0/e7a3aacd2ab7d135fa958e7209d10b1fa03807d44c486e3c34897aa08ea8ffab. Subsequent calls will reuse this data.\n"
     ]
    }
   ],
   "source": [
    "raw_data = load_dataset('civil_comments', split='train[:1%]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18049"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(raw_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>identity_attack</th>\n",
       "      <th>insult</th>\n",
       "      <th>obscene</th>\n",
       "      <th>severe_toxicity</th>\n",
       "      <th>sexual_explicit</th>\n",
       "      <th>text</th>\n",
       "      <th>threat</th>\n",
       "      <th>toxicity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>This is so cool. It's like, 'would you want your mother to read this??' Really great idea, well done!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Thank you!! This would make my life a lot less anxiety-inducing. Keep it up, and don't let anyone get in your way!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>This is such an urgent design problem; kudos to you for taking it on. Very impressive!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Is this something I'll be able to install on my site? When will you be releasing it?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.87234</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.0</td>\n",
       "      <td>haha you guys are a bunch of losers.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.893617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   identity_attack   insult  ...  threat  toxicity\n",
       "0         0.000000  0.00000  ...     0.0  0.000000\n",
       "1         0.000000  0.00000  ...     0.0  0.000000\n",
       "2         0.000000  0.00000  ...     0.0  0.000000\n",
       "3         0.000000  0.00000  ...     0.0  0.000000\n",
       "4         0.021277  0.87234  ...     0.0  0.893617\n",
       "\n",
       "[5 rows x 8 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toxic_df = pd.DataFrame(raw_data)\n",
    "toxic_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e318b3d73f77>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtoxic_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'toxic_df' is not defined"
     ]
    }
   ],
   "source": [
    "toxic_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['insult',\n",
       " 'obscene',\n",
       " 'severe_toxicity',\n",
       " 'sexual_explicit',\n",
       " 'threat',\n",
       " 'toxicity']"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_columns = list(toxic_df.columns[1:]);\n",
    "label_columns.remove('text');\n",
    "label_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>identity_attack</th>\n",
       "      <th>insult</th>\n",
       "      <th>obscene</th>\n",
       "      <th>severe_toxicity</th>\n",
       "      <th>sexual_explicit</th>\n",
       "      <th>text</th>\n",
       "      <th>threat</th>\n",
       "      <th>toxicity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>This is so cool. It's like, 'would you want your mother to read this??' Really great idea, well done!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Thank you!! This would make my life a lot less anxiety-inducing. Keep it up, and don't let anyone get in your way!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>This is such an urgent design problem; kudos to you for taking it on. Very impressive!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Is this something I'll be able to install on my site? When will you be releasing it?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.87234</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.0</td>\n",
       "      <td>haha you guys are a bunch of losers.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.893617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   identity_attack   insult  ...  threat  toxicity\n",
       "0         0.000000  0.00000  ...     0.0  0.000000\n",
       "1         0.000000  0.00000  ...     0.0  0.000000\n",
       "2         0.000000  0.00000  ...     0.0  0.000000\n",
       "3         0.000000  0.00000  ...     0.0  0.000000\n",
       "4         0.021277  0.87234  ...     0.0  0.893617\n",
       "\n",
       "[5 rows x 8 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toxic_df.round({col: 0 for col in label_columns}) #round the toxic dataframe to a var number of decimal places\n",
    "toxic_df = toxic_df.convert_dtypes()\n",
    "\n",
    "toxic_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get your huggingface objects"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_cls = AutoModelForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the pretrained model that we want to use\n",
    "\n",
    "#The DistilRoBERTa model distilled from the RoBERTa model roberta-base checkpoint.\n",
    "#Compressed model from roberta  having 35% less params & runs twice faster & preserves 95%\n",
    "pretrained_model_name = 'distilroberta-base' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pretrained models from HuggingFace can be found here\n",
    "https://huggingface.co/transformers/pretrained_models.html\n",
    "\n",
    "Community uploaded models\n",
    "https://huggingface.co/models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- `t5`: :class:`~transformers.T5Config` (T5 model)\n",
    "- `distilbert`: :class:`~transformers.DistilBertConfig` (DistilBERT model)\n",
    "- `albert`: :class:`~transformers.AlbertConfig` (ALBERT model)\n",
    "- `camembert`: :class:`~transformers.CamembertConfig` (CamemBERT model)\n",
    "- `xlm-roberta`: :class:`~transformers.XLMRobertaConfig` (XLM-RoBERTa model)\n",
    "- `longformer`: :class:`~transformers.LongformerConfig` (Longformer model)\n",
    "- `roberta`: :class:`~transformers.RobertaConfig` (RoBERTa model)\n",
    "- `reformer`: :class:`~transformers.ReformerConfig` (Reformer model)\n",
    "- `bert`: :class:`~transformers.BertConfig` (Bert model)\n",
    "- `openai-gpt`: :class:`~transformers.OpenAIGPTConfig` (OpenAI GPT model)\n",
    "- `gpt2`: :class:`~transformers.GPT2Config` (OpenAI GPT-2 model)\n",
    "- `transfo-xl`: :class:`~transformers.TransfoXLConfig` (Transformer-XL model)\n",
    "- `xlnet`: :class:`~transformers.XLNetConfig` (XLNet model)\n",
    "- `xlm`: :class:`~transformers.XLMConfig` (XLM model)\n",
    "- `ctrl` : :class:`~transformers.CTRLConfig` (CTRL model)\n",
    "- `flaubert` : :class:`~transformers.FlaubertConfig` (Flaubert model)\n",
    "- `electra` : :class:`~transformers.ElectraConfig` (ELECTRA model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1dc0cafbe3ff4450a4b3354cc8e493af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=480.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "config = AutoConfig.from_pretrained(pretrained_model_name)  # Download configuration from S3 and cache."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobertaConfig {\n",
       "  \"architectures\": [\n",
       "    \"RobertaForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"bos_token_id\": 0,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-05,\n",
       "  \"max_position_embeddings\": 514,\n",
       "  \"model_type\": \"roberta\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 6,\n",
       "  \"pad_token_id\": 1,\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.6.1\",\n",
       "  \"type_vocab_size\": 1,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 50265\n",
       "}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-babecf6ae69b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel_columns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'label_columns' is not defined"
     ]
    }
   ],
   "source": [
    "config.num_labels = len(label_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4 id=\"BlurrUtil.get_hf_objects\" class=\"doc_header\"><code>BlurrUtil.get_hf_objects</code><a href=\"https://github.com/ohmeow/blurr/tree/master/blurr/utils.py#L140\" class=\"source_link\" style=\"float:right\">[source]</a></h4><blockquote><p><code>BlurrUtil.get_hf_objects</code>(<strong><code>pretrained_model_name_or_path</code></strong>, <strong><code>task</code></strong>=<em><code>None</code></em>, <strong><code>config</code></strong>=<em><code>None</code></em>, <strong><code>tokenizer_cls</code></strong>=<em><code>None</code></em>, <strong><code>model_cls</code></strong>=<em><code>None</code></em>, <strong><code>config_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>tokenizer_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>model_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>cache_dir</code></strong>=<em><code>None</code></em>)</p>\n",
       "</blockquote>\n",
       "<p>Returns the architecture (str), config (obj), tokenizer (obj), and model (obj) given at minimum a\n",
       "<code>pre-trained model name or path</code>. Specify a <code>task</code> to ensure the right \"AutoModelFor<task>\" is used to\n",
       "create the model.</p>\n",
       "<p>Optionally, you can pass a config (obj), tokenizer (class), and/or model (class) (along with any\n",
       "related kwargs for each) to get as specific as you want w/r/t what huggingface objects are returned.</p>\n",
       "<p><a href=\"https://ohmeow.github.io/blurr/utils#BlurrUtil.get_hf_objects\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(BLURR.get_hf_objects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "AutoModelForSequenceClassification.from_pretrained(pretrained_model_name, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "BLURR.get_hf_objects()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Returns the architecture (str), config (obj), tokenizer (obj), and model (obj) given at minimum a pre-trained model name or path. Specify a task to ensure the right \"AutoModelFor\" is used to create the model.\n",
    "\n",
    "hf_arch, hf_config, hf_tokenizer, hf_model = BLURR.get_hf_objects(\n",
    "    pretrained_model_name, \n",
    "    model_cls=model_cls,\n",
    "    config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roberta\n",
      "<class 'transformers.models.roberta.configuration_roberta.RobertaConfig'>\n",
      "<class 'transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast'>\n",
      "<class 'transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None, None)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(hf_arch), print(type(hf_config)), print(type(hf_tokenizer)), print(type(hf_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build your DataBlock & DataLoaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 id=\"HF_TextBlock\" class=\"doc_header\"><code>class</code> <code>HF_TextBlock</code><a href=\"https://github.com/ohmeow/blurr/tree/master/blurr/data/core.py#L75\" class=\"source_link\" style=\"float:right\">[source]</a></h2><blockquote><p><code>HF_TextBlock</code>(<strong><code>hf_arch</code></strong>=<em><code>None</code></em>, <strong><code>hf_config</code></strong>=<em><code>None</code></em>, <strong><code>hf_tokenizer</code></strong>=<em><code>None</code></em>, <strong><code>hf_model</code></strong>=<em><code>None</code></em>, <strong><code>before_batch_tfm</code></strong>=<em><code>None</code></em>, <strong><code>after_batch_tfm</code></strong>=<em><code>None</code></em>, <strong><code>max_length</code></strong>=<em><code>None</code></em>, <strong><code>padding</code></strong>=<em><code>True</code></em>, <strong><code>truncation</code></strong>=<em><code>True</code></em>, <strong><code>is_split_into_words</code></strong>=<em><code>False</code></em>, <strong><code>input_return_type</code></strong>=<em><code>HF_BaseInput</code></em>, <strong><code>dl_type</code></strong>=<em><code>SortedDL</code></em>, <strong><code>before_batch_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>after_batch_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>tok_kwargs</code></strong>=<em><code>{}</code></em>, <strong>**<code>kwargs</code></strong>) :: <code>TransformBlock</code></p>\n",
       "</blockquote>\n",
       "<p>A basic wrapper that links defaults transforms for the data block API</p>\n",
       "<p><a href=\"https://ohmeow.github.io/blurr/data-core#HF_TextBlock\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(HF_TextBlock)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "HF_TextBlock handles setting up the HF_TokenizerTransform transforms and HF_BatchTransform transform regardless of data source (e.g., this will work with files, DataFrames, whatever).\n",
    "\n",
    "HF_TokenizerTransform was inspired by this [article](http://dev.fast.ai/tutorial.transformers). It handles both the tokenization and numericalization traditionally split apart in the fastai text DataBlock API. Addtionally, it's been updated to add a prefix space for the huggingface architectures that need it.\n",
    "\n",
    "You can pass a string or list into this Transform, the later being common in token classification tasks like named entity recognition.\n",
    "\n",
    "build_hf_input uses fastai's @typedispatched decorator to provide for complete flexibility in terms of how your numericalized tokens are assembled, and also what you return via HF_BaseInput and as your targets. You can override this implementation as needed by assigning a type to the task argument (and optionally the tokenizer argument as well).\n",
    "\n",
    "What you return here is what will be fed into your huggingface model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note how we have to configure the num_labels to the number of labels we are predicting. Given that our labels are already encoded, we use a MultiCategoryBlock with encoded=True and vocab equal to the columns with our 1's and 0's.\n",
    "# single input \n",
    "# Define the datatypes of your item X,y\n",
    "X = HF_TextBlock(hf_arch,\n",
    "                 hf_config,\n",
    "                 hf_tokenizer,\n",
    "                 hf_model)\n",
    "y = MultiCategoryBlock(encoded=True, vocab=label_columns)\n",
    "blocks = (X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<blurr.data.core.HF_TextBlock at 0x7fac41ef36d0>,\n",
       " <fastai.data.block.TransformBlock at 0x7fac66a73190>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 id=\"DataBlock\" class=\"doc_header\"><code>class</code> <code>DataBlock</code><a href=\"https://github.com/fastai/fastai/tree/master/fastai/data/block.py#L58\" class=\"source_link\" style=\"float:right\">[source]</a></h2><blockquote><p><code>DataBlock</code>(<strong><code>blocks</code></strong>=<em><code>None</code></em>, <strong><code>dl_type</code></strong>=<em><code>None</code></em>, <strong><code>getters</code></strong>=<em><code>None</code></em>, <strong><code>n_inp</code></strong>=<em><code>None</code></em>, <strong><code>item_tfms</code></strong>=<em><code>None</code></em>, <strong><code>batch_tfms</code></strong>=<em><code>None</code></em>, <strong><code>get_items</code></strong>=<em><code>None</code></em>, <strong><code>splitter</code></strong>=<em><code>None</code></em>, <strong><code>get_y</code></strong>=<em><code>None</code></em>, <strong><code>get_x</code></strong>=<em><code>None</code></em>)</p>\n",
       "</blockquote>\n",
       "<p>Generic container to quickly build <code>Datasets</code> and <code>DataLoaders</code></p>\n",
       "<p><a href=\"https://docs.fast.ai/data.block#DataBlock\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(DataBlock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['insult',\n",
       " 'obscene',\n",
       " 'severe_toxicity',\n",
       " 'sexual_explicit',\n",
       " 'threat',\n",
       " 'toxicity']"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "identity_attack    float64\n",
       "insult             float64\n",
       "obscene            float64\n",
       "severe_toxicity    float64\n",
       "sexual_explicit    float64\n",
       "text                string\n",
       "threat             float64\n",
       "toxicity           float64\n",
       "dtype: object"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toxic_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dblock = DataBlock(blocks=blocks, # Define the datatypes of your item\n",
    "                   get_x=ColReader('text'), #where to get the X from\n",
    "                   get_y=ColReader(label_columns), #where to get the y from\n",
    "                   splitter=RandomSplitter(valid_pct=0.2, seed=42)) #how to split the data for training & validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting-up type transforms pipelines\n",
      "Collecting items from        identity_attack    insult  ...  threat  toxicity\n",
      "0             0.000000  0.000000  ...     0.0  0.000000\n",
      "1             0.000000  0.000000  ...     0.0  0.000000\n",
      "2             0.000000  0.000000  ...     0.0  0.000000\n",
      "3             0.000000  0.000000  ...     0.0  0.000000\n",
      "4             0.021277  0.872340  ...     0.0  0.893617\n",
      "...                ...       ...  ...     ...       ...\n",
      "18044         0.000000  0.000000  ...     0.2  0.200000\n",
      "18045         0.000000  0.000000  ...     0.0  0.000000\n",
      "18046         0.000000  0.000000  ...     0.0  0.000000\n",
      "18047         0.000000  0.000000  ...     0.0  0.000000\n",
      "18048         0.000000  0.166667  ...     0.0  0.166667\n",
      "\n",
      "[18049 rows x 8 columns]\n",
      "Found 18049 items\n",
      "2 datasets of sizes 14440,3609\n",
      "Setting up Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "Setting up Pipeline: ColReader -- {'cols': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'pref': '', 'suff': '', 'label_delim': None} -> EncodedMultiCategorize -- {'vocab': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'sort': False, 'add_na': False}\n",
      "\n",
      "Building one sample\n",
      "  Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "    starting from\n",
      "      identity_attack                                                                                                                                                                                                                                                                                                                                                                                      0\n",
      "insult                                                                                                                                                                                                                                                                                                                                                                                             0.3\n",
      "obscene                                                                                                                                                                                                                                                                                                                                                                                              0\n",
      "severe_toxicity                                                                                                                                                                                                                                                                                                                                                                                      0\n",
      "sexual_explicit                                                                                                                                                                                                                                                                                                                                                                                      0\n",
      "text               If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes.\n",
      "threat                                                                                                                                                                                                                                                                                                                                                                                               0\n",
      "toxicity                                                                                                                                                                                                                                                                                                                                                                                           0.4\n",
      "Name: 9524, dtype: object\n",
      "    applying ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None} gives\n",
      "      If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes.\n",
      "  Pipeline: ColReader -- {'cols': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'pref': '', 'suff': '', 'label_delim': None} -> EncodedMultiCategorize -- {'vocab': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'sort': False, 'add_na': False}\n",
      "    starting from\n",
      "      identity_attack                                                                                                                                                                                                                                                                                                                                                                                      0\n",
      "insult                                                                                                                                                                                                                                                                                                                                                                                             0.3\n",
      "obscene                                                                                                                                                                                                                                                                                                                                                                                              0\n",
      "severe_toxicity                                                                                                                                                                                                                                                                                                                                                                                      0\n",
      "sexual_explicit                                                                                                                                                                                                                                                                                                                                                                                      0\n",
      "text               If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes.\n",
      "threat                                                                                                                                                                                                                                                                                                                                                                                               0\n",
      "toxicity                                                                                                                                                                                                                                                                                                                                                                                           0.4\n",
      "Name: 9524, dtype: object\n",
      "    applying ColReader -- {'cols': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'pref': '', 'suff': '', 'label_delim': None} gives\n",
      "      [0.30000001192092896, 0.0, 0.0, 0.0, 0.0, 0.4000000059604645]\n",
      "    applying EncodedMultiCategorize -- {'vocab': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'sort': False, 'add_na': False} gives\n",
      "      TensorMultiCategory([0.3000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4000])\n",
      "\n",
      "Final sample: ('If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes.', TensorMultiCategory([0.3000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4000]))\n",
      "\n",
      "\n",
      "Collecting items from        identity_attack    insult  ...  threat  toxicity\n",
      "0             0.000000  0.000000  ...     0.0  0.000000\n",
      "1             0.000000  0.000000  ...     0.0  0.000000\n",
      "2             0.000000  0.000000  ...     0.0  0.000000\n",
      "3             0.000000  0.000000  ...     0.0  0.000000\n",
      "4             0.021277  0.872340  ...     0.0  0.893617\n",
      "...                ...       ...  ...     ...       ...\n",
      "18044         0.000000  0.000000  ...     0.2  0.200000\n",
      "18045         0.000000  0.000000  ...     0.0  0.000000\n",
      "18046         0.000000  0.000000  ...     0.0  0.000000\n",
      "18047         0.000000  0.000000  ...     0.0  0.000000\n",
      "18048         0.000000  0.166667  ...     0.0  0.166667\n",
      "\n",
      "[18049 rows x 8 columns]\n",
      "Found 18049 items\n",
      "2 datasets of sizes 14440,3609\n",
      "Setting up Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "Setting up Pipeline: ColReader -- {'cols': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'pref': '', 'suff': '', 'label_delim': None} -> EncodedMultiCategorize -- {'vocab': ['insult', 'obscene', 'severe_toxicity', 'sexual_explicit', 'threat', 'toxicity'], 'sort': False, 'add_na': False}\n",
      "Setting up after_item: Pipeline: ToTensor\n",
      "Setting up before_batch: Pipeline: HF_BeforeBatchTransform\n",
      "Setting up after_batch: Pipeline: HF_AfterBatchTransform\n",
      "Could not do one pass in your dataloader, there is something wrong in it\n",
      "\n",
      "Building one batch\n",
      "Applying item_tfms to the first sample:\n",
      "  Pipeline: ToTensor\n",
      "    starting from\n",
      "      (If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes., TensorMultiCategory([0.3000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4000]))\n",
      "    applying ToTensor gives\n",
      "      (If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes., TensorMultiCategory([0.3000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4000]))\n",
      "\n",
      "Adding the next 3 samples\n",
      "\n",
      "Applying before_batch to the list of samples\n",
      "  Pipeline: HF_BeforeBatchTransform\n",
      "    starting from\n",
      "      [(If you factor in all the related costs of collection as well as lost productivity from a stoned populace, as well as considering where the retails sales would have been spent had they not been spent on pot, what would the final tally be ? This issue \"bonanza\" at the RG calls it in another article is simply their way of hyping a ridiculous issue to further their causes., TensorMultiCategory([0.3000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4000])), (This case is barbaric and the actions are deranged.\n",
      "\n",
      "Again, my point is that cat owners have culpability too. You cannot simply turn your pet loose without any supervision and expect it's actions to be loved, wanted, or even tolerated on someone else's property.\n",
      "\n",
      "That said, the property owner is not entitled to fetch said animal from its home/property to enforce punishment., TensorMultiCategory([0., 0., 0., 0., 0., 0.])), (You can thank Sean Parnell for that! He had to give us one more kick in teeth when he knew he was gone!! He got paid well for that!!, TensorMultiCategory([0.1000, 0.0000, 0.0000, 0.0000, 0.1000, 0.2000])), (And, your values and ideology are right out of Marx., TensorMultiCategory([0., 0., 0., 0., 0., 0.]))]\n",
      "    applying HF_BeforeBatchTransform gives\n",
      "      [({'input_ids': tensor([    0,   318,    47,  3724,    11,    70,     5,  1330,  1042,     9,\n",
      "         2783,    25,   157,    25,   685,  8106,    31,    10,  1690, 11469,\n",
      "        36921,     6,    25,   157,    25,  2811,   147,     5,  5494, 11791,\n",
      "          647,    74,    33,    57,  1240,    56,    51,    45,    57,  1240,\n",
      "           15,  4728,     6,    99,    74,     5,   507, 11154,    28, 17487,\n",
      "          152,   696,    22, 12573, 19209,   113,    23,     5, 38281,  1519,\n",
      "           24,    11,   277,  1566,    16,  1622,    49,   169,     9, 15671,\n",
      "          154,    10, 10861,   696,     7,   617,    49,  4685,     4,     2,\n",
      "            1,     1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 0, 0])}, TensorMultiCategory([0.3000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4000])), ({'input_ids': tensor([    0,   152,   403,    16, 38025,     8,     5,  2163,    32,  1935,\n",
      "        17770,     4, 50118, 50118, 34790,     6,   127,   477,    16,    14,\n",
      "         4758,  2203,    33, 29410,  4484,   350,     4,   370,  1395,  1622,\n",
      "         1004,   110,  4716,  7082,   396,   143, 13702,     8,  1057,    24,\n",
      "           18,  2163,     7,    28,  2638,     6,   770,     6,    50,   190,\n",
      "        22639,    15,   951,  1493,    18,  1038,     4, 50118, 50118,  1711,\n",
      "           26,     6,     5,  1038,  1945,    16,    45,  7919,     7, 23366,\n",
      "           26,  3477,    31,    63,   184,    73, 41723,     7, 10914,  8653,\n",
      "            4,     2]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1])}, TensorMultiCategory([0., 0., 0., 0., 0., 0.])), ({'input_ids': tensor([    0,   370,    64,  3392,  4640,   221,  4422,  1641,    13,    14,\n",
      "          328,    91,    56,     7,   492,   201,    65,    55,  3151,    11,\n",
      "         9927,    77,    37,  1467,    37,    21,  1613, 12846,    91,   300,\n",
      "         1199,   157,    13,    14, 12846,     2,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}, TensorMultiCategory([0.1000, 0.0000, 0.0000, 0.0000, 0.1000, 0.2000])), ({'input_ids': tensor([    0,   178,     6,   110,  3266,     8, 14320,    32,   235,    66,\n",
      "            9, 28029,     4,     2,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}, TensorMultiCategory([0., 0., 0., 0., 0., 0.]))]\n",
      "\n",
      "Collating items in a batch\n",
      "\n",
      "Applying batch_tfms to the batch built\n",
      "  Pipeline: HF_AfterBatchTransform\n",
      "    starting from\n",
      "      ({'input_ids': tensor([[    0,   318,    47,  3724,    11,    70,     5,  1330,  1042,     9,\n",
      "          2783,    25,   157,    25,   685,  8106,    31,    10,  1690, 11469,\n",
      "         36921,     6,    25,   157,    25,  2811,   147,     5,  5494, 11791,\n",
      "           647,    74,    33,    57,  1240,    56,    51,    45,    57,  1240,\n",
      "            15,  4728,     6,    99,    74,     5,   507, 11154,    28, 17487,\n",
      "           152,   696,    22, 12573, 19209,   113,    23,     5, 38281,  1519,\n",
      "            24,    11,   277,  1566,    16,  1622,    49,   169,     9, 15671,\n",
      "           154,    10, 10861,   696,     7,   617,    49,  4685,     4,     2,\n",
      "             1,     1],\n",
      "        [    0,   152,   403,    16, 38025,     8,     5,  2163,    32,  1935,\n",
      "         17770,     4, 50118, 50118, 34790,     6,   127,   477,    16,    14,\n",
      "          4758,  2203,    33, 29410,  4484,   350,     4,   370,  1395,  1622,\n",
      "          1004,   110,  4716,  7082,   396,   143, 13702,     8,  1057,    24,\n",
      "            18,  2163,     7,    28,  2638,     6,   770,     6,    50,   190,\n",
      "         22639,    15,   951,  1493,    18,  1038,     4, 50118, 50118,  1711,\n",
      "            26,     6,     5,  1038,  1945,    16,    45,  7919,     7, 23366,\n",
      "            26,  3477,    31,    63,   184,    73, 41723,     7, 10914,  8653,\n",
      "             4,     2],\n",
      "        [    0,   370,    64,  3392,  4640,   221,  4422,  1641,    13,    14,\n",
      "           328,    91,    56,     7,   492,   201,    65,    55,  3151,    11,\n",
      "          9927,    77,    37,  1467,    37,    21,  1613, 12846,    91,   300,\n",
      "          1199,   157,    13,    14, 12846,     2,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1],\n",
      "        [    0,   178,     6,   110,  3266,     8, 14320,    32,   235,    66,\n",
      "             9, 28029,     4,     2,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], device='cuda:0')}, TensorMultiCategory of size 4x6)\n",
      "    applying HF_AfterBatchTransform gives\n",
      "      ({'input_ids': tensor([[    0,   318,    47,  3724,    11,    70,     5,  1330,  1042,     9,\n",
      "          2783,    25,   157,    25,   685,  8106,    31,    10,  1690, 11469,\n",
      "         36921,     6,    25,   157,    25,  2811,   147,     5,  5494, 11791,\n",
      "           647,    74,    33,    57,  1240,    56,    51,    45,    57,  1240,\n",
      "            15,  4728,     6,    99,    74,     5,   507, 11154,    28, 17487,\n",
      "           152,   696,    22, 12573, 19209,   113,    23,     5, 38281,  1519,\n",
      "            24,    11,   277,  1566,    16,  1622,    49,   169,     9, 15671,\n",
      "           154,    10, 10861,   696,     7,   617,    49,  4685,     4,     2,\n",
      "             1,     1],\n",
      "        [    0,   152,   403,    16, 38025,     8,     5,  2163,    32,  1935,\n",
      "         17770,     4, 50118, 50118, 34790,     6,   127,   477,    16,    14,\n",
      "          4758,  2203,    33, 29410,  4484,   350,     4,   370,  1395,  1622,\n",
      "          1004,   110,  4716,  7082,   396,   143, 13702,     8,  1057,    24,\n",
      "            18,  2163,     7,    28,  2638,     6,   770,     6,    50,   190,\n",
      "         22639,    15,   951,  1493,    18,  1038,     4, 50118, 50118,  1711,\n",
      "            26,     6,     5,  1038,  1945,    16,    45,  7919,     7, 23366,\n",
      "            26,  3477,    31,    63,   184,    73, 41723,     7, 10914,  8653,\n",
      "             4,     2],\n",
      "        [    0,   370,    64,  3392,  4640,   221,  4422,  1641,    13,    14,\n",
      "           328,    91,    56,     7,   492,   201,    65,    55,  3151,    11,\n",
      "          9927,    77,    37,  1467,    37,    21,  1613, 12846,    91,   300,\n",
      "          1199,   157,    13,    14, 12846,     2,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1],\n",
      "        [    0,   178,     6,   110,  3266,     8, 14320,    32,   235,    66,\n",
      "             9, 28029,     4,     2,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "             1,     1]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], device='cuda:0')}, TensorMultiCategory of size 4x6)\n"
     ]
    }
   ],
   "source": [
    "dblock.summary(source=toxic_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = dblock.dataloaders(toxic_df, bs=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, torch.Size([16, 391]), torch.Size([16, 6]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = dls.one_batch(); len(b), b[0]['input_ids'].shape, b[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         ...,\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 0, 0, 0]], device='cuda:0'),\n",
       " 'input_ids': tensor([[    0, 24268,  5257,  ...,     1,     1,     1],\n",
       "         [    0,  1234,     6,  ...,     1,     1,     1],\n",
       "         [    0,   140,    34,  ...,     1,     1,     1],\n",
       "         ...,\n",
       "         [    0,    38,  1395,  ...,     1,     1,     1],\n",
       "         [    0,    38,   524,  ...,     1,     1,     1],\n",
       "         [    0,    20,  6304,  ...,     1,     1,     1]], device='cuda:0')}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorMultiCategory([[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.1667, 0.0000, 0.0000, 0.0000, 0.0000, 0.1667],\n",
       "        [0.1667, 0.0000, 0.0000, 0.0000, 0.0000, 0.1667],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.1667, 0.0000, 0.0000, 0.0000, 0.0000, 0.1667],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]], device='cuda:0')"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With our DataLoaders built, we can now build our Learner and train. We'll use mixed precision so we can train with bigger batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HF_BaseModelWrapper(hf_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same as nn.Module, but no need for subclasses to call super().__init__. This uses a model wrapper in order to pass named arguments into the huggingface model. We do this because all arguments for a given model are not all used or needed for each architecture, and fastai does not support passing in None.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 id=\"Learner\" class=\"doc_header\"><code>class</code> <code>Learner</code><a href=\"https://github.com/fastai/fastai/tree/master/fastai/learner.py#L83\" class=\"source_link\" style=\"float:right\">[source]</a></h2><blockquote><p><code>Learner</code>(<strong><code>dls</code></strong>, <strong><code>model</code></strong>, <strong><code>loss_func</code></strong>=<em><code>None</code></em>, <strong><code>opt_func</code></strong>=<em><code>Adam</code></em>, <strong><code>lr</code></strong>=<em><code>0.001</code></em>, <strong><code>splitter</code></strong>=<em><code>trainable_params</code></em>, <strong><code>cbs</code></strong>=<em><code>None</code></em>, <strong><code>metrics</code></strong>=<em><code>None</code></em>, <strong><code>path</code></strong>=<em><code>None</code></em>, <strong><code>model_dir</code></strong>=<em><code>'models'</code></em>, <strong><code>wd</code></strong>=<em><code>None</code></em>, <strong><code>wd_bn_bias</code></strong>=<em><code>False</code></em>, <strong><code>train_bn</code></strong>=<em><code>True</code></em>, <strong><code>moms</code></strong>=<em><code>(0.95, 0.85, 0.95)</code></em>) :: <code>GetAttr</code></p>\n",
       "</blockquote>\n",
       "<p>Group together a <code>model</code>, some <code>dls</code> and a <code>loss_func</code> to handle training</p>\n",
       "<p><a href=\"https://docs.fast.ai/learner#Learner\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(Learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls,\n",
    "                model,\n",
    "                opt_func=partial(Adam),\n",
    "                loss_func=BCEWithLogitsLossFlat(), # multilabel classification loss_func, same as BCEWithLogitsLoss  but flattens the input & target\n",
    "                metrics=partial(accuracy_multi, thresh=0.2),\n",
    "                cbs=[HF_BaseModelCallback],\n",
    "                splitter=hf_splitter).to_fp16()\n",
    "\n",
    "learn.loss_func.thresh = 0.2\n",
    "learn.create_opt() # -> will create your layer groups based on your \"splitter\" function\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "HF_BaseModelWrapper (Input shape: 16 x 391)\n",
       "============================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "============================================================================\n",
       "                     16 x 391 x 768      \n",
       "Embedding                                 38603520   False     \n",
       "Embedding                                 394752     False     \n",
       "Embedding                                 768        False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 3072     \n",
       "Linear                                    2362368    False     \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 768      \n",
       "Linear                                    2360064    False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 3072     \n",
       "Linear                                    2362368    False     \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 768      \n",
       "Linear                                    2360064    False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 3072     \n",
       "Linear                                    2362368    False     \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 768      \n",
       "Linear                                    2360064    False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 3072     \n",
       "Linear                                    2362368    False     \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 768      \n",
       "Linear                                    2360064    False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 3072     \n",
       "Linear                                    2362368    False     \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 768      \n",
       "Linear                                    2360064    False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Linear                                    590592     False     \n",
       "Dropout                                                        \n",
       "Linear                                    590592     False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 3072     \n",
       "Linear                                    2362368    False     \n",
       "____________________________________________________________________________\n",
       "                     16 x 391 x 768      \n",
       "Linear                                    2360064    False     \n",
       "LayerNorm                                 1536       True      \n",
       "Dropout                                                        \n",
       "Linear                                    590592     True      \n",
       "Dropout                                                        \n",
       "____________________________________________________________________________\n",
       "                     16 x 6              \n",
       "Linear                                    4614       True      \n",
       "____________________________________________________________________________\n",
       "\n",
       "Total params: 82,123,014\n",
       "Total trainable params: 615,174\n",
       "Total non-trainable params: 81,507,840\n",
       "\n",
       "Optimizer used: functools.partial(<function Adam at 0x7facd0fe83b0>)\n",
       "Loss function: FlattenedLoss of BCEWithLogitsLoss()\n",
       "\n",
       "Model frozen up to parameter group #2\n",
       "\n",
       "Callbacks:\n",
       "  - TrainEvalCallback\n",
       "  - Recorder\n",
       "  - ProgressCallback\n",
       "  - HF_BaseModelCallback\n",
       "  - MixedPrecision"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.blurr_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4 id=\"Learner.create_opt\" class=\"doc_header\"><code>Learner.create_opt</code><a href=\"https://github.com/fastai/fastai/tree/master/fastai/learner.py#L148\" class=\"source_link\" style=\"float:right\">[source]</a></h4><blockquote><p><code>Learner.create_opt</code>()</p>\n",
       "</blockquote>\n",
       "<p>Create an optimizer with default hyper-parameters</p>\n",
       "<p><a href=\"https://docs.fast.ai/learner#Learner.create_opt\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(Learner.create_opt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This method is called internally to create the optimizer, the hyper-parameters are then adjusted by what you pass to Learner.fit or your particular schedulers (see callback.schedule)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([16, 6]),\n",
       " SequenceClassifierOutput([('logits',\n",
       "                            tensor([[-3.5993, -5.3291, -7.0173, -5.7250, -5.5425, -3.3405],\n",
       "                                    [-4.1521, -5.5553, -7.3682, -6.2844, -5.9261, -3.7864],\n",
       "                                    [-1.8160, -3.6050, -5.5187, -5.4348, -4.9802, -1.5784],\n",
       "                                    [-2.9193, -4.6364, -4.9824, -3.6422, -2.1356, -1.8673],\n",
       "                                    [-2.1799, -4.2955, -6.0301, -5.6385, -5.4961, -1.9690],\n",
       "                                    [-1.5147, -3.8444, -5.5366, -5.6994, -4.1171, -1.3472],\n",
       "                                    [-3.8328, -5.9142, -7.4598, -6.3970, -6.3127, -3.6225],\n",
       "                                    [-4.5958, -5.7680, -7.6138, -6.4669, -6.2175, -4.1220],\n",
       "                                    [-4.1937, -5.7697, -7.4271, -6.3916, -5.6011, -3.6791],\n",
       "                                    [-4.1763, -5.9607, -7.3833, -6.1609, -6.1598, -3.8454],\n",
       "                                    [-3.9674, -5.4476, -7.1179, -5.9332, -5.5704, -3.5202],\n",
       "                                    [-2.0081, -4.2572, -5.7168, -4.4927, -5.2646, -1.8351],\n",
       "                                    [-4.5274, -5.7065, -7.3542, -6.1187, -5.9234, -3.9988],\n",
       "                                    [-3.2808, -4.9199, -6.6695, -6.1206, -5.6042, -3.0266],\n",
       "                                    [-4.8323, -6.3275, -7.8633, -6.7194, -6.2105, -4.3376],\n",
       "                                    [-0.0707, -0.8957, -3.2828, -2.7965, -4.0081,  0.2760]],\n",
       "                                   device='cuda:0', grad_fn=<AddmmBackward>))]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check the learner work for a sample\n",
    "preds = model(b[0])\n",
    "preds.logits.shape, preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "print(len(learn.opt.param_groups))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=5.248074739938602e-06, lr_steep=7.585775847473997e-07)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAELCAYAAADDZxFQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxdVbn/8c+TpEmbpEk6pPM8UUrpAKFMUgoyKjJ4QUFRJsHhx1Wvw734Q1FxvHL1Olz8CXoBQaEiilYolBkUaE1LJzpBm84DSZp5Pid5fn+ck3KaJmlCs3POSb7v1+u8es7ae5/9rJ6cPFlr7b2WuTsiIiJtpcQ7ABERSUxKECIi0i4lCBERaZcShIiItEsJQkRE2qUEISIi7Qo0QZjZRWa2xcy2mtlt7Wz/kpltNLN1Zva8mU2Mls8zs9fNbEN020eDjFNERI5kQd0HYWapwFvA+cAeoBC4xt03xuxzDrDC3evM7LPAInf/qJnNANzd3zazMcAq4Hh3r+jofMOHD/dJkyYFUhcRkb5q1apVpe6e3962tADPuwDY6u5FAGa2GLgMOJQg3P3FmP2XA9dGy9+K2WefmRUD+UCHCWLSpEmsXLmyRysgItLXmdnOjrYF2cU0Ftgd83pPtKwjNwFPtS00swVAOrCtR6MTEZFOBdmC6DIzuxYoAM5uUz4aeAi4zt1b2jnuFuAWgAkTJvRCpCIi/UeQLYi9wPiY1+OiZYcxs/OA24FL3b0xpjwHeBK43d2Xt3cCd7/X3QvcvSA/v90uNBEReY+CTBCFwHQzm2xm6cDVwJLYHcxsPnAPkeRQHFOeDjwOPOjujwUYo4iIdCCwBOHuYeBWYBmwCXjU3TeY2Z1mdml0t7uAbOCPZrbGzFoTyEeAhcD10fI1ZjYvqFhFRORIgV3m2tsKCgpcVzGJiHSPma1y94L2tulOahGRJLZyRxmFO8oCeW8lCBGRJPbT597m+0s3BfLeShAiIkmsor6JIZnpgby3EoSISBIrrw2RN2hAIO+tBCEiksQq60PkqQUhIiKxmsIt1DSGyctUC0JERGJU1ocAGKIEISIisSrqmgDIVReTiIjEqoi2IDRILSIihymvjbQgdJmriIgc5lALQmMQIiISq3UMQglCREQOU1EXIi3FyM4IZu03JQgRkSRVXhciL3MAZhbI+ytBiIgkqcr6JnIDuoIJlCBERJJWRV0osCuYQAlCRCRptXYxBUUJQkQkSVXWNQU2UR8oQYiIJK3yuuCm+gYlCBGRpNQQaqY+1MyQLLUgREQkRutMrrqKSUREDlNR1zrVd5K2IMzsIjPbYmZbzey2drZ/ycw2mtk6M3vezCbGbLvOzN6OPq4LMk4RkWRTHvA0GxBggjCzVOBu4GJgFnCNmc1qs9tqoMDd5wCPAT+KHjsU+CZwKrAA+KaZDQkqVhGRZNPagkjKBEHkF/tWdy9y9yZgMXBZ7A7u/qK710VfLgfGRZ9fCDzr7mXuXg48C1wUYKwiIknl3Yn6krOLaSywO+b1nmhZR24CnurOsWZ2i5mtNLOVJSUlxxiuiEjyqAh4uVFIkEFqM7sWKADu6s5x7n6vuxe4e0F+fn4wwYmIJKDyuibSU1MYNCA1sHMEmSD2AuNjXo+Llh3GzM4DbgcudffG7hwrItJfVQY8kysEmyAKgelmNtnM0oGrgSWxO5jZfOAeIsmhOGbTMuACMxsSHZy+IFomIiJEWhBBDlADBLPKBODuYTO7lcgv9lTgPnffYGZ3AivdfQmRLqVs4I/RLLjL3S919zIz+w6RJANwp7uXBRWriEiyqagLBTpADQEmCAB3XwosbVN2R8zz8zo59j7gvuCiExFJXhV1ISYOywz0HAkxSC0iIt1TUR98F5MShIhIknF3ygNeLAiUIEREkk5DqIWmcAu5akGIiEis1nmY1IIQEZHDHJqHKcCpvkEJQkQk6fTGPEygBCEiknRa52HSVUwiInIYjUGIiEi7emMtCFCCEBFJOhV1TQwckMLAAGdyBSUIEZGkU1EXIm9QsN1LoAQhIpJ0yqNTfQdNCUJEJMlU9sI8TKAEISKSdHpjHiZQghARSToV6mISEZG23J2KuqbA76IGJQgRkaRS29RMuMUDn4cJlCBERJJKeW3v3EUNShAiIkmlMjoPU9BrQYAShIhIUumteZhACUJEJKn01jxMEHCCMLOLzGyLmW01s9va2b7QzN4ws7CZXdlm24/MbIOZbTKzn5uZBRmriEgyeHctiCROEGaWCtwNXAzMAq4xs1ltdtsFXA883ObYM4AzgTnAbOAU4OygYhURSRbvriYXfBdTWoDvvQDY6u5FAGa2GLgM2Ni6g7vviG5raXOsAwOBdMCAAcA7AcYqIpIUyutCZKWnkp4W/AhBkGcYC+yOeb0nWnZU7v468CKwP/pY5u6bejxCEZEkU1HfOzfJQYIOUpvZNOB4YByRpHKumZ3Vzn63mNlKM1tZUlLS22GKiPS6iroQub1wkxwEmyD2AuNjXo+LlnXFFcByd69x9xrgKeD0tju5+73uXuDuBfn5+cccsIhIonunqoGhWcnfgigEppvZZDNLB64GlnTx2F3A2WaWZmYDiAxQq4tJRPq14qoGNuyr4tTJQ3vlfIElCHcPA7cCy4j8cn/U3TeY2Z1mdimAmZ1iZnuAq4B7zGxD9PDHgG3AemAtsNbd/xZUrCIiyeCZjZFrdS6cPapXzhfkVUy4+1JgaZuyO2KeFxLpemp7XDPw6SBjExFJNs9sfIdJwzKZPiK7V86XkIPUIiJyuKqGEK9vK+XCE0bRW/cNK0GIiCSBFzcXE2p2LjhhZK+dUwlCRCQJPLPhHYZnZzB//JBeO6cShIhIgmsINfPSlmLOnzWSlJTem5ZOCUJEJMG9vu0gtU3Nvdq9BEoQIiIJb9mGA2RnpHHG1GG9el4lCBGRBNbc4jy36R0WHZdPRlpqr55bCUJEJIG9sauc0pomLjihd26Oi6UEISKSwJ7ZcIABqcY5x/X+fHNKECIiCewfWw9y6uRhDB7YOzO4xlKCEBFJYGW1jYzNGxSXcytBiIgksKr6MDmDAp02r0NKECIiCaop3EJ9qJmcOHQvgRKEiEjCqm4IAZDTSyvItaUEISKSoKobwgAMHqguJhERiVHV2oJQF5OIiMSqqo+0INTFJCIihznUgtBVTCIiEqt1kDoeN8mBEoSISMI61MWkQWoREYlV1RAixSArvQ8mCDO7yMy2mNlWM7utne0LzewNMwub2ZVttk0ws2fMbJOZbTSzSUHGKiKSaKrqQwweOKBXV5GLFViCMLNU4G7gYmAWcI2ZzWqz2y7geuDhdt7iQeAudz8eWAAUBxWriEgiqmoIx+0eCIAgz7wA2OruRQBmthi4DNjYuoO774hua4k9MJpI0tz92eh+NQHGKSKSkKobQnG7BwKC7WIaC+yOeb0nWtYVM4AKM/uzma02s7uiLRIRkX4jnhP1QeIOUqcBZwFfAU4BphDpijqMmd1iZivNbGVJSUnvRigiErCqPtyC2AuMj3k9LlrWFXuANe5e5O5h4C/ASW13cvd73b3A3Qvy83t/tSURkSC1DlLHS5AJohCYbmaTzSwduBpY0o1j88ys9bf+ucSMXYiI9AfVDX20iyn6l/+twDJgE/Cou28wszvN7FIAMzvFzPYAVwH3mNmG6LHNRLqXnjez9YABvw4qVhGRRNPc4lQ3huPaxRRoanL3pcDSNmV3xDwvJNL11N6xzwJzgoxPRCRR1TTEd6I+SNxBahGRfq3q0DxMCd7FZGZZZpYSfT7DzC41s/ilNRGRPi7ea0FA11sQrwADzWws8AzwCeCBoIISEenv3l0LIsFbEIC5ex3wYeCX7n4VcEJwYYmI9G/J1IIwMzsd+DjwZLRMdzaLiASkqj55EsQXga8Bj0cvVZ0CvBhcWCIi/Vt1Q/y7mLp0Znd/GXgZIDpYXerunw8yMBGR/qy1iyk7I8HHIMzsYTPLMbMs4E1go5l9NdjQRET6r6r6MNkZaaSlxu9uhK6eeZa7VwGXA08Bk4lcySQiIgGoagjF9R4I6HqCGBC97+FyYIm7hwAPLiwRkf4t3mtBQNcTxD3ADiALeMXMJgJVQQUlItLfxXstCOhignD3n7v7WHf/gEfsBM4JODYRkX4r3mtBQNcHqXPN7Ceti/OY2Y+JtCZERCQAyTQGcR9QDXwk+qgC7g8qKBGR/i6yFkR8WxBdTU9T3f1fYl5/28zWBBGQiEh/5+5U1SdJFxNQb2bva31hZmcC9cGEJCLSv9U2NdPi8b2LGrregvgM8KCZ5UZflwPXBROSiEj/1joPUzzXo4auT7WxFphrZjnR11Vm9kVgXZDBiYj0R4fmYUqSLiYgkhiid1QDfCmAeERE+r1DU30nw30QHbAei0JERA5JhKm+4dgShKbaEBEJQCKsRw1HGYMws2raTwQGDAokIhGRfu7dtSASuAXh7oPdPaedx2B3P2pqM7OLzGyLmW01s9va2b7QzN4ws7CZXdnO9hwz22Nm/9O9aomIJK93r2JK3jGITplZKnA3cDEwC7jGzGa12W0XcD3wcAdv8x3glaBiFBFJRFUNYQYOSCEjLb4rOwe5EsUCYKu7F7l7E7AYuCx2B3ff4e7rgJa2B5vZycBI4JkAYxQRSThV9aG43wMBwSaIscDumNd7omVHFV3W9MfAV46y3y2tEwiWlJS850BFRBJJdUOYnDh3L0GwCeJYfA5Y6u57OtvJ3e919wJ3L8jPz++l0EREglXVEIr7ADV0faqN92IvMD7m9bhoWVecDpxlZp8DsoF0M6tx9yMGukVE+pqq+hB5menxDiPQBFEITDezyUQSw9XAx7pyoLt/vPW5mV0PFCg5iEh/UdUQZvzQzHiHEVwXk7uHgVuBZcAm4FF332Bmd5rZpQBmdoqZ7QGuAu4xsw1BxSMikiyq+0EXE+6+FFjapuyOmOeFRLqeOnuPB4AHAghPRCThRNaCCMd9mg1I3EFqEZF+qTHcQlNzS9wn6gMlCBGRhJIoa0GAEoSISEI5NNW37oMQEZFYVQkyUR8oQYiIJJREWQsClCBERBLKoRaEuphERCTWoRaEuphERCTWocWC1MUkIiKxqhpCDEg1Bg6I/6/n+EcgIiKHVNWHyBk4ADOLdyhKECIiiaSqIRz3pUZbKUGIiCSQRJmoD5QgREQSSmsXUyJQghARSSBVDeGEmKgPlCBERBJGqLmFd6oaEmI1OVCCEBFJGK9uLaW6IczZM/LjHQqgBCEikjCWrN3H4IFpLDpOCUJERKIaQs0se/MAF88eRUZaarzDAZQgREQSwgubi6ltaubSuWPjHcohShAiIglgyZp9DM/O4PSpw+IdyiFKECIicVbVEOKFLcVcMmc0qSnxn2KjVaAJwswuMrMtZrbVzG5rZ/tCM3vDzMJmdmVM+Twze93MNpjZOjP7aJBxiojE0zMb3qEp3MKH5o6JdyiHCSxBmFkqcDdwMTALuMbMZrXZbRdwPfBwm/I64JPufgJwEfBTM8sLKta26puacffeOp2I9HNL1u5j3JBBnDSh137NdUmQLYgFwFZ3L3L3JmAxcFnsDu6+w93XAS1tyt9y97ejz/cBxUCvXPcVam7hjB8+z4Ov7+yN04lIP1da08irW0v50NwxCTGDa6wgE8RYYHfM6z3Rsm4xswVAOrCth+LqVHltE+V1IZ5ct783Tici/dzS9ftpbnEum5dY3UuQ4IPUZjYaeAi4wd1b2tl+i5mtNLOVJSUlPXLOg7VNAKzaVU5ldOk/EZGgLFmzjxkjs5k5KifeoRwhyASxFxgf83pctKxLzCwHeBK43d2Xt7ePu9/r7gXuXpCf3zM9UOXRBNHc4vzj7dIeeU8RkfZUN4RYtauci04YFe9Q2hVkgigEppvZZDNLB64GlnTlwOj+jwMPuvtjAcZ4hNYWhBm8uKW4N08tIv3M2t2VuEPBpKHxDqVdgSUIdw8DtwLLgE3Ao+6+wczuNLNLAczsFDPbA1wF3GNmG6KHfwRYCFxvZmuij3lBxRqrvC6SIE6bPIyX3yqhpUVXM4lIMN7YVQ7A3PGJdfVSq0AnHXf3pcDSNmV3xDwvJNL11Pa43wG/CzK2jhysiSSIK+aP5d//tI6N+6uYPTY3HqGISB+3elc500dkk5sgK8i1ldCD1PFQXtdEXuYAzpk5AoCX1M0kIgFwd1bvruCkCUPiHUqHlCDaOFjbxNDMdPIHZzBnXC4vbumZq6NERGJtL62loi7E/AS7OS6WEkQb5bVNDM2KrOa0aEY+q3eVUxEdlxAR6Slv7KoA4KSJakEkjbLaJoa0JoiZI2hxeEWXu4pID3tjVzmDM9KYlp8d71A6pATRRlltE8OiCWLuuDyGZA7QOISI9LjVuyqYNyGPlASavbUtJYgY7k553bstiNQU46zp+by8RZe7ikjPqWkMs+VAFfMTeIAalCAOU90YJtTsh1oQAOfMzOdgbRPr91bGMTIR6UvW7amgxUm42VvbUoKI0TrNxpDMdxPEwun5mMHTGw50eNzyooNsLa4JPD4R6RtWRweo549XCyJptE6zMTT73QQxLDuDi2eP4sHXdlBa03jEMbvL6vjkff/ke09u7LU4RSS5rd5VztT8LHIzE/MGuVZKEDFaWxBDY1oQAF++4Dgawi38zwtbjzjmh09tpincwprdFVpkSESOyt15Y1di3yDXSgkixqEWRNbhCWJqfjYfKRjH71fsZNfBukPly4sO8uT6/UzNz6K8LsSusjpERDqz82AdZbVNCT9ADUoQhynvIEEAfOH9M0gx4yfPbgEi04Hf+beNjMkdyI+unAvAmt0VvResiCSl1bsjE/SdNDGxB6hBCeIwZbVNpKelkJmeesS2UbkDueHMyfx17T427qvijyt3s3F/FV/7wPHMHZfLoAGpShAiclRv7KwgOyON6SMGxzuUowp0Ntdk03qTXEfrwn727Kk8vGIn33liI28XV3PKpCFcMmc0ZsaJY3NZqwQhIkexenc5c8fnkprAN8i1UgsiRllt02GXuLaVmzmAz50zjdeLDnKwtok7LjnhUDKZOz6XN/dV0RQ+YmVUERGaW5yfP/82G/ZVsWDSsHiH0yVKEDHK6poYlt1xggC4/oxJTM3P4rrTJ3HiuHfXiZg3fghN4RY2H6gKOkwRSTLFVQ184n9X8JNn3+LyeWO5eeHkeIfUJepiilFW28T4IZmd7jNwQCrLvriQtNTDc+vc8ZFksXZ3BXPGJf7gk4gEq6XFKa1tZPWuCm5/fD21jc3cdeUcrjx5XIfd2IlGCSJGWcxU351pmxwAxuYNYnh2Bqt3V/CJ04OITkQS3fbSWu7465tsK66huLqRcHQOtxkjs3nk5pOYPjLxB6ZjKUFENYVbqG4IdylBtMfMmDdeA9Ui/dU/t5dxy0MrMeCcmSMYlTOQUbkDGZ07iLOmD2fggCOvjkx0ShBRrYsCvdcEATBvfB7PbSqmsj6UsGvMikjP+/Mbe/iPP61j/NBM7r/+FCYOy4p3SD1CCSKqo7uou2Pu+MjYw/o9lbxv+vAeiUtEEss7VQ2U1TZR1xSmtrGZV7eVcs/LRZw+ZRi/uvbkhJ9fqTsCTRBmdhHwMyAV+I27/7DN9oXAT4E5wNXu/ljMtuuAr0dfftfdfxtkrJ3dRd1VrYPTa3aXK0GI9EH/3F7GR+55/Yjyq04ex/euOJH0tL51YWhgCcLMUoG7gfOBPUChmS1x99hpT3cB1wNfaXPsUOCbQAHgwKroseVBxdsTLYjcQQOYkp/Fmt2Hrx1RuKMMAwomDT2WEEUkzn772g5yBw3gBx8+kayMNLLSU8nLTGdqflbSXJnUHUG2IBYAW929CMDMFgOXAYcShLvviG5re3fZhcCz7l4W3f4scBHwSFDBlvfAGARExiFeeasUd8fMePrN/fyfh1eTO2gAr912blIOVIkIFFc3sGzDAa4/YxIfOHF0vMPpFUG2h8YCu2Ne74mWBX3se3KwJpIg8o5xcHne+DxKaxrZVxn5Ybr14dWMzRtEWW0TT6zb3xOhikgcPFq4m3CL87FTJ8Q7lF6T1B1mZnaLma00s5UlJSXH9F7ldU3kZQ5o9x6H7pgXHaj+2XNvcevDbzB7bC5PfP59TB+RzW9f29Fn1owINbf0mbqIHE1zi/PIP3dz5rRhTMnPjnc4vSbIBLEXGB/zely0rMeOdfd73b3A3Qvy8/Pfc6AQGYNou1DQezFzVA7paSk8unIPs0bn8OBNC8gZOIBPnjGJ9XsrWd0H7pMorm7g9B+8wILvP8+XHl3DX9fsbXe1PZG+4qUtxeytqOfjp06Mdyi9KsgEUQhMN7PJZpYOXA0s6eKxy4ALzGyImQ0BLoiWBaa8tokhxzj+AJCelsLZM/KZPyGPB288lZyBkS6rD88fy+CMNH772o5jPkc8uTtff/xNqhpCLJg0lBc3F/OFxWso+O5z/OSZLfEOTyQQv1+xi/zBGZw/a2S8Q+lVgQ1Su3vYzG4l8os9FbjP3TeY2Z3ASndfYmanAI8DQ4APmdm33f0Edy8zs+8QSTIAd7YOWAelrLaJ8UM7n4epq3517clHTOWblZHGlQXj+N3yndz+weMZMXhgj5yrty1Zu49nNr7D1y6eyafPnkpzi7NhXyX3vFLEz1/YyrwJeZw7s399iaRv211Wx4tbirn1nGkMOMYu6GQTaG3dfam7z3D3qe7+vWjZHe6+JPq80N3HuXuWuw9z9xNijr3P3adFH/cHGSdE52HqgS4moMN53j95+iRCzc7DK3b1yHl6W3F1A99csoH5E/L41FlTgEhd54zL48dXzeX40Tl8+dG17K+sj3OkIj1nceEuDLh6Qf8ZnG7Vv9JhB9yd8romhh5lqu9jNXl4FouOy+f3K3Z1a92I6oYQT6zbx4ubi1m9q5ydB2tpCDUHFmdlXYhP/baQnzz7FiXVkbEFd+f2x9+krqmZu66ce0QSHDgglbs/Np/GcAtfeGQN4WatiyHJryncwh8K93DuzBGMzRsU73B6nabaAKobw4SavcdaEJ257oxJ3HB/IU+9uZ9L544h3OKEm530tJR2Wx4tLc7ND65kedHhPWz5gzNY+vmzyB+c0eMxfuOvb/LC5mKe21TMr17exhXzxjJxeCbPRruWpo1o/yqOKfnZfO+K2fzbH9bys+ff5ssXHNfjsYn0psdW7aG0prHfDU63UoIAymp65ia5rjh7ej6Th2fxhcVr+MLiNYfKp+Zn8cgtpx0xNnHfq9tZXlTGNy6ZxfwJeVTUNbGvooFv/PVNHnx9R4//Ev7rmr0sWbuPL50/gw/OGc39r27nsVV7aAi1HNa11JEr5o/jta0H+Z8Xt3Lq5GGackSSVmlNI//59GYWTB7KouOO7SrJZKUEQWQlOeidBJGSYvz4I3N5YVMxaalGWorhDr98aRs33F/IHz59OtkZkY/lrXeq+dGyLZw/ayQ3njnpsFv5X36rhIeW7+Szi6aSmd4zH+O+inq+8Zc3mT8hj88tmkpaagrfvfxEvnz+cTyxfj/nHT+iS+vofvuyE1izu4JbH3mDxz93JpOH942ZLaV/+f6Tm6hrCvP9K2b3yWk0ukJjEPRuCwLgpAlD+MqFx/HF82Zw67nT+df3T+eX157E5gPVfOahVTSFW2gKt/DFxWvIGZjGDz584hE/oJ9eOIWKuhCPFu7u4Cwde2LdPs74wfPc8dc32VsRGVBuaXG+8se1hFucn3503mE3DA7JSucTp01kdG7X+mAz09P4zXUFGHDjA4WHplLvzPbSWv7+dgkrig6yelc5G/ZVUtUQ6nbdRHrCa9tK+fPqvdyycArTRiTXIj89SS0IercF0ZFzjhvBDz98Il99bB3//thaRucNYuP+Kn79yQKGZx85zlAwaSgnTcjjN//YzrWnTezSHeAtLc6Pn93C3S9uY/LwLB755y4eXrGLfzlpHMOy03lt20F++OETe2Qu+4nDsrj3kwV8/Ncr+MzvVvHgjaceNtNlY7iZFUVlvLilmBc3F7PjYN0R7zE0K51Hbj6N40b13y+o9L7GcDNff/xNJgzN5F/PnR7vcOJKCYLIJa4Q3wQBcFXBeIqrG7lrWeSGs48WjO/0xpxbFk7lM79bxdMbDnDJnDGdvnd1Q4gvLl7D85uLufqU8Xz7shMorWni3pe38UjhbprCLZx3/Eg+esr4Tt+nO06ZNJT/vPJE/u0Pa/n6X9bzvStO5LVtB/nb2n0s23CA6oYwGWkpnD51GDe+bzIzR+UQao60nmqbwnzniY18/DfLWXzL6R0OjEv7qhpCbC+pxQytkd5N97xcRFFpLQ/ccEq/n1xTCYLIXdTpaSlkpsf/h+Fzi6ZSVR/i9aKDfONDszrd9/xZI5k8PIt7XynigyeO7rCftLi6gY/9egXbS2u587IT+MRpEzEzxuYN4tuXzeb/nDONpev3c/n8sT3e13rF/HEUldTyixe28tT6A1Q3hhmckcaFs0dx8exRnDF1OIM6+H+fOSqHq+9dzsd+vZw/fPp0jWW0UVHXxEtbSiiubqCkupGS6kb2VTRQVFp72NQnV508jm9degJZGfq6x2ppcZ7fXMwLm4tpbmmhxcEd/rZuHx+cM5pFx42Id4hxp58YIvMwDctKT4iBKDPjax84vkv7pqYYnzprMrc//ibLi8o4feqwI/Zxd772p/XsLqvjoZsWcMbUI68qGpEzkOvPnHzMsXfk386bQU1jmNKaJi6ZM5qzZ+R36S+zaSOyefjmUw8liUc/fXqP3e2e7KoaQnzkntd5650aADLSUhiRk8GonIGcOzOfycOzmTw8i/V7K/jlS9so3FHGz66ef2jVQ4isjLanvI4TxuT2yb+U3Z1N+6vZU17HuCGZTByWSVZGGvVNzTz2xh7u+8d2tpfWkjMwjcz0NFIs8v2bPSaHOy7p/I+z/sL6yoycBQUFvnLlyvd07E0PFLK/soGlXzirh6MKXkOomTN/+AJzxuVy/w0Ljtj+2Ko9fOWPa/nGJbO46X3BJYEgbdxXxTW/Xk5VQ4jMAakMSk8jMz2V3EEDGD90EOOHZjJxaBaThmcya3QOeb1wP0s8hZpbuOH+QnlxqHoAAA8qSURBVJYXHeSXHz+J06cOIzsjrcM/cFYUHeTf/rCG4upGrlkwgZLqRtbsruBAVQMQmT/s1MlDWTg9n0XH5TN9ZOdjPg2hZrYcqGbDvip2Hqzlg3NGJ0w3Vl1TmGc3vsPLb5Xw97dLD93o2Sp/cAaNoWaqGsLMHZfLp86awsWzRx3zLM7JzMxWuXtBu9uUIODyu19l8MA0Hrrp1B6Oqnf84vm3+fGzb3H7B47nU2dNPvSLYn9lPRf89yscPyqHxbecRkoXLlFNVFuLq1myZh+1Tc3UNTVT3xSmrC7E7rI69pTXEWp+9+d4bN4gZo3JYd74PK48eRwjc5Jz3qv2uDu3/Wk9f1i5mx9dOYePFHRtzKiyLsT//ct6nly3n4nDMpk3Po+54/IYkzeIwh1lvPJWCW8XR1oj588ayW0Xz2RqzLTWjeFm/vzGXn63fCebD1TT3BL5/zYDA244czJfOn9GXLuxiqsb+OT//pPNB6rJyxzA+6YNZ+GMfKaPyGZvRT07D9ax82At4RbnmgUTKJg4JCF6DeJNCeIozr7rReaOy+Pn18zv4ah6R0OomS8/upYn1+/n+jMm8Y1LZpFicN39hRRuL+PpL57VI1cmJarmFudAVQPbimvYuL+KDfuq2LivkqLSWlLN+OCc0dx45uRD3SuV9SG2l9ZSWt3IyROH9Mgsvj2lKdzCO1UN7K2oZ39lPakpKcwek8OkYVmkpBi/fGkrP3p6C7eeM42vXNj9myQbw81kpLXfnbSvop4/v7GHX71cRH2omY+fOoGbz5rC028e4Nd/L6K4upHZY3NYNGMEJ4zJYfbYXHIGDeBHT2/m9yt2MTZvEN+7YnZc+u73VtRz7W9WcKCygZ9dPY/3Hz+yS/fsiBLEUZ34rWX8y0mRgbxk1dLi/OCpTfz679u5YNZITp86jG//bSN3XnYCnzx9UrzDi4tdB+t44LUdPLpyNzWNYabmZ1FZH6K05t37MlIscsnwBbNG8r7pwymvDVFUWkNRSS0Haxq5ZeFUZo3JCTzWyvoQX1i8mpffKqG9r+TgjDRmjBrMqp3lXDp3DD+7el5gf/2W1jTys+fe5uF/7jrUUjhj6jA+t2gaZ04b1u55C3eUcduf1rGtpJZL5ozmG5fM6pGWW31TMzWNYfIyB3Q4k2pRSQ3X/mYF1Y1hHrjhFE6eqLXfu0MJohNN4RZmfP0pvnT+DD7//uS/5vmBV7fz7Sc24h75Uv/uplOTumupJ1Q3hHhs1R5e3FLCmNyBTB6exeThWeQOGsA/tpby7MZ32Hyg+rBjBg5IIS0lhbRU4+FPndbtJOHu7C6rJ3tg2lEvn95XUc8N9xdSVFrDje+bzNT8bMbkDmJ03kAaQs28ubeS9XsrWb+3iolDM/nRlXN6ZVB5a3ENf1u7j3Nmjji0UmJnGsPN/L+XtvHLl7aRnprCF8+bzvVnTOqwf78x3Mzjb+ylpjHMlPwspgzPZtyQQbxT3cgLm97huU3FvF508NDEltkZaeRlDmBYdgajcjIYnTuI/MEZ3P/qdtzhwZsWcMKY3B79P+gPlCA6UVzVwILvP893L5/Ntaf1jQm5nn7zAPe/up3/umqurvrpol0H6yjcUcaInAym5GczOmcgu8vruPre5TSEmvl9J0miucXZV1HPtpIa1u2pZPWuclbvrqCiLnIn+LghgzhxbC4njsvl5AlDmDch71A3z+YDVVx/XyE1jWHu+cTJnDkt+eeu2nmwlm8u2cBLW0qYOWowtyycwrkzRxy6eKClxXli/X7uWraZ3WWHTw0/INUOjSdNGpbJ+48fyYShmVTWhyiva6KiLkRpTSMHKhs4UNVAdUOYsXmD+O2NC3SvzHukBNGJlhanpKaRQemph1Z/E2m1o7SWq+9dTmO4mYdvPo2ZowazvbSWV7eWsryojLeLq9lxsO7QX7lmMH1ENvPHD2Hu+DxqGkOs2xNpAeyM3i2ekZbCSRMi23+/fCeZGancf/2CXunK6i3uzrIN7/DdJzeyp7ye1BSjYOIQFs7IZ9mGA6zbU8nxo3P42sUzmT02l6KSSLfettIahmWlc+7MkUzNzzpqN1ptY5j0tJR+t5BPT1KCEDkGrUmiPtRMZnoq+ysjl4eOyR3ICWNzmRLtspo8PIvjx+R0+IdGRV0ThTvKWV50kNe3HWTTgSqm5WfzwI0L+uxaAy0tzrq9lTy38R2e2xTpyhuTO5CvXHgcl88b2++7PxOBEoTIMdpeWsvtj69nSGY6Z0wbxplThzNxWOYxDRRXN4TITE/rV1fbFFc1kJs5oMMrqaT3dZYgdCe1SBdMHp7Fwzef1qPvObgfdmmO6EP3pPQH6rgTEZF2KUGIiEi7Ak0QZnaRmW0xs61mdls72zPM7A/R7SvMbFK0fICZ/dbM1pvZJjP7WpBxiojIkQJLEGaWCtwNXAzMAq4xs7ZTJN4ElLv7NOC/gf+Mll8FZLj7icDJwKdbk4eIiPSOIFsQC4Ct7l7k7k3AYuCyNvtcBvw2+vwx4P0WuSzEgSwzSwMGAU1AVYCxiohIG0EmiLFA7ILJe6Jl7e7j7mGgEhhGJFnUAvuBXcB/uXtZgLGKiEgbiTpIvQBoBsYAk4Evm9mUtjuZ2S1mttLMVpaUlPR2jCIifVqQCWIvEDtZ/bhoWbv7RLuTcoGDwMeAp9095O7FwKvAETdyuPu97l7g7gX5+fkBVEFEpP8K8ka5QmC6mU0mkgiuJvKLP9YS4DrgdeBK4AV3dzPbBZwLPGRmWcBpwE87O9mqVatKzWxnTFEukS6r9l63Po8tGw6UdquGHZ+rO/u0V95Z7LGv26vTsdSjszi7sk9363K05/H6TDralox1OZafr9jnyfhdCfIz6SzOruyTSHXpeJZSdw/sAXwAeAvYBtweLbsTuDT6fCDwR2Ar8E9gSrQ8O1q+AdgIfPU9nPvejl63Pm9TtvIY6nnve92nvfLOYu8k/tay91yP3q7L0Z7H6zPpS3U5lp+vTn7WkqIuQX4mfa0uHT0CnWrD3ZcCS9uU3RHzvIHIJa1tj6tpr7yb/tbJ6791sE9Pnas7+7RX3lnssa/bq9Ox6s26dOX5e3Us9ehoWzLW5Vh+vmKf6+era/F0dZ9Eq0u7+sxkfcfKzFZ6BxNWJZO+Ug9QXRJVX6lLX6kHBFeXRL2KKR7ujXcAPaSv1ANUl0TVV+rSV+oBAdVFLQgREWmXWhAiItIuJQgREWmXEoSIiLRLCeIozOwsM/uVmf3GzF6LdzzHwsxSzOx7ZvYLM7su3vEcCzNbZGZ/j342i+Idz7Ews6zolDGXxDuWY2Fmx0c/j8fM7LPxjudYmNnlZvbr6HIEF8Q7nmNhZlPM7H/N7LHuHtunE4SZ3WdmxWb2ZpvyTtepiOXuf3f3zwBP8O7Ms72uJ+pCZPbccUCIyOSJcdFDdXGghsjNlnGpSw/VA+A/gEeDibJreui7sin6XfkIcGaQ8Xamh+ryF3e/GfgM8NEg4+1MD9WlyN1vek/n78tXMZnZQiK/RB5099nRslQid3efT+QXSyFwDZAK/KDNW9zokbmgMLNHgZvcvbqXwj9MT9Ql+ih393vM7DF3v7K34o/VQ3UpdfcWMxsJ/MTdP95b8bfqoXrMJTKD8UAidXqid6I/XE99V8zsUuCzwEPu/nBvxR+rh7/3PwZ+7+5v9FL4h+nhunT7Ox/ondTx5u6vtLPQ0KF1KgDMbDFwmbv/AGi3iW9mE4DKeCUH6Jm6mNkeImtrQGS23Ljoqc8lqhzICCLOo+mhz2QRkEVkUa16M1vq7i1Bxt2envpM3H0JsMTMngTikiB66HMx4IfAU/FKDtDj35Vu69MJogPtrVNx6lGOuQm4P7CI3rvu1uXPwC/M7CzglSADew+6VRcz+zBwIZAH/E+woXVLt+rh7rcDmNn1RFtFgUbXPd39TBYBHyaSsJd2tF+cdPe78q/AeUCumU1z918FGVw3dfdzGQZ8D5hvZl+LJpIu6Y8Jotvc/ZvxjqEnuHsdkWSX9Nz9z0QSXp/g7g/EO4Zj5e4vAS/FOYwe4e4/B34e7zh6grsfJDKW0m19epC6A11ZpyJZqC6Jp6/UA1SXRNVrdemPCeLQOhVmlk5knYolcY7pvVJdEk9fqQeoLomq9+oSxBziifIAHiGyrnXrZZ03RcuPWKci0R+qS+I9+ko9VJfEfcS7Ln36MlcREXnv+mMXk4iIdIEShIiItEsJQkRE2qUEISIi7VKCEBGRdilBiIhIu5QgpE8zs5pePl+PrBlikfUuKs1sjZltNrP/6sIxl5vZrJ44vwgoQYh0i5l1On+Zu5/Rg6f7u7vPA+YDl5jZ0dZYuJzIrLAiPUIJQvodM5tqZk+b2SqLrEo3M1r+ITNbYWarzey56FoTmNm3zOwhM3sVeCj6+j4ze8nMiszs8zHvXRP9d1F0+2PRFsDvo1NIY2YfiJatMrOfm1mna0C4ez2whsgsnpjZzWZWaGZrzexPZpZpZmcAlwJ3RVsdUzuqp0hXKUFIf3Qv8K/ufjLwFeCX0fJ/AKe5+3xgMfDvMcfMAs5z92uir2cSmW58AfBNMxvQznnmA1+MHjsFONPMBgL3ABdHz59/tGDNbAgwnXenaP+zu5/i7nOBTUSmX3iNyHw8X3X3ee6+rZN6inSJpvuWfsXMsoEzgD9G/6CHdxccGgf8wcxGA+nA9phDl0T/km/1pLs3Ao1mVgyM5MilT//p7nui510DTCKyOliRu7e+9yPALR2Ee5aZrSWSHH7q7gei5bPN7LtE1sLIBpZ1s54iXaIEIf1NClAR7dtv6xdEli9dEl385lsx22rb7NsY87yZ9r9LXdmnM39390vMbDKw3Mwedfc1wAPA5e6+NrrQ0KJ2ju2sniJdoi4m6VfcvQrYbmZXQWRpSTObG92cy7vz6l8XUAhbgCkxy0h+9GgHRFsbPwT+I1o0GNgf7daKXYu7OrrtaPUU6RIlCOnrMs1sT8zjS0R+qd4U7b7ZAFwW3fdbRLpkVgGlQQQT7ab6HPB09DzVQGUXDv0VsDCaWL4BrABeBTbH7LMY+Gp0kH0qHddTpEs03bdILzOzbHeviV7VdDfwtrv/d7zjEmlLLQiR3ndzdNB6A5FurXviHI9Iu9SCEBGRdqkFISIi7VKCEBGRdilBiIhIu5QgRESkXUoQIiLSLiUIERFp1/8HxA2RK2PCYmYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find(suggestions=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy_multi</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.090206</td>\n",
       "      <td>0.088547</td>\n",
       "      <td>0.912626</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, lr_max=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=1.3460866057357635e-07, lr_steep=3.62389823749254e-06)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXyU9bX48c/JTkISthB2whJAkE0DKgrFBUVvK7Z1QetCXdD2p9UuVtve9nrtrdZq1dpalRZcq6hoLSqC+45C2AlrWJOwZN/XyZzfHzPBECbJAPNkJsx5v155MfMsM2cekufMdxdVxRhjjGkpItgBGGOMCU2WIIwxxvhkCcIYY4xPliCMMcb4ZAnCGGOMT5YgjDHG+BQV7AACpVevXpqWlhbsMIwxplNZtWpVoaqm+Np3wiSItLQ0MjMzgx2GMcZ0KiKyp7V9VsVkjDHGJ0cThIjMFJGtIpItInf72D9NRFaLiEtELm22fYKILBeRLBFZLyJXOBmnMcaYIzmWIEQkEngcuBAYDVwpIqNbHLYXmAO82GJ7NXCtqo4BZgKPikg3p2I1xhhzJCfbICYD2aq6E0BEFgKzgE1NB6jqbu8+d/MTVXVbs8f7RCQfSAFKHYzXGGNMM05WMfUHcpo9z/VuOyoiMhmIAXb42DdXRDJFJLOgoOCYAzXGGHOkkG6kFpG+wPPAD1XV3XK/qs5T1QxVzUhJ8dlLyxhjzDFyMkHkAQObPR/g3eYXEUkC3gZ+o6pfBTg2Y4wJCTsLKqmobTimc+tcjazZW8KX2YUBjsrDyTaIlUC6iAzBkxhmA1f5c6KIxAD/Bp5T1UXOhWiMMcGxak8Jj3+UzYdb8hnSK4Hnrp/MwB7x7Z6nqsz7dCfvbDzApn3l1De6Gd03iSW3Tw14jI4lCFV1icitwDIgEligqlkici+QqaqLRWQSnkTQHfiOiPyvt+fS5cA0oKeIzPG+5BxVXetUvMYYE0j/WZvHM1/upqHRjatRcasSFx1JXHQktQ2NrM8to3t8NHOnDeXllTl874kvefaHkxndL6nN131jbR73v7OF8QOSmXNmGhMGdmPCQGc6ecqJsqJcRkaG2khqY0wocDW6OeuBj4gQGNU3iehIQRBqXY3U1DficisXntyHKycPIiE2im0HK7huwQoqa13833dPZvyAbgzo3oWoyMNbAfaX1XD+I58yMjWRl28+g8gIOe5YRWSVqmb42nfCTLVhjDGh4oMt+Rwor+XJq09l5sl92j1+RGoir/1oCtctWMHtCz0VJVERwqi+idw1cxRT01NQVX65aD2uRuWhy8YHJDm0xxKEMcYcBVejm+35lYzqk4iI75v0C1/toU9SHOed1Nvv1+3XrQtv3nYWG/LK2FVYxa7CKt7ZsJ9r5q/g4vH9SO/dlc+2F/L7S04mrVdCoD5OmyxBGGOMn9xu5eevruM/a/fx8xkjuO3c9COO2V1YxWfbC7njvPQjqojaExcdyaS0HkxK6wHA7eem88THO3ji4x0sbnQzNb0XV582KCCfxR+WIIwxxg+qyj1vZvGftfsY1SeRP7+3jS4xkdw4dehhx724Yi+REcLsScd/I4+LjuSnM0Ywa0I/Fq7M4cazhrRaanGCJQhjjAFKqup59P1tlNY0EB8TSZfoKAZ078Kpg7szul8Sj32wneeW7+HmaUO584KR/GThGv7v7c3Ex0RxlfdbfW1DI69k5nD+6FT6JMcFLLahKV359UUnBez1/GUJwhgT9tbnlvKjF1aTX1FL3+QuVNc3UlPvoqq+EYCYqAjqXW6uyBjI3ReOQkR49IqJ1NRn8ps3NvB5dgEzRqdSVt1AaXUDV58+OMifKDAsQRhjwpaq8tKKHO5ZnEVKYiyLbpnC+GZjCvaX1bB6Tymr95aQFBfNrecMP1TFExMVwRNXn8of39nC2xv2s2TDAQCG9kpgyrCeQfk8gWbjIIwxYWvB57u4961NTE3vxV9mT6RHQswxvY7brazLLeWjLfmcMawXZ3SiBGHjIIwxpoUvdxTyhyWbOX90Kk9cfepxjSuIiBAmDurOxEHdAxhh8IX0bK7GGOOEnOJq/t+/VjOkVwIPXzGhQwaddUaWIIwxYaWmvpGbn1+Fy63Mu+ZUusZaRUprLEEYY04YNd5eR62pqG3gpucy2XygnMdmT2RoStcOiqxzsgRhjOlUVBVfnWuWbjzAyfcs45eL1lFWc+T6CvvLarjsyeV8tbOIP31/HGeP8n8ajHBlZStjTMi7Zv7XrMsppc7lps7lZvKQHiyYM+lQ9dCBslrufn09KV1jWbQql0+3FXL/98cyOa0HB8pr2V1YxW/+vZHKOhdP/3ASU9NtBUp/WIIwxoS0/IpaPtteyFnDezGmfxJut7Lgi93c9GwmT/9wEjGREfzi1XXUNbh57UdTqKx1ceeidfzw6ZWHvU6fpDheveUMTurb9noL5huWIIwxIS1rXzkAt54znNOHesYXjOmXzE9fWcutL65hUlp3Ps8u5L7vjmWYt03hzdvOYuGKHGoaGklNiiU1KY6T+yeTFBcdtM/RGVmCMMaEtE3eBNF8pbVLJvanoraB3/4ni/c3H2TG6FSunDzw0P7YqEium5LW0aGecBxtpBaRmSKyVUSyReRuH/unichqEXGJyKUt9i0VkVIRecvJGI0xoW3TvnIG9Yg/4tv/NWek8euLRnFy/yT++L2xHTrLabhwrAQhIpHA48AMIBdYKSKLVXVTs8P2AnOAX/h4iQeBeOBmp2I0xoS+rH1ljGllnea504Yxd9qwDo4ofDhZgpgMZKvqTlWtBxYCs5ofoKq7VXU94G55sqp+AFQ4GJ8xJsRV1Dawu6ia0dawHBROJoj+QE6z57nebcYY45fN+z3fEcf0twQRDJ16oJyIzBWRTBHJLCgoCHY4xpgAy9pXBnh6LZmO52SCyAMGNns+wLstYFR1nqpmqGpGSooNfDHmRLNpXzm9usbQOzE22KGEJScTxEogXUSGiEgMMBtY7OD7GWNOMFn7yjmpb5L1UAoSxxKEqrqAW4FlwGbgFVXNEpF7ReRiABGZJCK5wGXAUyKS1XS+iHwGvAqcKyK5InKBU7EaY0JPvcvN9vwKq14KIkcHyqnqEmBJi22/a/Z4JZ6qJ1/nTnUyNmNMaNt2sIKGRm21i6txXqdupDbGnLiaRlBbgggeSxDGmJCUta+M+JhI0nomBDuUsGUJwhgTkjbt9zRQR9hyoEFjCcIYE3LcbmXTvnKrXgoySxDGmJCzp7iaqvpGm2IjyCxBGGNCzrqcUgDGDegW5EjCmyUIY0zIWZtTSpfoSEakdg12KGHNEoQxJuSsySll7IBkoiLtFhVMdvWNMSGlztXI5n3lTBxo1UvBZgnCGBNSsvaVU9/oZuIgSxDBZgnCGBNS1u71NFBPGNg9yJEYSxDGmJCyNqeUPklx9EmOC3YoYc8ShDEmpKzNKWWCtT+EBEsQxpiQUVRZx97iaiZY+0NIsARhjAkZ63Kb2h8sQYQCSxDGmJCxZm8pEQLjBtgiQaHAEoQxJmSszSllZJ8k4mMcXcvM+MkShDEmJLjdag3UIcbRBCEiM0Vkq4hki8jdPvZPE5HVIuISkUtb7LtORLZ7f65zMk5jTHCs2VvCG2vy2JBbRta+cipqXTaCOoQ4Vo4TkUjgcWAGkAusFJHFqrqp2WF7gTnAL1qc2wP4HyADUGCV99wSp+I1xnQMVeXjbQU88fEOVuwqPmK/9WAKHU5W9E0GslV1J4CILARmAYcShKru9u5ztzj3AuA9VS327n8PmAm85GC8xhiHrcsp5b/f2MiGvDL6Jsfxu2+PZsrwnuwqqGJ7fiUA6b1tBtdQ4WSC6A/kNHueC5x2HOf2b3mQiMwF5gIMGjTo2KI0xhw3VeXjrQWU1tRzyYT+iBy+TGhFbQN/fncbzy7fTUrXWP506TgumdCfmChPLfeoPklcGIS4Tds6dVcBVZ0HzAPIyMjQIIdjTNhRVT7cks+j729nQ14ZAO9tOsgD3x9HYlw0brfyxto8Hli6hfyKOq49fTA/v2AkSXHRQY7c+MPJBJEHDGz2fIB3m7/nTm9x7scBicoY0y5Vz/etliWB5vLLa7n1xTWs2F3MwB5d+NP3x1FcXc+Dy7ayef8X3HFeOvM/38X63DLGDUjmqWsyrIdSJ+NkglgJpIvIEDw3/NnAVX6euwy4T0SapnM8H/hV4EM0xrSkqtz92gaWZh3gysmDuG7KYPomdznsmLU5pdz8fCblNS7u++5YLssYQLR3cZ+JA7tx20truH3hWvokxfHw5eO5ZEJ/IiJaTzYmNEnTNwVHXlzkIuBRIBJYoKp/EJF7gUxVXSwik4B/A92BWuCAqo7xnns98GvvS/1BVZ9u670yMjI0MzPTqY9iTNj424fbeejdbYwbkMzGvDIiRJgxOpX01ERSk2KprmvkwXe30jsxln9cm8FJfZOOeI2Cijo+3prPt8f1o0tMZBA+hfGXiKxS1Qyf+5xMEB3JEoQxx++t9fu49cU1XDKhH49cMYHckhqe+XI3b63fR35FHU23izOG9uTxH5xCj4SY4AZsjpslCGNMu9bsLWH2vK8Y2z+ZF248jbjow7/5NzS6Kayso7S6gfTeXW296BNEWwmiU/diMsYERklVPbe8sIreSbE8dc2pRyQHgOjICPomdzmiPcKcuCxBGBPmVJW7XltPcVU9//7xmfTsGhvskEyIsDKiMWHuxRV7eXfTQe6aOYqT+9s02+YbliCMCWPZ+RX8/q1NTE3vxfVnDgl2OCbEWIIwJkw1upWfvLSW+Jgo/nzZeBunYI5gCcKYMLV6bwmb9pfzqwtH0TspLtjhmBBkCcKYMLVs4wFiIiOYeXKfYIdiQpQlCGPCkKqybNMBpgzvSaJNnGdaYQnCmDC0eX8FOcU1XDDGSg+mdZYgjAlDS7MOIAIzRqcGOxQTwixBGBOG3s06wKTBPehlg+JMGyxBGBNm9hRVseVABeePsdKDaZslCGPCzLKsAwDW/mDaZQnCmDCzdOMBRvdNYmCP+GCHYkKcJYgAm//5Lm58diUVtQ3BDsWYI+SV1rAmp9RKD8YvliAC7P1NB3l/cz4/+OfXlFbXBzscYw5Zn1vK5U8uJzoygu+M7xvscEwn4GiCEJGZIrJVRLJF5G4f+2NF5GXv/q9FJM27PUZEnhaRDSKyTkSmOxlnIOWV1jAsJYEtByqYPe8rCirqgh2SMby8ci+XPrkcgEW3nMHQlK5Bjsh0Bo4lCBGJBB4HLgRGA1eKyOgWh90AlKjqcOAR4AHv9psAVHUsMAP4s4iEfGmn0a3sK63h/DF9WHDdJPYUVXPlP76itqEx2KGZMLbg813c9doGJqf14M3bzmLcgG7BDsl0Ek7edCcD2aq6U1XrgYXArBbHzAKe9T5eBJwrIoInoXwIoKr5QCngc0m8UJJfUYvLrfTv1oWz0nvx8OXjyc6v5LPthcEOzYSpspoG/vLBdqam9+LZ6yfbGtLmqDiZIPoDOc2e53q3+TxGVV1AGdATWAdcLCJRIjIEOBUY2PINRGSuiGSKSGZBQYEDH+Ho5JXUANC/u2dJxvNGp9ItPpq31+8LZlgmjM3/fBdlNQ3cNXMUkTadtzlKoVptswBPQskEHgW+BI6op1HVeaqaoaoZKSkpHRzikfJKPQliQDdPgoiOjGDmmD68t+mgVTOZDldcVc/8z3Zy0dg+tlKcOSZOJog8Dv/WP8C7zecxIhIFJANFqupS1Z+q6gRVnQV0A7Y5GGtA5LYoQQB8e1w/quob+WRb8Es45sRW29CI262Hnj/1yQ5qGhr52YwRQYzKdGZOJoiVQLqIDBGRGGA2sLjFMYuB67yPLwU+VFUVkXgRSQAQkRmAS1U3ORhrQOSW1NAjIYb4mKhD204f2oMeCTG8vX5/ECMzTXYXVpFTXH3YjfREsD63lMl/eJ/zHvmE55fvZldhFc8u380lE/ozvHdisMMznVRU+4ccG1V1icitwDIgEligqlkici+QqaqLgfnA8yKSDRTjSSIAvYFlIuLGU8q4xqk4AymvtIb+3bocti3KuyDLG2vyqKlvpEtMZJCiC2819Y08sHQLz3y5G4CEmEiGpyYyoFsXEuOiSOoSTVrPBC6Z2O+wBN8ZbD1QwbULVpAYF01ibBS//U8WIhApwu3npQc7PNOJOfqXoKpLgCUttv2u2eNa4DIf5+0GRjoZmxPySqpJ9/Ft7dtj+/Li13v5eGs+F461AUqB1uhW3tm4n66xUUwf2fuI/Rtyy7jj5TXsKKhizpQ0RqQmsu1gBVsPVLDlQDnltS7Kaxqoc7l56N2tzJmSxnVnpJEcH/oL6ewsqOQH//ya2KgIXrzpNAb1iGdNTikvLN/DyD6JDO6ZEOwQTSfWub4qhTBVJa+0hrN93KAmD+lBr64xvLVhvyWIAFJV3tt0kD+/u42tBysAuPOCkfx4+jBEBFejm79/vIPHPthOz64xPH/DZKamt96ZIXN3MU98vIOH39vGvE938tcrJ3L2qCP/P0NFQUUdV//za9yqLLzx9EPJ4JRB3TllUPcgR2dOBJYgjlF+eS0pibF4hm1AUVU9tQ3uwxqomzRVM722Ko/qelenq8IIRTX1jcx5egVf7ypmaK8E/jJ7Ah9uyefBZVvZWVDFjVOHcPdr61mXW8Z3xvfj97PG0C2+7TEAGWk9mD+nB5v3l3PnonXc+Fwm9333ZK6YNKiDPtXRuW/JZgor63n9x1OsncE4IlS7uYa0/WU1TPnjh7zVrOH50BiIbkcmCICLxvalpqGRL7KLOiTGE929b2Xx9a5ifj9rDO/+dBqzJvTn0SsmcMd56by2OpcL//IZe4qr+dtVE/nrlRPbTQ7NndQ3iYVzz+DM4b2467UNPPzeNvIraimpqqeitgHVIxu4D5TVsmpPSSA/YptW7Crm32vyuGnaEOvCahxjX2WPwca8clxu5bPtBXxnfD/gmzEQvkoQABMGdkMENu8vt2Uej9Pidft4aUUOP5o+jGvOSDu0XUS447wRpPdO5LPtBfxsxgh6J8Ud03t0jY1i/nUZ/Pr1DTz2wXYe+2D7oX2JsVGc1DeJ0f2SEIHPtxeyPb8SgHtnjeHaZjE5wdXo5nf/2Ui/5Dj+39nDHX0vE94sQRyDbd767szd33xjbCpBDOjme479+JgoBvWIP1RXbo7N7sIqfv36Bk4d3L3V/v3/Na4v/zXu+Nt6oiMj+NOl45gxOpWDFXW4Gt00NLrJKa4ha18ZL6/Mwa3K5CE9uCxjAMt3FHHP4iwGdO/COaOc+xLw3PI9bDlQwZNXn2LVlcZR9tt1DLK93xZ3FlZRVFlHz66x5JXWkBgbRVKX1i/piNREth2wBHE0Pt9eyKPvb6NX11j6dotj+Y4iIiOEx66cSHSk8zWkIsL5rayd0OhWGt1KTJQnjh+cNpjLn1rOrS+u4dVbzmBMv8BX/eRX1PLIe9uYmt7L1nQwjrM2iGOw7WAF3b1dIDO99c65JdX0797lUKO1LyNTE9lVWEWdy6bd8Ieqct+SzWQXVJJdUMkrK3PYU1TNQ5eNb7WtpyNFRsih5ACQEBvFgjmTSO4SzfXPrOSVzBy2H6wI6KC8B5dupdbVyD0Xj2nzd82YQLASxFFqdCvZ+ZVcMWkgC1fmkLm7mAvG9CG35MhBci2lp3bF5VZ2FVYxqk9SB0XceX22vZBN+8t54PtjuWLSIFQVl1s7pORwrFKT4lgwZxLXLljBLxetBzxtFj88awg/PS/9uG7qG/PKWLQ6l5umDmWYredgOoAliKOUU1xNncvNmH5JjB+QzEpvO0ReaQ2Th/Ro89yRfTxdEbceqLAE4YcnP9lBalIsl0z0TAIsIkRHhv635pP6JvH1r85lZ2Ela/aW8sHmfB77YDu5JdU88P1xx5TgVJV739pE9/gYbj3HGqZNx/DrN1VEEpoW7BGRESJysYiE/jBTBzT1VklPTSQjrQdZ+8rIL6+lotbFgFZ6MDUZ2qsrURFyqJHbtG59bilf7ijihrOGEBvV+aYniYgQhvdO5LKMgTxx9Sn8fMYIXl+dx/XPrKSyznXUr7d04wFW7CrmZzNGkBQXln96Jgj8/SrzKRAnIv2Bd/HMjfSMU0GFsqabe3rvrkxK605Do/L2Bs94iP6t9GBqEhMVwZBeCWw9UOl4nEejoKKOF77ag6vRHexQDnnykx0kxkVx5eTQHKR2NESE285N50/fH8eXO4q47Mnl5BRX+31+nauR+97ZzMjURGZPOmJZFGMc42+CEFWtBr4H/F1VLwPGOBdW6Np+sIK+yXEkxkVz6iBPldIbaz0LArU2BqK5EX0SQ6oEUVRZx1X/+Ir/fmMjn24PjSnJdxVW8c7GA1xz+mAST6Bvy5dPGsj86zLILanm4r99zpfZ/q00OP/zXeQU1/Df3z6JqBBufzEnHr8ThIicAfwAeNu7rfOV+wNge34l6ametoTk+GhGpHZlXU4p0Poo6uZGpiayt7ia6vqjr2YItLLqBq6Zv4K9xdXERkXw0ZbQSBB//yib6MgIfnjmkGCHEnDTR/Zm8a1n0bNrLNcsWME/Pt1JQxslt9V7S3jkvW1cMCa1zXmkjHGCvwniDuBXwL+9U3YPBT5yLqzQ1NSDaUTvb3qQZKR5ShGxURH06tr+dA4jvMll+8HgVjNV1DZw7dMryM6vZN61GUxNT+Gjrfk+p5HoSF9mF/LqqlzmTEkjJTE2qLE4ZUivBP794ymcO6o3f1iymakPfMSTn+ygrKbhsOMKK+v48Qur6ZvchT99f3yQojXhzK8EoaqfqOrFqvqAt7G6UFV/4nBsIaepB1N66jcJYlKaZ9bM9sZANDnUkymI1UzLdxTxnb9+zsa8Mv521US+NSKFs0elkFtSw46C4CWuqjoXv3xtPUN6JZzwq6AlxkXz1DWn8vScSQxNSeCP72xhyv0fcM/iLHKKq3E1urntxTWUVNfzxNWndIqpx82Jx69uriLyInALnnWhVwJJIvIXVX3QyeBCzaEG6tRvZs7MGOwpQfg7cGtQj3hioyKCMqK6rKaB+5dsZuHKHAb1iOf5GyYzZVgvgEPrKHy0pSBoM4M+sHQLeaU1vHLzGcRFn/g1mCLC2aN6c/ao3mTtK2P+Z7t44as9PLd8NyP7JLF5fzkPXTbekRHZxvjD3yqm0apaDlwCvAMMoZOs8hZIh7q4NqtiGtC9C8NSEhjdz79xDZERQnpq1w4vQdQ2NHL5k8t5JTOHm6cNZdkd0w4lB/AkuFF9Evloa36HxtXkq51FPLd8D3OmpDEpre3xJCeiMf2SefiKCXx+1znMnTaMvJJq5kxJ49JTBwQ7NBPG/B0oF+0d93AJ8DdVbRCRdiurRWQm8Bc8Ddr/VNU/ttgfCzwHnAoUAVeo6m7ve/0TOMUb43Oqer+/H8op2w9W0M/bg6mJiPDmbWcd1eCnEamJfOFnD5aW8kpreGVlDq9k5jAiNZFnfjjJr6qtB5ZuYevBCp6eM6nVRXCmj+zN/M93UlHb0KG9hz7ccpBfvb6BwT3jufOCTreQYED1SY7j7gtHcdfM8L4OJjT4e1d7CtgNJACfishgoLytE0QkEngcuBAYDVwpIqNbHHYDUKKqw4FHgAe82y8DYlV1LJ7kcbOIpPkZq2O2HaxkeOqR1S/xMVFHlSBGpiZysLyOsuqG9g/2UlV+/so6pj7wIY99uJ3u8TF8sq2A11fntXvuZ9sLePqL3cyZktbmCmlnj0yhoVE7bM2KnOJqbnouk+ufyaRrbBR//4HNTtpERGyuJRN0/jZSP6aq/VX1IvXYA5zdzmmTgWxV3amq9cBCYFaLY2YBz3ofLwLOFc9fhQIJIhIFdAHqaSchOa3RrewoOLwH07Ea4W2o3pbvfzXT5v0VvLY6l++dMoBP7zybt247i4mDuvGHJZspra5v9bzS6np+8eo6hqUkcPeFo9p8j1MGdycxLoqPtjhfzZRbUs3MRz/l8+2F3H3hKN65fZrVtRsTYvydaiNZRB4WkUzvz5/xlCba0h/IafY817vN5zGq6gLKgJ54kkUVsB/YCzykqsX+xOqUph5MI3yUII7WyNRv5mTy17ubDiACd80cxcAe8URECPd9dyxlNQ08sHRLq+f99xsbKaqs5y+zJ7bb8BsdGcG0Duru+tb6/VTVN/LmbWdyy7eGHTYrqjEmNPj7V7kAqAAu9/6UA087FRSe0kcj0A9Pg/jPvWMvDiMic5uSVkGBs4O8mhqoh6cefwmib3IcibFRR5Ug3tt0kFMGdT9sbMBJfZO44awhvLQih1V7jsyfX+4o5K31+/nJuel+L0s5fWQK+RV1/P3jHTzzxS6e+mQH2UdR0vHX0o0HGNs/2dZSNiaE+Zsghqnq/3iri3aq6v8CR9ywW8gDmk8cM8C7zecx3uqkZDyN1VcBS1W1QVXzgS+AjJZvoKrzVDVDVTNSUpwdZVpQUQdAv+TjX4dARBjRJ9HvBJFbUk3WvnLO97FU6e3nptMvOY5fv77xsHUmVJWHlm2lT1Icc6e191/1jekjexMbFcGDy7Zyz5ubuP+dLVz+1Ffs8y6p6kt5bQMPLtvCrsIqv97jQFkta3NKmXmyLXhjTCjzN0HUiMhZTU9E5Eyg9TuGx0ogXUSGiEgMMBtY3OKYxcB13seXAh+qp25jL3CO970SgNOB1utROkCVdwbOhNjA9M8f2SeRrQcr/KrKeX/TQQCfK5slxEbxf989ma0HK3hw6dZD2z/ams/qvaXcdu7woxpTkJIYy1e/OpflvzqHNb+dwTu3T6Xe5eZH/1rtc6Gjwso6Zj/1FY9/tIPZ85azp6j9JPHupgMAXDDG1uY2JpT5myBuAR4Xkd0ishv4G3BzWyd42xRuBZYBm4FXvNN03CsiF3sPmw/0FJFs4GfA3d7tjwNdRSQLT6J5WlXXH8XnCrimKZoD1ctmVJ9EymoaOFhe1+6x7246yPDeXRnSy3ezzzmjUrn69EH88/NdfLqtALdbeXDZNgb1iOfyjKOf/bN7Qgx9k7vQPSGGk/om8dBl41mXU8o9izcddlxeaQ2XP7mcnYWV3POd0dS73Fz1j6/JLWl7ptJlWQcYlpJg1ZEH7c8AABJbSURBVEvGhDi/7naqug4YLyJJ3uflInIH0OZNW1WXAEtabPtds8e1eLq0tjyv0tf2YKqud9ElOpLIiMB0PWxqqN5yoJw+yXGHtmfnV7JoVS63n5tOl5hIyqob+HpXMTe3U030m4tG89XOYn7+6jpuO2c4m/eX8+gVEwKy+trMk/vwo+nDeOLjHSR1iaJ7fAzFVfW8uW4flXUuXrjhNDLSepCR1oOr/vEVV/7jK165+Qz6+qiOK6mq56ud7X8eY0zwHdXdQ1XLvSOqwfONP2xU1jWSEBu4PvrNV5dr7ukvdvHkJzuY+3wmtQ2NfLj1II1u9Vm91FyXmEgemz2RsuoGfvefLEakduU74/sFLN5fnD+Sb41I4alPdvLHd7bwzJe7SYyLYuHc0w9NWHhy/2Seu+E0Sqs8s8T66n77wZZ8Gt1q7Q/GdALHc8cLq1E8VXUuugao/QGgW3wMqUmxRySIzN0l9E6M5bPthdz64mpEhN6JsYzzoxfS6H5J3HXhKH7/1ibuvGBUwEo74Jki5Ok5k9hfXku3LtHEx0T6HMg1YWA3/nFdBtfOX8H1z6zkXzeeTpeYb67b0o0H6Jccx1g/e1UZY4LneOofgjsvdAerqnMFfJTvyD5Jh83JVFbdwLb8Cq45fTC/v+Rk3t+cz3ubDjJjdCoRft7sbzhrCF//+lxm+OjxdLwiIoT+3bqQEBvV5ijf04f25NHZE1iTU8ptL60+tFJdVZ2Lz7YXcP6YPjZK2JhOoM07nohU4DsRCJ4RzmGjqt5F1wBWMYGnofqZL4twNbqJioxg9d4SVOHUtO5MGdaLuoZG7n9nCxcfZVVRalJc+wc57KKxfbn34jH89j9ZzHr8CxJioiiurqfO5eaCdqrLjDGhoc07nqpaNxOvqrpGvxYEOhojUhOpd7nZXVTN8N5dWbm7mKgIYcLAbgDcOHUosycPCnhi6ijXnJFGncvNm+v3ExEBA7t34YyhPZk8JPxmazWmM+qcd54gqKpzMbhnfEBfc1SzhurhvbuSuaeEMf2TD6vK6qzJocmNU4dy41TrsWRMZ2QT4Pipqt5FQoDbIIb37kqEeFaXq3M1si6nlIzB3QP6HsYYc6wsQfipKsDdXAHioiNJ65XA1gPlbMwrp87lPrSEqTHGBJslCD+oqreROvDLYI5M9czJ1DTZ3qmDrX7eGBMaLEH4obq+EVWId6A9YGSfRPYUV/PptkLSesYfNlurMcYEkyUIP1TVN03UF/gEMapPIqrweXbhoRHJxhgTCixB+KGqzjOLqSNVTH2SDj229gdjTCixBOGHqgDP5NrcoB7xxEV7/husBGGMCSWWIPzQlCCcGJMQGSGk906kR0IMQ1uZztsYY4Khc4/C6iBOtkEA/Hj6MMprG2x+ImNMSLEE4YdKB9sgAC4c29eR1zXGmONhVUx+cLINwhhjQpUlCD98sx61JQhjTPhwNEGIyEwR2Soi2SJyt4/9sSLysnf/1yKS5t3+AxFZ2+zHLSITnIy1LU3dXBNinKliMsaYUORYghCRSOBx4EJgNHCliIxucdgNQImqDgceAR4AUNV/qeoEVZ0AXAPsUtW1TsXanqp6F7FREUQFYH1nY4zpLJy8400GslV1p6rWAwuBWS2OmQU86328CDhXjuzKc6X33KDxLDdq1UvGmPDiZILoD+Q0e57r3ebzGFV1AWVAzxbHXAG85OsNRGSuiGSKSGZBQUFAgvalqs5l7Q/GmLAT0nUmInIaUK2qG33tV9V5qpqhqhkpKSmOxVHpwFTfxhgT6pxMEHnAwGbPB3i3+TxGRKKAZKCo2f7ZtFJ66EhVdS5roDbGhB0nE8RKIF1EhohIDJ6b/eIWxywGrvM+vhT4UFUVQEQigMsJcvsDQHW9VTEZY8KPYwnC26ZwK7AM2Ay8oqpZInKviFzsPWw+0FNEsoGfAc27wk4DclR1p1Mx+qvSGqmNMWHI0bueqi4BlrTY9rtmj2uBy1o592PgdCfj81dVXSPxVsVkjAkzId1IHSqqrIrJGBOGLEG0Q1VtHIQxJixZgmhHbYMbt9o8TMaY8GMJoh3frAVhbRDGmPBiCaIdh2Zytam+jTFhxhJEOyptqm9jTJiyBNGOQ1N9WxWTMSbMWIJoh9PrURtjTKiyBNGOpjYI6+ZqjAk3liDaYcuNGmPClSWIdthyo8aYcGUJoh1WgjDGhCtLEO2orHcRExVBtK1HbYwJM3bXa4ctFmSMCVeWINpRbcuNGmPClCWIdthiQcaYcGUJoh22FoQxJlw5miBEZKaIbBWRbBG528f+WBF52bv/axFJa7ZvnIgsF5EsEdkgInFOxtoaW03OGBOuHEsQIhIJPA5cCIwGrhSR0S0OuwEoUdXhwCPAA95zo4AXgFtUdQwwHWhwKta22GJBxphw5WQJYjKQrao7VbUeWAjManHMLOBZ7+NFwLkiIsD5wHpVXQegqkWq2uhgrK2qqrMqJmNMeHIyQfQHcpo9z/Vu83mMqrqAMqAnMAJQEVkmIqtF5JcOxtmmSuvmaowJU6H61TgKOAuYBFQDH4jIKlX9oPlBIjIXmAswaNCggAehqlTXWzdXY0x4crIEkQcMbPZ8gHebz2O87Q7JQBGe0sanqlqoqtXAEuCUlm+gqvNUNUNVM1JSUgL+AepcblxutQRhjAlLTiaIlUC6iAwRkRhgNrC4xTGLgeu8jy8FPlRVBZYBY0Uk3ps4vgVscjBWn2yqb2NMOHPszqeqLhG5Fc/NPhJYoKpZInIvkKmqi4H5wPMikg0U40kiqGqJiDyMJ8kosERV33Yq1tZU13vaxa2bqzEmHDn61VhVl+CpHmq+7XfNHtcCl7Vy7gt4uroGTaWVIIwxYcxGUrfBpvo2xoQzSxBtqPJWMSXEWhWTMSb8WIJog5UgjDHhzBJEG5raIBJiLEEYY8KPJYg2WDdXY0w4swTRhkPdXK0NwhgThixBtKGyzkV0pBAbZQnCGBN+LEG0wWZyNcaEM0sQbSisrKNHfEywwzDGmKCwBNGGPUXVDO4ZH+wwjDEmKCxBtEJVvQkiIdihGGNMUFiCaEVRVT2VdS4rQRhjwpYliFbsKaoGIM1KEMaYMGUJohV7iqoAGGQlCGNMmLIE0YrdRdVECAzo3iXYoRhjTFBYgmjF3qIq+nXrYoPkjDFhyxJEK3ZbF1djTJizBNGKPUVV1sXVGBPWHE0QIjJTRLaKSLaI3O1jf6yIvOzd/7WIpHm3p4lIjYis9f486WScLZXVNFBS3UCalSCMMWHMsYmGRCQSeByYAeQCK0VksapuanbYDUCJqg4XkdnAA8AV3n07VHWCU/G1Za+3i6uVIIwx4czJEsRkIFtVd6pqPbAQmNXimFnAs97Hi4BzRUQcjMkvu71dXK0NwhgTzpxMEP2BnGbPc73bfB6jqi6gDOjp3TdERNaIyCciMtXXG4jIXBHJFJHMgoKCgAV+aAxED0sQxpjwFaqN1PuBQao6EfgZ8KKIJLU8SFXnqWqGqmakpKQE7M33FFWTmhRLvC01aowJY04miDxgYLPnA7zbfB4jIlFAMlCkqnWqWgSgqquAHcAIB2M9zJ6iagb3sPYHY0x4czJBrATSRWSIiMQAs4HFLY5ZDFznfXwp8KGqqoikeBu5EZGhQDqw08FYD7O7qMraH4wxYc+xOhRVdYnIrcAyIBJYoKpZInIvkKmqi4H5wPMikg0U40kiANOAe0WkAXADt6hqsVOxNldd7yK/oo60XlaCMMaEN0cr2VV1CbCkxbbfNXtcC1zm47zXgNecjK01e4uburhaCcIYE95CtZE6aHYXehOEtUEYY8KcJYgWbJpvY4zxsATRwp7ianokxJDcJTrYoRhjTFBZgmhhT1GVDZAzxhgsQRwmv6KWNXtLGdUnMdihGGNM0FmCaOaR97bR0Ojm5m8NC3YoxhgTdJYgvDbvL+fllTlce0YaQ2wMhDHGWIIAUFXuW7KZpC7R/OSc9GCHY4wxIcESBPDxtgI+217IT85JJzneei8ZYwxYgsDV6OYPb29mSK8Erj59cLDDMcaYkBH2CWJfaS0NjW5+deEoYqLC/nIYY8whYb/gwaCe8bz3028RHRn0heyMMSakhH2CAKzkYIwxPtid0RhjjE+WIIwxxvhkCcIYY4xPliCMMcb4ZAnCGGOMT5YgjDHG+GQJwhhjjE+iqsGOISBEpADY08YhyUDZMez3tb3ltrae9wIK23jf49HeZzqec9o6zqlrBc5dr2O5Vv6e59S1armto65Va7EE4pxj+Tv09xra32Hr29u6NoNVNcXnO6hqWPwA845lv6/tLbe19RzIDNZnOp5z2jrOqWvl5PU6lmvl73lOXauW2zrqWh3r9Trea3U016WV62F/h35eQ3/fP5yqmN48xv2+trfc1t5zpxzL+/h7TlvHhcu18vc8p65Vy20dda2O9b2O91q1tt/fa9iZfrdC7e/QpxOmiilUiUimqmYEO47Owq6X/+xa+c+u1bEJpxJEsMwLdgCdjF0v/9m18p9dq2NgJQhjjDE+WQnCGGOMT5YgjDHG+GQJwhhjjE+WIIJIRCJE5A8i8lcRuS7Y8YQyEZkuIp+JyJMiMj3Y8XQGIpIgIpki8u1gxxLKROQk7+/VIhH5UbDjCSWWII6RiCwQkXwR2dhi+0wR2Soi2SJydzsvMwsYADQAuU7FGmwBulYKVAJxnMDXCgJ2vQDuAl5xJsrQEIhrpaqbVfUW4HLgTCfj7WysF9MxEpFpeG5Yz6nqyd5tkcA2YAaem9hK4EogEri/xUtc7/0pUdWnRGSRql7aUfF3pABdq0JVdYtIKvCwqv6go+LvaAG6XuOBnngSaqGqvtUx0XesQFwrVc0XkYuBHwHPq+qLHRV/qLM1qY+Rqn4qImktNk8GslV1J4CILARmqer9wBHFfBHJBeq9Txudiza4AnGtmikBYp2IM1QE6HdrOpAAjAZqRGSJqrqdjDsYAvW7paqLgcUi8jZgCcLLEkRg9Qdymj3PBU5r4/jXgb+KyFTgUycDC0FHda1E5HvABUA34G/OhhaSjup6qepvAERkDt7Sl6PRhZaj/d2aDnwPzxePJY5G1slYgggiVa0Gbgh2HJ2Bqr6OJ6Gao6CqzwQ7hlCnqh8DHwc5jJBkjdSBlQcMbPZ8gHebOZJdq6Nj18t/dq0CxBJEYK0E0kVkiIjEALOBxUGOKVTZtTo6dr38Z9cqQCxBHCMReQlYDowUkVwRuUFVXcCtwDJgM/CKqmYFM85QYNfq6Nj18p9dK2dZN1djjDE+WQnCGGOMT5YgjDHG+GQJwhhjjE+WIIwxxvhkCcIYY4xPliCMMcb4ZAnCnNBEpLKD3+/LAL3OdBEpE5G1IrJFRB7y45xLRGR0IN7fGLAEYcxREZE25y9T1SkBfLvPVHUCMBH4toi0t1bBJXhmbzUmICxBmLAjIsNEZKmIrPKuUjfKu/07IvK1iKwRkfe9a08gIveIyPMi8gXwvPf5AhH5WER2ishPmr12pfff6d79i7wlgH+JiHj3XeTdtkpEHhORNtdqUNUaYC2eWUoRkZtEZKWIrBOR10QkXkSmABcDD3pLHcNa+5zG+MsShAlH84DbVPVU4BfA373bPwdOV9WJwELgl83OGQ2cp6pXep+PwjP9+GTgf0Qk2sf7TATu8J47FDhTROKAp4ALve+f0l6wItIdSOebKeFfV9VJqjoez1QSN6jql3jmG7pTVSeo6o42PqcxfrHpvk1YEZGuwBTgVe8XevhmAaIBwMsi0heIAXY1O3Wx95t8k7dVtQ6oE5F8IJUjl0Jdoaq53vddC6ThWf1sp6o2vfZLwNxWwp0qIuvwJIdHVfWAd/vJIvJ/eNbG6IpnzqGj+ZzG+MUShAk3EUCpt26/pb/iWc50sXcRmXua7atqcWxds8eN+P5b8ueYtnymqt8WkSHAVyLyiqquBZ4BLlHVdd4Fgab7OLetz2mMX6yKyYQVVS0HdonIZQDiMd67O5lv1g24zqEQtgJDmy2TeUV7J3hLG38E7vJuSgT2e6u1mq/NXeHd197nNMYvliDMiS7eOw1008/P8NxUb/BW32QBs7zH3oOnSmYVUOhEMN5qqh8DS73vUwGU+XHqk8A0b2L5LfA18AWwpdkxC4E7vY3sw2j9cxrjF5vu25gOJiJdVbXS26vpcWC7qj4S7LiMaclKEMZ0vJu8jdZZeKq1ngpyPMb4ZCUIY4wxPlkJwhhjjE+WIIwxxvhkCcIYY4xPliCMMcb4ZAnCGGOMT5YgjDHG+PT/AYnC7hFSwfdJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find(suggestions=True, start_lr=5e-7, end_lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy_multi</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.091295</td>\n",
       "      <td>0.088013</td>\n",
       "      <td>0.913873</td>\n",
       "      <td>01:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.088233</td>\n",
       "      <td>0.087246</td>\n",
       "      <td>0.913965</td>\n",
       "      <td>01:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.087213</td>\n",
       "      <td>0.087196</td>\n",
       "      <td>0.914104</td>\n",
       "      <td>01:37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, lr_max=slice(1e-7, 3e-6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>None</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>As usual WW plumbing the depths for deeper meaning... that is unless it involves an issue on which they disagree then it is ridicule 24/7.   Clever creating the Bundyland series complete with cartoon banner.  Set the tone for the level of journalism to expect... journalism? ... fatastisticism. \\n\\nI did notice you soft pedaling the ridicule of David Fry identifying him as troubled.  My guess is that has more to do with sympathy for his pot smoking withdrawl rants than respect for his politics.  Respect is never a factor with liberals as evidenced by your series of vapid caricatures.  \\n\\nDid you happen to see the stories actual journalists did on Refuge mis-managment, fires, floods, and the millions of Carp that are harassing the birds away from the Bird Refuge?  The stories of arbitrary miss-management that are driving unemployment ever higher in eastern Oregon.  Curry County Sheriff turning in his badge in frustration for lack of resources dud to dwindling tax base engineered by arbitrary over reaching Federal Government policies.  Or how about the thousands of miles of roads proposed to be removed from Oregon wild lands, cutting off public and fire access?  What is the one issue the people of Oregon demand universally?  Access to the wild lands?  You are not reporting that it is being taken away.  More closed roads.  More illegal Federal Police, guns drawn, stops... as reported by the Sheriff of Grant County.  \\n\\nI suspect the real problem was real people you couldn't care less about were articulate, were actual victims, and made rational arguments you could not respond to.  The exact opposite of the great unbathed OWS movement, apart from the paid organizers, that held downtown Portland hostage and trashed 3 park blocks for 3 weeks.  \\n\\nShame on you.  You missed some really great stories and even greater people... people who don't drink $5 cups of coffee.</td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@motleycrew: You're right when you say that it is time to wonder about the differences between schools, and to do something about it.\\n\\nBut you've missed the point beyond that. This \"School Choice\" policy become a self-enforcing cycle: as attendance drops at a school, funding and staffing follow shortly there-after. As staff are lost and the remaining teachers are stretched even thinner the school struggles to provide the same programs as other schools. (Note that I saw this first-hand as music and other elective programs became harder and harder to enroll in during my 4 years at North). The lack of those programs encourage more families to request that their students be transferred to a different school. It's natural for those families to want the best education for their children, and they can't be blamed for this.\\n\\nBut here's the problem: It should surprise exactly no one that this option to transfer to a different school is over-overwhelmingly taken by more affluent families who...</td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.show_results(learner=learn, max_n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.loss_func.thresh = 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(((#2) ['insult','toxicity'],),\n",
       "  (#1) [tensor([ True, False, False, False, False,  True])],\n",
       "  (#1) [tensor([0.7143, 0.3579, 0.0679, 0.0543, 0.0276, 0.7736])])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.blurr_predict(\"You are the biggest loser! go to hell\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(((#2) ['insult','toxicity'],),\n",
       "  (#1) [tensor([ True, False, False, False, False,  True])],\n",
       "  (#1) [tensor([0.7061, 0.6684, 0.1649, 0.2593, 0.0840, 0.8123])])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.blurr_predict(\"Who the fuck you think you are!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(((#3) ['insult','obscene','toxicity'],),\n",
       "  (#1) [tensor([ True,  True, False, False, False,  True])],\n",
       "  (#1) [tensor([0.3693, 0.1071, 0.0180, 0.0107, 0.0128, 0.5056])])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.loss_func.thresh = 0.02\n",
    "comment = \"\"\"\n",
    "Those damned affluent white people should only eat their own food, like cod cakes and boiled potatoes. \n",
    "No enchiladas for them!\n",
    "\"\"\"\n",
    "learn.blurr_predict(comment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([3609, 6]), torch.Size([3609, 6]), torch.Size([3609]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds, targs, losses = learn.get_preds(with_loss=True)\n",
    "preds.shape, targs.shape, losses.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}