05-ULMFiT_Stance.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ULMFiT_Stance.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nyGNET-sG54v",
        "colab_type": "text"
      },
      "source": [
        "ULMFiT trained for Stance Detection Task\n",
        "\n",
        "Can try for other labels too\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03H7YqxypB-S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from fastai import *\n",
        "from fastai.text import *\n",
        "from pathlib import Path\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "07UnBczFx7-c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !wget 'https://storage.googleapis.com/kaggle-data-sets/2477%2F4140%2Fcompressed%2Ftraining.1600000.processed.noemoticon.csv.zip'\n",
        "!unzip '/content/kaggle140.zip'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BerZr-Ljzs0s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "kaggle140=pd.read_csv('/content/training.1600000.processed.noemoticon.csv',header=None,encoding = \"latin-1\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vN7kWi_uy1DU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! wget https://raw.githubusercontent.com/prrao87/tweet-stance-prediction/master/data/semeval2016-task6-trainingdata.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-raXxSk5y9iE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sem_eval = pd.read_csv('/content/semeval2016-task6-trainingdata.txt', delimiter='\\t', header=0, encoding = \"latin-1\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgT2QvSw3Wsv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# function to convert tweets into ASCII\n",
        "def clean_ascii(text):\n",
        "    return ''.join(i for i in text if ord(i) < 128)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o-aGeTfezzbd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "kaggle140.columns = ['col1', 'col2',\"col3\",\"Stance\",\"col5\",\"Tweet\"]\n",
        "kaggle140 = kaggle140[['Stance','Tweet']]\n",
        "# Clean Data and Remove URL's from the tweets\n",
        "kaggle140 = kaggle140.sample(n=200000,random_state=77)\n",
        "kaggle140['Tweet'] = kaggle140['Tweet'].apply(clean_ascii)\n",
        "kaggle140['Tweet'] = kaggle140['Tweet'].replace(r'http\\S+', '', regex=True).replace(r'www\\S+', '', regex=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KJA6voCPz_rl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sem_eval=sem_eval[['Stance','Tweet']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8-z4psDB0ONF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lang_dt=kaggle140.append(sem_eval, ignore_index=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "32j0xHDu3_40",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "9356f2c7-0d2e-4166-a109-2cc5b7767282"
      },
      "source": [
        "lang_dt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Stance</th>\n",
              "      <th>Tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>is terrified about exams</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>I ate waaaay too many cookies this weekend.  I...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>Perfect day.. And i get to spend it at work.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>Lost my phone in yesterdays rockshow at PESIT ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>Eventful morning/afternoon  Now to tackle the ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202809</th>\n",
              "      <td>AGAINST</td>\n",
              "      <td>There's a law protecting unborn eagles, but no...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202810</th>\n",
              "      <td>AGAINST</td>\n",
              "      <td>I am 1 in 3... I have had an abortion #Abortio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202811</th>\n",
              "      <td>AGAINST</td>\n",
              "      <td>How dare you say my sexual preference is a cho...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202812</th>\n",
              "      <td>AGAINST</td>\n",
              "      <td>Equal rights for those 'born that way', no rig...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202813</th>\n",
              "      <td>AGAINST</td>\n",
              "      <td>#POTUS seals his legacy w/ 1/2 doz wins. The #...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>202814 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          Stance                                              Tweet\n",
              "0       NO_QUERY                          is terrified about exams \n",
              "1       NO_QUERY  I ate waaaay too many cookies this weekend.  I...\n",
              "2       NO_QUERY     Perfect day.. And i get to spend it at work.  \n",
              "3       NO_QUERY  Lost my phone in yesterdays rockshow at PESIT ...\n",
              "4       NO_QUERY  Eventful morning/afternoon  Now to tackle the ...\n",
              "...          ...                                                ...\n",
              "202809   AGAINST  There's a law protecting unborn eagles, but no...\n",
              "202810   AGAINST  I am 1 in 3... I have had an abortion #Abortio...\n",
              "202811   AGAINST  How dare you say my sexual preference is a cho...\n",
              "202812   AGAINST  Equal rights for those 'born that way', no rig...\n",
              "202813   AGAINST  #POTUS seals his legacy w/ 1/2 doz wins. The #...\n",
              "\n",
              "[202814 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "scX_hcj005nG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tdt=pd.read_csv('/content/trainable.csv')\n",
        "tdt=tdt.rename(columns={'Tweets': 'Tweet','Support':'Stance'})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "boT6cQ5l1O_S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "7b4c55d4-5452-4107-ecfd-ecc301cfe368"
      },
      "source": [
        "lang_dt=lang_dt.append(tdt[['Stance','Tweet']],ignore_index=True)\n",
        "lang_dt.tail(3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Stance</th>\n",
              "      <th>Tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>209680</th>\n",
              "      <td>1</td>\n",
              "      <td>#metoo4publicity   now  s c women should also ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>209681</th>\n",
              "      <td>0</td>\n",
              "      <td>d lady behind d #metoo movemnt.... #metoodebat...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>209682</th>\n",
              "      <td>0</td>\n",
              "      <td>things that #metoo victims are tired of listen...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Stance                                              Tweet\n",
              "209680      1  #metoo4publicity   now  s c women should also ...\n",
              "209681      0  d lady behind d #metoo movemnt.... #metoodebat...\n",
              "209682      0  things that #metoo victims are tired of listen..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FmDFSphq1iwa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5Qt74mx1he8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "8c5064e3-8b00-467f-d20f-71ede4a291cc"
      },
      "source": [
        "path = Path('/content/')\n",
        "lang_dt.to_csv(path/'kaggle_train_sub_v2.csv', index=False, header=False)\n",
        "twitter_lm = TextLMDataBunch.from_csv(path, 'kaggle_train_sub_v2.csv', min_freq=1)\n",
        "twitter_lm.save()\n",
        "twitter_lm.show_batch()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>bank charge . evil bastards . regardless , how you all doing this am ! xxbos @shortyazk that sucks was hoping to see you there and was going to steal you for stewarding again lol . xxbos i received a txt msg in a foreign language from an international # &amp; &amp; thought it was from indonesia , when it was actually from serbia ! xxbos @calicowboy956 hey mister what</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>to watch xxbos @chaoseed there need to be more args xxbos xxmaj work sucks sometimes xxbos @phersure i hope you feel more sane later . xxbos @weblivz i guess son trumps over daughter in law xxbos @geneticgenesis xxmaj yes , but unfortunately i can not aid you all in your cause as i am not from the xxup uk and so can not vote # stopthebnp xxbos xxmaj yesterday was</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>xxmaj player of the xxmaj tournament &amp; ' today ! xxbos xxmaj nevada my xxmaj haflinger has a ' baby ' cold . xxmaj cough , runny nose . xxmaj started antibiotics when his temp went up xxmaj feeling better yesterday . xxbos @jreyes12 haha that s fact . xxbos @chazzdaddy you are so right xxrep 4 . i 'm so glad xxmaj he lives within me ! xxmaj hope</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>do i get to see you again ? xxbos been rescuing a broken biscuit from the bottom of my cup of tea xxbos @marijuana_news xxmaj hey , i 'm emailing you right now . xxmaj let 's sort this out your situation together . xxbos @ladaisyd xxmaj leave my poor baby alone xxbos 5th xxmaj sunday xxmaj singing xxbos xxmaj going to an audition ! xxmaj wish me luck !</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>xxbos yummy sushi place with the best . bummed to be leaving tomorrow flying outta jfk is a bitch xxbos think i clean the bedroom and living room now - well actually i should do the kitchen and bathroom either but i hate these ones xxbos i hate how dark my hair is xxbos @mitchelmusso hey mitchel c u tmro at walmart.could not make tonite bro . graduated college tonite</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FietGISI2lKx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "outputId": "b30a52a4-ae81-4f32-ff80-996f7cae0053"
      },
      "source": [
        "learn_twitter = language_model_learner(twitter_lm, AWD_LSTM, drop_mult=0.1)\n",
        "learn_twitter.lr_find(start_lr=1e-8, end_lr=1e2)\n",
        "learn_twitter.recorder.plot()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://s3.amazonaws.com/fast-ai-modelzoo/wt103-fwd.tgz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/1 00:00<00:00]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='95' class='' max='720' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      13.19% [95/720 01:07<07:23 23.3121]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeNElEQVR4nO3deZRcZ3nn8e9TVV29d6u71dqszZI3sPESy8aYwNixQ5yEYIfADD7MxIAPPhnmBJjMkIEhE0JO/iCBLAMkAZ3EOAvjTDAYGxiMTVgMCbaRbMlI3mQtllqWuktq9Vbd1bU988e9JbVlSd1S961b1fX7nFOnu24t71NXrd+99d73vtfcHRERaRyJuAsQEZHqUvCLiDQYBb+ISINR8IuINBgFv4hIg0nFXcBcLF261NevXx93GSIidWXr1q1H3L3/5OV1Efzr169ny5YtcZchIlJXzOylUy1XV4+ISINR8IuINBgFv4hIg1Hwi4g0GAW/iEiDUfCLiDQYBb+ISINR8IuI1KDBsRx/+vDz7MlMLPh7K/hFRGrQ3iNZPve9Fzk0mlvw91bwi4jUoGPZPAA9bekFf28Fv4hIDToaBn9fh4JfRKQhDGuPX0SksQxn83Q2p0inFj6mFfwiIjVoOJunN4JuHlDwi4jUpOFsnt52Bb+ISMMYzubpjaB/HxT8IiI1qS73+M3sbjMbMrMdM5Z92syeM7Onzex+M1sSVfsiIvXK3eu2j/8e4JaTlj0CXObulwMvAB+LsH0RkbqUzZfIl8r01dsev7s/CgyftOxhdy+Gdx8DVkfVvohIvRqeiG4MP8Tbx/8+4Nune9DM7jKzLWa2JZPJVLEsEZF4Hc1OA9GctQsxBb+ZfRwoAl8+3XPcfbO7b3L3Tf39/dUrTkQkZpWzdnvbmyN5/1Qk73oGZvYe4K3ATe7u1W5fRKTWHQ/+iLp6qhr8ZnYL8LvAv3P3yWq2LSJSL44Hf7119ZjZvcBPgIvNbMDM7gQ+D3QCj5jZNjP7QlTti4jUq+FsnnQqQXs6Gcn7R7bH7+63n2Lx30bVnojIYlE5a9fMInl/nbkrIlJjojxrFxT8IiI152g2H9lQTlDwi4jUnGOT+chO3gIFv4hIzRmeUFePiEjDmC6WGJ8uRjZPDyj4RURqyshkAYAeBb+ISGM4Gk7Qpj1+EZEGcWKeHgW/iEhDGJ5U8IuINJThiWBKZgW/iEiDGM7mMYMlGscvItIYhifzLGltIpmIZp4eUPCLiNSUqOfpAQW/iEhNOTqRpy+iK29VKPhFRGrIsck8Pe1Nkbah4BcRqSFBV4/2+EVEGkK57BybLER61i4o+EVEasZYrkCp7Dq4KyLSKI5WYboGUPCLiNSMaszTAwp+EZGaoeAXEWkwCn4RkQZT98FvZneb2ZCZ7Zix7J1mttPMyma2Kaq2RUTq0XA2T3s6SUtTMtJ2otzjvwe45aRlO4C3A49G2K6ISF0azuYjveRiRSqqN3b3R81s/UnLngUwi27WORGRenU0m4/85C2o4T5+M7vLzLaY2ZZMJhN3OSIikTtWhZk5oYaD3903u/smd9/U398fdzkiIpGrVldPzQa/iEijOZqdbuyuHhGRRpKdLpIrlCOfmROiHc55L/AT4GIzGzCzO83s181sAHgD8C0z+05U7YuI1JPDYzkAVnRHH/xRjuq5/TQP3R9VmyIi9WpwNAz+rtbI21JXj4hIDTixx98SeVsKfhGRGnDo+B6/gl9EpCEMjuXoaknRmo52ugZQ8IuI1ITDo7mqdPOAgl9EpCYMjuVY0R39gV1Q8IuI1ITDYzlWdEU/lBMU/CIisSuWymTGp6tyYBcU/CIisctMTFN2WK4+fhGRxnC4ikM5QcEvIhK7wSqevAUKfhGR2GmPX0SkwRway5FOJqpyERZQ8IuIxG5wNMeyruaqXZZWwS8iErNgDH91unlAwS8iErvBsemqHdgFBb+ISKzcPZinR3v8IiKNYWyqyFShpD1+EZFGUbkAy3Lt8YuINIZqXnmrQsEvIhKjwSqfvAUKfhGRWFUuuaiuHhGRBnF4LEdfe5p0qnpxrOAXEYnR4Fiuqnv7EGHwm9ndZjZkZjtmLOs1s0fMbFf4syeq9kVE6kE1r7VbEeUe/z3ALSct+yjwL+5+IfAv4X0RkYYVXGt3kQS/uz8KDJ+0+Fbg78Lf/w64Lar2RURq3XSxxNFsvqojeqD6ffzL3f1Q+PthYPnpnmhmd5nZFjPbkslkqlOdiEgVDY1NA9UdygkxHtx1dwf8DI9vdvdN7r6pv7+/ipWJiFTH8bN2F0tXz2kMmtlKgPDnUJXbFxGpGZUrb61c5MH/IHBH+PsdwANVbl9EpGYMxjBPD0Q7nPNe4CfAxWY2YGZ3Ap8CftHMdgE3h/dFRBrSodEcrU1JulpSVW03stbc/fbTPHRTVG2KiNSTw+FQzmpdcrFCZ+6KiMRkcDTH8q7mqrer4BcRicnLI1Os6m6tersKfhGRGEwXSxway7Gmt63qbSv4RURi8PJIDndYq+AXEWkM+4cnAVjbp+AXEWkIleBf06PgFxFpCAPDk6RTCZZ1alSPiEhD2D88yZqeVhKJ6o7hBwW/iEgs9g9PxjKiB+YY/GbWbmaJ8PeLzOxtZtYUbWkiIovXgeHJWEb0wNz3+B8FWszsPOBh4D8RXGFLRETO0uhkgbFcseaD39x9Eng78Ffu/k7g0ujKEhFZvCojelbHMKIHziL4zewNwLuBb4XLktGUJCKyuB04Fo7hr/E9/g8DHwPud/edZrYB+H50ZYmILF7Hx/D3Vn+eHpjjtMzu/kPghwDhQd4j7v7BKAsTEVms9g9P0tPWRGdLPGNk5jqq5/+YWZeZtQM7gGfM7CPRliYisjjFOaIH5t7V81p3HwNuA74NnE8wskdERM7SgRjH8MPcg78pHLd/G/CguxcAj64sEZHFqVR2Bo5N1UXwfxHYB7QDj5rZOmAsqqJERBarQ6NTFMsea1fPXA/ufhb47IxFL5nZjdGUJCKyeB0YngLiG8oJcz+4221mf2ZmW8LbnxLs/YuIyFk4MBzvGH6Ye1fP3cA48O/D2xjwpaiKEhFZrPYPT5JMGCu7W2KrYU5dPcBGd/+NGfc/aWbboihIRGQxO3BsklVLWkgl45scea4tT5nZz1fumNkbgalzbdTMPmRmO8xsp5l9+FzfR0Sk3uyPeQw/zH2P/7eAvzez7vD+MeCOc2nQzC4D3g9cC+SBh8zsm+7+4rm8n4hIPTkwPMnNr1keaw1z2uN39+3ufgVwOXC5u18F/MI5tvka4HF3n3T3IsFUEG8/x/cSEakbk/kiRybysY7hh7O8Ape7j4Vn8AL8zjm2uQN4k5n1mVkb8CvAmpOfZGZ3VUYRZTKZc2xKRKR21MJQTpjfpRfP6UKR7v4s8McEF3R5CNgGlE7xvM3uvsndN/X398+jTBGR2nBiVs76Df5znrLB3f/W3a929zcTHC94YR51iIjUhVoYww+zHNw1s3FOHfAGnPNE0ma2zN2HzGwtQf/+def6XiIi9WL/8CQdzSl62uK9ZPkZg9/dOyNq96tm1gcUgP/i7iMRtSMiUjP2Hc2ytrcNs3PqKV8wcx3OuaDc/U1xtCsiEqfdmQmuXNMTdxnz6uMXEZE5yhVKDBybYmN//NOcKfhFRKpg39Es7rChvyPuUhT8IiLVsHsoC6A9fhGRRrEnMwHA+UsV/CIiDWF3ZoLzlrTSlo5lTM0rKPhFRKpgz5EsG2qgmwcU/CIikXN3dg9NsLEGDuyCgl9EJHKDY9Nk86WaOLALCn4RkchVDuzWwlBOUPCLiERudxj86uoREWkQuzNZ2tNJlnc1x10KoOAXEYnc7swEG/o7Yp+crULBLyISsT2ZbM0c2AUFv4hIpCbzRQ6OTNVM/z4o+EVEIrX3SDBHT62M6AEFv4hIpHZnwsnZlqmrR0SkIezJTGAG6/sU/CIiDWF3JsvqnlZampJxl3Kcgl9EJEK1NEdPhYJfRCQi5bKz90iWDUsV/CIiDeHQWI6pQqmmDuyCgl9EJDK7h2prjp4KBb+ISEReHKrMyqk9fszsv5rZTjPbYWb3mllLHHWIiETp6YERlnU2099RG5OzVVQ9+M3sPOCDwCZ3vwxIAu+qdh0iIlHbPjDKlWuW1MzkbBVxdfWkgFYzSwFtwMsx1SEiEomRyTx7j2S5Ys2SuEt5laoHv7sfBD4D7AcOAaPu/vDJzzOzu8xsi5ltyWQy1S5TRGRetg+MAnCVgh/MrAe4FTgfWAW0m9l/PPl57r7Z3Te5+6b+/v5qlykiMi/b9o9gBq9b3R13Ka8SR1fPzcBed8+4ewH4GnB9DHWIiERm+8AIF/R30NnSFHcprxJH8O8HrjOzNguOeNwEPBtDHSIikXB3th0Yqcn+fYinj/9x4D7gSeBnYQ2bq12HiEhUBo5NMZzNc2WNBn8qjkbd/RPAJ+JoW0Qkak8dGAGo2eDXmbsiIgts+4ERmlMJLl7RGXcpp6TgFxFZYNsOjHDZed00JWszYmuzKhGROlUoldlxcLRmu3lAwS8isqCePzzOdLFcsyN6QMEvIrKgtoUHdmvxjN0KBb+IyALadmCE3vY0q3ta4y7ltBT8IiILaPuBkZqckXMmBb+IyAIZzxV4MTPBFatrt5sHFPwiIgtm+4FR3OGKNbU3MdtMCn4RkQXyxL5hEgZXr+uJu5QzUvCLiCyQx/cc5dJV3TU5I+dMCn4RkQUwXSzx1IERrj2/N+5SZqXgFxFZAE8PjJIvlhX8IiKN4om9wwBcs17BLyLSEB7fO8xFyzvobU/HXcqsFPwiIvNULJXZum+4Lrp5QMEvIjJvzxwaI5sv8frz++IuZU4U/CIi81Tp39cev4hIg3hszzDr+9pY3tUSdylzouAXEZmHctn5aR3174OCX0RkXl4YGmd0qsC1ddK/Dwp+EZF5qfTvv157/CIijeHxvcOs6m6p6QuvnKzqwW9mF5vZthm3MTP7cLXrEBGZL3fnib1B/34tX3jlZKlqN+juzwNXAphZEjgI3F/tOkRE5uvpgVEy49Ncv3Fp3KWclbi7em4Cdrv7SzHXISJy1h7Y9jLpZIJfumxF3KWclbiD/13Avad6wMzuMrMtZrYlk8lUuSwRkTMrlZ1vPP0yN17ST3drbc+/f7LYgt/M0sDbgK+c6nF33+zum9x9U39/f3WLExGZxU92HyUzPs1tV54XdylnLc49/l8GnnT3wRhrEBE5J1/fdpDO5hQ3XrIs7lLOWpzBfzun6eYREalluUKJh3Yc5pbLVtDSlIy7nLMWS/CbWTvwi8DX4mhfRGQ+vvfcEBPTRW6tw24eiGE4J4C7Z4H6Ob9ZRGSGrz91kGWdzbxhY33GWNyjekRE6sroZIEfPJ/h165YRTJRPydtzaTgFxE5C9/ecYh8qcytV66Ku5RzpuAXEZkjd+e+rQNsWNrO687rjrucc6bgFxGZowe2vcyWl47xnjeur6u5eU6m4BcRmYOjE9N88hs7uWrtEt79+nVxlzMvCn4RkTn4w28+w8R0kT/+jcvr9qBuhYJfRGQW33tukAe2vcwHbriAi5Z3xl3OvCn4RUTOYGK6yO/dv4MLl3XwgRs3xl3OgojlBK5qOTyaYzibp+wOgDvkS2VyhRKT+RJThRIJg3QyQVMqQTqZoFR2iuUyhZJTLjtmRjJhJMNNZL7o5EtlCsUyyYTR3pyivTlJR3OKVCJBIgEJMwwolp1S2SmUypTKYQ1hbQmD5lSSlqYEzakkiYRRKJbJl8rki2WOTeYZHJtmcCxHZnya7HSRXDGoPV8skzBIJoyEGU3JBK3pJG3pJK3pJOlkgrI77kF7TckELU0JWpuStDQlaUomaEoaqUQi/GzB5zML3q9cdsrulB0MaEolaEoYTalE8NnsxGdMpxK0NAWfoyWVJJ1KHH//ej74JQLBKJ5PPLCTQ2M57vut62lO1d/0DKeyqIP/89/fxT8+tj/uMuatszlFe3MqCNemJM2pBGUPpoUte7AhyuVLTIYbtEKpfDyYzaBQ8lnbiEJT0giqCJhBKmEkEkYqYSQT4QYoaTQlEic2KOFLiiVnOtwYlstOOpWgORVsKJtSRjLcKKcSCRynUDqxoXUn3HgFG8CSB4+Vy45D+Do7vvGsbOjK7iQtrCkZbMSCHQMLfiaDjWUiEbSfMEiE75G0YHllo9qUDHYMetqa6GlP09uWpqMlRWtTsIFuT6fobEmRSuqLd636zMPP89UnB/jQTRdy9bqeuMtZMIs6+G+/di0/f0E/ZoQhaKRTwZ5va7iX6kA+DJfKXnwqmTgeCuUwMCp77JXwSSeTFMtlstMlJqaLZKeLFMuVwAkCJHXSe1UCzTBK7uQKJabDvXh3P763nE4m6GlPs6yzmWWdLbSm57eX4R4E6FS+RK5YolB0CuXy8ZAsl4N6S+64exBiMwKxUAqeV/nm4gDhhidfKjNdLJErnPg2Ugi/tRTKr9zglMP1WJzxs1gqU5wR1o5TLgfPb0oF4dmcSmB24hvRdCH4WdnwBRu6BC1NdjyYgxAO/s0NZiwL7lc2BMVwY1AJ74QF/36FyucIP3euUGY8VyRfDNotefC6UuWbVfhvfvwzlSrrpjzrv09XS4qe9jRL2tL0tjXR0xb83teRZmV3Cyu6W1jV3UpfR5qO5pS+SVXJ3T/ey19+fze3X7uWD998YdzlLKhFHfyXrurm0lX1e5LFQjGzsDtmcXxNrSf5YpmRqTzHsgWGs3my00WmCiWm8iWy+SIjkwVGJvMcmyxwbDLPkYk8u4YmGJksMDFdfNX7JQw6mlN0tTbR19Ec7hw0s7yrhZXdLaxa0nr8p/69z90D2w7yh998hlsuXcEf3XbZotvYLurgF4lbOpVgWWcLyzpbzvq1U/kSh8dyHBqd4tBIcLxqLFdgPFdkdKrAkYlp9h+dZMu+YY5NFl71+q6WFMu7WljW1cyKrlbW9raxti/4ub6vnb6O5oX4iIuKu/PVJw/y0a8+zXUbevmLd11Z90M3T0XBL1KjWtNJzl/azvlL22d9bq5QYnAsx8sj4YZiNMfQWC4YIDCe48cvZhgcm37Fa3rb01y4rIOLlndy0YpOLlnRyUXLO+vuMoILZXAsx8fv/xnffXaIa9f3svk3Ny3ab00KfpFFoKUpybq+dtb1nX4jkSuUODgyxf6jk+w5kmXX4Di7hib4+raDjOdOdCut6m7hkpVdvGZlJ5esCH6u72tftAeh3Z2vPXmQT35jJ9PFMr/3q6/hvW88f1Hu6Vco+EUaREtTko39HWzs7+DGGcvdnUOjOZ4/PM5zh8d57vAYzx0a59EXMhQrgxqSCTb0t3PR8k4uXtHJBcs6uGBZB+t62+p2g1AuO9/ZeZjPfe9Fnjk0xjXre/iTd1wxp29Y9U7BL9LgzIxVS1pZtaT1FdePnS6W2D2U5bnDY7wwOMELg+NsfekYD25/+fhzTt4gXLKik439Hazuaa3ZDcLoZIHvPjvIF364m11DE2xY2s5n3nkFb7/qPBKLeC9/JgW/iJxScyrJa1d18dpVXa9Ynp0usjszwa7BCV4YGmfX4MSrNgiphLGmt431fW2s62tnbW8b6/raWNvbxvLuFjqrOCy1WCqz4+UxfvRChh++kOGpAyOUys5Fyzv47O1X8auvW7mou3VORcEvImelvTnF5auXcPnqJa9YPpYrsGtwnN2ZLPuOZNl3NMveI5M8sXeYbL70iue2NiVZ1tXM8s4WVi4JzlVY2dXCsq4WetvTx29t6eTxk+Fm21AUSmWOTEwzFJ7x/tzhcX66b5itLx1jMmz/8tXdfOCGjdxwcT9XrelpmD38kyn4RWRBdLU0cfW6Xq5e1/uK5e7O0Wyel45OMnBsksGxXBDO49MMjuZ4av8Ih0dz5EtnPtktlQhO5mtNJ2lOBWewV06AnApvPuOcQTO4eHkn77h6Ndes7+W6DX30d2oIKyj4RSRiZsbSjmaWdjSfdtqDysYhMz7NcDZ//DZVKFEslY+fPT5dLDNVKB0/670lnO+qtSlJe3Pq+LeI5V0trO1to7utMYemzkbBLyKxm7lxkOjV5mF3ERGJTCzBb2ZLzOw+M3vOzJ41szfEUYeISCOKq6vnfwMPufs7zCwNtMVUh4hIw6l68JtZN/Bm4D0A7p4H8tWuQ0SkUcXR1XM+kAG+ZGZPmdnfmNmrzpE2s7vMbIuZbclkMtWvUkRkkYoj+FPAzwF/7e5XAVngoyc/yd03u/smd9/U399f7RpFRBatOIJ/ABhw98fD+/cRbAhERKQKqh787n4YOGBmF4eLbgKeqXYdIiKNytyrfyFuM7sS+BsgDewB3uvux87w/AzwUpXKq5alwJG4i6gjWl9nR+vr7CzW9bXO3V/VVx5L8AuY2RZ33xR3HfVC6+vsaH2dnUZbXzpzV0SkwSj4RUQajII/PpvjLqDOaH2dHa2vs9NQ60t9/CIiDUZ7/CIiDUbBLyLSYBT8C8DM7jazITPbcQ6vvdrMfmZmL5rZZ23GhUXN7LfDqat3mtmfLGzV8YlqfYWP/zczczNbunAVxyuK9WVmnw7/tp42s/vNbMls71Xr5rOeTvN+d5jZrvB2x4zlZ/wbrAcK/oVxD3DLOb72r4H3AxeGt1sAzOxG4FbgCne/FPjM/MusGfewwOsLwMzWAG8B9s+zvlpzDwu/vh4BLnP3y4EXgI/Ns8ZacA/nsJ7M7Admtv6kZb3AJ4DXA9cCnzCzynUjT/s3WC8U/AvA3R8FhmcuM7ONZvaQmW01sx+Z2SUnv87MVgJd7v6YB0fZ/x64LXz4PwOfcvfpsI2haD9F9US0vgD+HPhdYFGNWIhifbn7w+5eDJ/6GLA62k8RvXNdT6fxS8Aj7j4czirwCHDLHP4G64KCPzqbgd9296uB/w781Smecx7BpHUVA+EygIuAN5nZ42b2QzO7JtJq4zev9WVmtwIH3X171IXWiPn+fc30PuDbC15hbZjLejqV84ADM+5X1t1c12lN08XWI2BmHcD1wFdmdP+d7VWkU0AvcB1wDfDPZrbBF+H42/muLzNrA/4nQTfPordAf1+V9/o4UAS+vDDV1Y4zrSczey/woXDZBcD/M7M8sNfdf73atVabgj8aCWDE3a+cudDMksDW8O6DBH2FM79irwYOhr8PAF8Lg/4JMysTTCS1GK9KM9/1tZHgAj/bw//gq4EnzezacDbYxWYh/r4ws/cAbwVuWow7FJxmPQG4+5eAL0HQxw+8x933zXjKQeCGGfdXAz8Il592ndYLdfVEwN3HgL1m9k4AC1zh7iV3vzK8/b67HwLGzOy6cGTAbwIPhG/zdeDG8PUXEcxkuhhnD5z3+nL3n7n7Mndf7+7rCTaaP7dIQ39B/r7M7BaC4yFvc/fJuD5LlE63nub48u8AbzGznvCg7luA78zyf7Z+uLtu87wB9wKHgAJB6NxJsAf6ELCd4HoDv3+a124CdgC7gc9z4mzqNPCP4WNPAr8Q9+es5fV10nP2AUvj/py1vL6AFwn6sLeFty/E/TnjWk8Ee/LrT7H8feF6epFg6vg5/w3W+k1TNoiINBh19YiINBgFv4hIg1Hwi4g0GAW/iEiDUfCLiDQYBb/ULTObqHJ7/7ZA73ODmY2a2bZwhsxZJ+Azs9vM7LUL0b6Igl8kZGZnPJPd3a9fwOZ+5MEZpVcBbzWzN87y/NsABb8sCAW/LCqnm43RzH4tnPDuKTP7rpktD5f/gZn9g5n9K/AP4f27w6l695jZB2e890T484bw8fvCPfYvh2dxYma/Ei7basFc7d88U73uPkVwAlVlsrn3m9lPzWy7mX3VzNrM7HrgbcCnw28JG+cx66SIgl8WndPNxvhj4Dp3vwr4J4LpCipeC9zs7reH9y8hmJa3Mg970ynauQr4cPjaDcAbzawF+CLwy2H7/bMVG04HcCHwaLjoa+5+jbtfATwL3Onu/0Yw985HPJiOYfcZPqfIrDRJmywas8xauRr4v+F86mlg74yXPhjueVd8y4PrIEyb2RCwnFdOxQvwhLsPhO1uA9YDE8Aed6+8973AXacp901mtp0g9P/CT8wrdJmZ/RGwBOggmDPmbD6nyKwU/LKYnHY2RuBzwJ+5+4NmdgPwBzMey5703OkZv5c49f+TuTznTH7k7m81s/OBx8zsn919G8FVpG5z9+3h7Jk3nOK1Z/qcIrNSV48sGn7m2Ri7OTF97h2nev0CeB7YYCcu4/cfZntB+O3gU8D/CBd1AofC7qV3z3jqePjYbJ9TZFYKfqlnbWY2MOP2OwRheWfYjbKT4LrFEOzhf8XMthLR9NZhd9EHgIfCdsaB0Tm89AvAm8MNxv8CHgf+FXhuxnP+CfhIeHB6I6f/nCKz0uycIgvIzDrcfSIc5fOXwC53//O46xKZSXv8Igvr/eHB3p0E3UtfjLkekVfRHr+ISIPRHr+ISINR8IuINBgFv4hIg1Hwi4g0GAW/iEiD+f+IZ3ad/n6D0QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ML2gqdsj9CH8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "outputId": "4eb96e30-09f3-4f54-89f6-4a72bf0d907a"
      },
      "source": [
        "learn_twitter = language_model_learner(twitter_lm, AWD_LSTM,drop_mult=0.5)\n",
        "learn_twitter.fit_one_cycle(cyc_len=1, max_lr=1e-3, moms=(0.8, 0.7))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>5.053822</td>\n",
              "      <td>4.717834</td>\n",
              "      <td>0.208374</td>\n",
              "      <td>09:30</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HseNJNDH9FCK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "6d05d538-83d7-4527-c973-74e00898218e"
      },
      "source": [
        "learn_twitter.unfreeze()\n",
        "learn_twitter.fit_one_cycle(cyc_len=5, max_lr=1e-3, moms=(0.8, 0.7))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>4.592715</td>\n",
              "      <td>4.344318</td>\n",
              "      <td>0.250712</td>\n",
              "      <td>10:37</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>4.383563</td>\n",
              "      <td>4.187261</td>\n",
              "      <td>0.265000</td>\n",
              "      <td>10:37</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>4.192356</td>\n",
              "      <td>4.110231</td>\n",
              "      <td>0.270790</td>\n",
              "      <td>10:38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>4.029832</td>\n",
              "      <td>4.088607</td>\n",
              "      <td>0.274656</td>\n",
              "      <td>10:38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>3.928093</td>\n",
              "      <td>4.095024</td>\n",
              "      <td>0.274406</td>\n",
              "      <td>10:37</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LTcT-MIn9JTN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Save the fine-tuned encoder (Uncomment the line given below)\n",
        "learn_twitter.save_encoder('ft_enc_kaggle_v1')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZdryKWU_9KA5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Save the Vocab to a Pickle file for later use\n",
        "pickle.dump(twitter_lm.train_ds.vocab, open(f'vocab.pkl','wb'))\n",
        "vocab_load = pickle.load(open('vocab.pkl', 'rb'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cpbs6RgNhAOa",
        "colab_type": "text"
      },
      "source": [
        "**Training Strats here**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WT4_bBNyS3b8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trainable.to_csv('tt.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IVxPMdt39MU4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "outputId": "5f173c61-afb8-4c85-f8d3-c3e61049a9d1"
      },
      "source": [
        "data_clas = TextClasDataBunch.from_csv(path, '/content/tt.csv', vocab=vocab_load, min_freq=1, bs=64)\n",
        "data_clas.save()\n",
        "learn = text_classifier_learner(data_clas,AWD_LSTM ,drop_mult=0.5)\n",
        "learn.load_encoder('ft_enc_kaggle_v1')\n",
        "learn.freeze()\n",
        "\n",
        "#Find Optimum Learning Rate\n",
        "learn.lr_find(start_lr=1e-6, end_lr=1e2)\n",
        "learn.recorder.plot()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/1 00:00<00:00]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='82' class='' max='107' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      76.64% [82/107 00:05<00:01 2.1313]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcd3nv8c+j3ZK1S5ZteZEd23FsJ3YSOfviQCHLzVpI0sAtBNL4Ui5Q2kIpl3tJuJTXbaCUtKU0hL6CmwAOLWEJZCWQxNkdO9iO7XiLl1jeJFuypdE+muf+MSNHGG22dGaONN/3K/PSLGfOeX4aR8/8fr9znp+5OyIikr4yUh2AiIiklhKBiEiaUyIQEUlzSgQiImlOiUBEJM0pEYiIpLmsoHZsZg8A1wL17r6on9c/D3y4TxxnAJXu3jjYfisqKrympmaUoxURGd/Wrl172N0r+3vNgrqOwMwuAyLAg/0lghO2vQ74S3d/z1D7ra2t9TVr1oxSlCIi6cHM1rp7bX+vBTY05O6rgEG/3fdxG7AyqFhERGRgKZ8jMLN84CrgkVTHIiKSjlKeCIDrgJcGmxsws+VmtsbM1jQ0NCQxNBGR8S8MieBPGGJYyN3vd/dad6+trOx3rkNERE5RShOBmRUDlwO/SGUcIiLpLMjTR1cCy4AKM6sD7gKyAdz9vsRmNwFPu3trUHGIiMjgAksE7n7bMLZZAawIKgYRERlaGOYIRERkCPc+s40XtgdzsowSgYhIyLk7//LbHby2c7iXZp0cJQIRkZBr7+6hJ+ZMzAtmNF+JQEQk5CIdUQAm5ioRiIikpZbOeCIoVI9ARCQ99fYIlAhERNJUpLN3aCg7kP0rEYiIhFxLRzegOQIRkbTVoqEhEZH0FtFksYhIeuudLC7Q0JCISHqKdEbJy84gOzOYP9lKBCIiIdfSGQ3sjCFQIhARCb2Wjmhg8wOgRCAiEnqRju7ATh0FJQIRkdCLdKpHICKS1lo6ouoRiIiks0hnNLAS1KBEICISei0dUQrVIxARSU/urh6BiEg66+iOxVcn03UEIiLpqaUzXnlUZw2JiKSpoBelASUCEZFQawl4vWJQIhARCbV3VydTIhARSUvHewQaGhIRSU+9PYKivDF41pCZPWBm9Wa2cZBtlpnZOjPbZGbPBxWLiMhYFQl4vWIItkewArhqoBfNrAT4DnC9uy8Ebg4wFhGRMam3RxDU6mQQYCJw91VA4yCbfAj4qbu/k9i+PqhYRETGqpaOKLlZGeRkBfe9PZVzBPOAUjN7zszWmtlHUhiLiEgotQRcghog2L0PfexzgfcCE4BXzOxVd9924oZmthxYDjBjxoykBikikkqRgEtQQ2p7BHXAU+7e6u6HgVXA4v42dPf73b3W3WsrKyuTGqSISCrFF6UJ7owhSG0i+AVwiZllmVk+cD7wVgrjEREJnWT0CALbu5mtBJYBFWZWB9wFZAO4+33u/paZPQlsAGLAv7v7gKeaioiko+aObqaX5Qd6jMASgbvfNoxtvgF8I6gYRETGukhnsIvSgK4sFhEJtaAXpQElAhGR0HL3cX/WkIiIDKIzGiMa83F91pCIiAwiGZVHQYlARCS0WhIF5zRZLCKSppKxKA0oEYiIhFZEQ0MiIumtpTP4hetBiUBEJLR6ewSFuTprSEQkLfVOFmtoSEQkTb27OllmoMdRIhARCamWzig5WRnkZikRiIikpUhH8AXnQIlARCS0IklYphKUCEREQivSEXzlUVAiEBEJrZYkVB4FJQIRkdBq6YwyMeBrCECJQEQktCKd3ZojEBFJZ5EOTRaLiKQtd48vU6k5AhGR9NQZjdHd4zprSEQkXbUcLzinRCAikpaOL0qjHoGISHo6viiNTh8VEUlPLZ2J9YrVIxARSU8tHclZrxiUCEREQun46mRjuUdgZg+YWb2ZbRzg9WVmdszM1iVuXw4qFhGRseb4ZHESegRBHmEF8G3gwUG2ecHdrw0wBhGRMWlcnDXk7quAxqD2LyIynrV0RMnJDH51Mkj9HMGFZrbezJ4ws4UpjkVEJDSSVXAOgh0aGsobwEx3j5jZNcDPgbn9bWhmy4HlADNmzEhehCIiKdKSpEVpIIU9AndvdvdI4v7jQLaZVQyw7f3uXuvutZWVlUmNU0QkFSJJWpQGUpgIzGyymVni/nmJWI6kKh4RkTBpSVLlUQhwaMjMVgLLgAozqwPuArIB3P0+4IPAn5tZFGgH/sTdPah4RETGkkhHlKkleUk5VmCJwN1vG+L1bxM/vVRERE4Q6YxSmBd8nSFI/VlDIiLSj5aO7vE/RyAiIv07vjrZeD9rSERE+nd8dTL1CERE0lNveYlkXVCmRCAiEjKRJJagBiUCEZHQebdHoLOGRETSUnNHfHUy9QhERNJUMhelASUCEZHQSeaiNKBEICISOsfXK1aPQEQkPa3e3UhZQQ6l+TlJOZ4SgYhIiEQ6ozyz+RDXnjWFzAxLyjGVCEREQuSpjQfpjMa4YcnUpB1TiUBEJER+sX4/00oncM6M0qQdU4lARCQkGlo6eWnHYW5YMpXEul1JoUQgIhISj23YT0/MuWFJdVKPq0QgIhISv1i/nzOmFDGvqjCpx1UiEBEJgT1HWvndO0eTOkncS4lARCQEHl23H4DrFoc0EZhZgZllJO7PM7PrzSw5ZfFERMY5d+fn6/Zx3qwyqksmJP34w+0RrALyzKwaeBr4U2BFUEGJiKSTTfubebuhNSXDQjD8RGDu3gb8MfAdd78ZWBhcWCIi6ePR9fvJyjCuWTQlJccfdiIwswuBDwOPJZ7LDCYkEZH0setwKz9ZW8fl8yopLUhObaETDbe03WeBLwI/c/dNZjYbeDa4sERExr+dDRFu+96rGPC3V89PWRzDSgTu/jzwPEBi0viwu38myMBERMaztxsi3Hb/q/TEnB/deQFzk3ztQF/DPWvoR2ZWZGYFwEZgs5l9PtjQRETGpx318SQQc2fl8gs4fXLqkgAMf45ggbs3AzcCTwCziJ85JCIiJ2Hf0XZu+96rxBxW3nlB0q8i7s9wE0F24rqBG4FH3b0b8ODCEhEZn3696SANLZ2s+NjSlA4H9TXcRPBdYDdQAKwys5lA82BvMLMHzKzezDYOsd1SM4ua2QeHGYuIyJjV2NaNGZwxpSjVoRw3rETg7v/s7tXufo3H7QGuGOJtK4CrBtvAzDKBe4hfpCYiMu41tXZRPCE7aauPDcdwJ4uLzewfzWxN4vZN4r2DAbn7KqBxiF1/GngEqB9WtCIiY1xjWxdlSVqLeLiGOzT0ANAC3JK4NQPfH8mBE+UqbgL+bRjbLu9NQg0NDSM5rIhISh1t66IkP1yl2oabCE5z97vcfWfi9hVg9giPfS/wBXePDbWhu9/v7rXuXltZWTnCw4qIpE5jazdlKbqCeCDDvbK43cwucfcXAczsYqB9hMeuBR5OLMdWAVxjZlF3//kI9ysiElpNrV0smhqeiWIYfiL4BPCgmRUnHjcBHx3Jgd19Vu99M1sB/EpJQETGM3ePzxGMxR6Bu68HFptZUeJxs5l9Ftgw0HvMbCWwDKgwszrgLiA78f77Rhi3iMiY097dQ1c0RknIJouH2yMA4gmgz8O/Ij7OP9C2t53Efm8/mThERMaixtYuAMoKxuZkcX/CcxKsiMgY0NTaDUBpyHoEI0kEKjEhInISmtriPYJUrTswkEGHhsyshf7/4BuQ/IU1RUTGsOOJIGQ9gkETgbuHoyKSiMg48O4cQbgSwUiGhkRE5CQ0JQrOFU8YP5PFIiJyEsJYcA6UCEREkiaMBedAiUBEJGmaWrtCd8YQKBGIiCRNU1s3pSGrPApKBCIiSdPU2hW6U0dBiUBEJCnCWnAOlAhERJKit+Cc5ghERNJU78VkmiMQEUlTYS04B0oEIiJJ0dgWzvISoEQgIpIURxOJIGyL0oASgYhIUoS14BwoEYiIJEVTa1coC86BEoGISFI0tXWHsuAcKBGIiCRFWAvOgRKBiEhShLXgHCgRiIgkRbzgnBKBiEjaihecC99EMSgRiIgELswF50CJQEQkcG1d4S04B0oEIiKBa2oLb8E5UCIQEQlcmAvOQYCJwMweMLN6M9s4wOs3mNkGM1tnZmvM7JKgYhERSaUwF5yDYHsEK4CrBnn9N8Bid18CfBz49wBjERFJmd6Cc2k3R+Duq4DGQV6PuLsnHhYAPtC2IiJj2buL0qRZIhgOM7vJzLYAjxHvFYiIjDthLjgHKU4E7v4zd58P3Ah8daDtzGx5Yh5hTUNDQ/ICFBEZBU1t3ZSEtOAchOSsocQw0mwzqxjg9fvdvdbdaysrK5McnYjIyDS2dYV2WAhSmAjMbI6ZWeL+OUAucCRV8YiIBCXMBecAsoLasZmtBJYBFWZWB9wFZAO4+33AB4CPmFk30A7c2mfyWERk3Ghs7WJaaX6qwxhQYInA3W8b4vV7gHuCOr6ISFgcbevmrGnhnCiGkMwRiIiMV70F5zRHICKSpsJecA6UCEREAtVbcC6sy1SCEoGISKB6C86VhLTyKCgRiIgEKuwF50CJQEQkUE2t4S44B0oEIiKBendRGiUCEZG0FPaCc5BGieBYezeb9zcT7YmlOhQRSSONbV2hLjgHAV5ZHDYvbG/gUz/6HXnZGSycWsxZ04pZPK2EoglZtHREj99i7pwxpZCzppVQMTE31WGLyBjX1NYd6vkBSKNEcN6sMv7pT5awfu8xNtQdZeXqd/j+S7sHfU91yQQWTy/mYxfPYmlNWXICFZFxpak13FcVQxolgkmFedywpJobllQDEO2JsaMhQkd3jMK8LApzsyjMy6bHnU37jrGh7hjr647y2q5GfrulnhUfO48LZpenuBUiMpbEYk5DSyczywtSHcqg0iYRnCgrM4P5k4v6fe382eWcn/ijfyTSya33v8odK17nwTvO59yZpckMU0TGIHfn+W0N/MPTW9leH+HqM6ekOqRB2Vir/FxbW+tr1qxJ6jHrmzu45buvcKS1i5V3XsCi6uJT2k9Hdw87G1rZXt/C9kMRmtq6qC6dwPTSfKaVTmBqyQQOHutg68EWth5qYevBFrp7YrxvQRXXnDmFqSUTRrllIjLa1uxu5OtPbWX1rkamlU7gr943jxuWVKd8stjM1rp7bb+vKREMz76j7dxy3yu0dkV5ePkFf9Cb6Ik5bx1o5pW3j/DKziNsO9RC319tzJ1DzR3EEs9lZhiFeVkcbevu93i5WRnMqyqkuyfGloMtAJw9o4SrF02mND+Hrp4YXdEYndEYOZkZTC3JY0rxBKaU5FFRkEtGiM9QEBmvfr35EHc+uIbKwlw+85453Lp0BjlZ4Tg5U4lglLxzpI1bvvsKjW1dlOXnkJ+bSX5OJnlZmWw71EJzRxSA2RUFnDmtmKyM3/8HMLUkj7lVhcyrmsisigJyszJp7Yyyt6mNvY3tHDjWTuXEXE6fXMjM8oLj3yB2NkR4YuNBHttwgM0HmoeMMyczgykleVSXTIjfSidQXpADZhiQYUaGQU5WBrlZmYmfGZhBdyLBdPU40Z7Y8W3ysuM/KwtzmVGWn/JvNyJhdMt3X+HAsXae+uxl5OeEa+RdiWAU7TnSykOv7KG5o5u2rp7ELcrMsgIuPK2cC2aXM7k4L7DjH2ruoCsa/wOdk5lBTlYGndEY+4+2c+BYBweOtbPvaDv7mt79Wd/SOaox5GZlMLdqIvOqCpkzaSJl+TkUTcimeEI2RXnZVBXnUjkxl8RKpCJpYevBFq68dxVfvHo+/+Py01Idzh8YLBGEK2WNATPLC/jf1y5I2fGriv4wyRTkxgtaDTR30Rntobk9iuMk/qMn5nT3xIeWOrtjdEZ7iDl9EoyRlZFxfJuO7h46umPsP9bOtsQcxks7DvPTN/b1e8zC3CxmVRYwq6KA6aV/2IOYVjqBpTVlzCzPV8KQceEHr+4hJyuDm2unpzqUk6ZEkAbiQzqZgey7tTPKsfZumju6OdbWzbH2bvYfbWfX4VZ2Hm5lze4mHl2/n4E6npWFuZxXU8a5M0tZVF3MGVMKKcwL76X4Iv2JdEb56Rt1XHfW1FBXGR2IEoGMSEFuFgW5WUxl4DOa3P33vvXHYs7bDRFW727k9V2NvL67icfePHD89Znl+SycWsSVCyfz386cQlZmOCbbRAbyszfqaO3q4U8vnJnqUE6J5ggkFA41d7Bp/zE27Wtm84Fm1u89yv5jHcwsz+eTy07jprOnhebsC5G+3J0r711FblYmj37q4tAOdWqOQEKvqiiPqqI83jO/Coj3Gp7efIh/fXYHX3jkTf7pme3cedls/vjsaRSHeKUnST+rdzWy7VCEr3/grNAmgaEoEUgoZWQYVy2azJULq3h+WwPf/u0OvvLLzfy/J7bw/gVVfPDcaVw6t1KnsUrKPfTqHoonZHPd4qmpDuWUKRFIqJkZy06fxOXzKtm0v5mfrK3j5+v28asNB5hUmEtNRQGxmNPjfvxntCd+RlR3j9MTc2ZXFnD+rDKW1pSxeHoJednBTJxL+qlv6eDJjQe5/aIaJuSM3X9XSgQyJpgZi6qLWVRdzBevmc+zW+r5xbr9NLZ2kZOVQWaGkWFGZoaRlWFkZ2aQlRm/gG7LwRb+4eltQPxiuwVTi5hdUcDM8gJqKvKZWV7A3EkTKcgd+H+HSGf8YsG8rAxNXstxP169l2jM+fAFY3OSuJcSgYw5uVmZXLVoClctGn4hr6NtXazZ3cTq3Y28WXeMV3ce4ae/e/caCDOoKS9gwZQiFkwtomhCNm/XR47Xhep7UV52ppGXlcnk4jwun1fJFfMnsbSmTJPZaebpTQf5znNvc/m8SmZVhLu66FB01pCkrY7uHvY2trHzcCtbD7aweX/8jKV3GtsAyM/JZO6kicyZVMhpkwrIyjDau2J0RHvo6O5hR32E13Y20tUToyAnk4vnVHDj2dX80RlVSgrjmLvzwEu7+bvHNnNmdTH//tFaJhUGV01gtKTkrCEzewC4Fqh390X9vP5h4AuAAS3An7v7+qDiETlRXnYmc6sKmVtVyJULJx9/vrmjm5aOKFOK8oYs3tfaGeWVt4/w7NZ6fvNWPU9vPkRZQQ43nV3NrUunM6+qMOhmSBJFe2J89Veb+Y9X9nDlwiruvfXsMT030CuwHoGZXQZEgAcHSAQXAW+5e5OZXQ3c7e7nD7Vf9QgkrHpizgvbG/jPNXv59eZDdPc48ycXcsHs8vhk9ayyYS1/2tHdw/ZDEd46EO+hbDnYTHZmBounlbBkeglLZmgZ1VTo6O7hkz98g99uqefOS2fxt1efMabOWktZ0TkzqwF+1V8iOGG7UmCju1cPtU8lAhkLjkQ6+dnv9vHs1nrW7mmiozsGwKyKeP2lqSV5TE1Uh+3ucXbUR9hRH+HthgjvNLbRk6hXPiE7k9MnF9IVjbH1UMvx56eXTeCSOZUsO72Si04rV1mOJPjJ2jo+91/rufu6Bdx+8axUh3PSxsIFZXcAT6Q6CJHRUj4xlz+7dDZ/dulsuqIxNu4/xuu7Glm7p4m6pnZ+904TTX3WosjONGrKC5g/uZBrz5rC/MlFnDHl98uRt3VF2bivmXV7m1izu4lfrt/PytXvkJVh1NaUsuz0SVxx+iTmVU0csxc2hdmL2xuomJjDRy6sSXUooy7lPQIzuwL4DnCJux8ZYJvlwHKAGTNmnLtnz57RD1Ykydq6ouw/2o6ZMbMs/6RPS+2KxnjjnSae29rAc1vrjy9gNLU4j8sT116cN6tsTBZBCxt3Z+nXfsNFp5Xzz7ednepwTkloh4bM7CzgZ8DV7r5tOPvU0JBI/w4e6+D5bfU8u6WBF3ccPn7tw+zKAmpnllI7s4xza0qZXVGgHsNJ6l1r4OsfOItblo69MtMQ0qEhM5sB/BT40+EmAREZ2OTiPG5dOoNbl86gKxpj3d6jrNnTyNrdTTy9+RD/uaYOiK9dcc6MUmprSjlvVhlLppVoadMhvLjjMAAXz61IcSTBCPL00ZXAMqDCzOqAu4BsAHe/D/gyUA58J/HtJDpQthKRk5OTlcF5s8o4b1YZEC/it/NwhLV7mnh9dxNr9zTxzFuHgPhQ0rWLp3L94qksnFqk3kI/Xt5xmFkVBVSXDFxufSzTBWUiaaqhpZMXdzTwy/UHWLWtgWiiLtP1iaQwu3JiqkMMhe6eGEu+8jQ3nVPN3914ZqrDOWWhHBoSkdSqLMzlprOncdPZ02hq7eKJjQd5dP0+/uk327n3me2cWV3M9Yuncu3iKUwpHp/fhIdj/d6jtHb1cMmc8TksBEoEIgKUFuTwofNn8KHzZ3DwWAe/2rCfR9fv52uPv8XXHn+LhVOLuOL0SVwxfxJLppeMqQupRurFHYcxgwtnj99EoKEhERnQzoYIT246yLNb4hfGxRxK87O5dG4ll8+r5NJ5FWOizs5I3Hzfy3RFY/ziU5ekOpQR0dCQiJyS2ZUT+eSyOXxy2RyOtnWxavthnt1SzwvbG3h0/X4AzphSxHvnT+KW2unMKM9PccSjK9IZ5XfvHGX5ZbNTHUqglAhEZFhK8nOOTyTHYs7mA808v62BVdsa+Lfn3+bbz+7g0rkVfPj8Gbz3jCqyx8G6Dat3HSEa83E9PwBKBCJyCjIy3l0o6H9eMYeDxzr48et7efj1d/jED95gUmEuty6dzq1LpzOtdOz2El7cfoTcrAzOmVma6lACpTkCERk10Z4Yz25t4Iev7eH5bQ0ALJtXyYfOn8kVp1eOudXdrvzWKiYV5fLQHUMWRg49zRGISFJkZWbwvgVVvG9BFXVNbfz49b38+PW93PngGmaU5XPPB87iwtPKUx3msNS3dLD1UAs3nTNkUeQxb2ylZxEZM6aV5vPX7z+dl/72Pdz3388hw+C2773K3Y9uor2rJ9XhDenlHfEamON9fgDUIxCRgGVnZnDVoilcNq+Srz+5lRUv7+a5rfV885bFnDuzLNXhAbBp/zE+918biMWc4vxsSvOz2XW4lZL8bBZMKUp1eIFTIhCRpMjPyeLu6xdy5cLJfP4n6/ngfa9Qlp9DdmYGOVkZZGcaE/OyqS7JozqxaM/0snwunlNBXnZwy0F2dPfwFw+v42hbN+fMKOFoWze7DrfS1NbNrbXT06IgnxKBiCTVhaeV8+RnL+P7L+7iUEsH3VGnuydGZ0+M5vZuthxs4Tdv1dMZja/qNr1sAl+6ZgFXLqwKpCDe3z+xhR31ER664zwunVs56vsfC5QIRCTpJuZm8en3zh3wdXfnSGsXG+qOcs8TW/nED9Zy8ZxyvnztQk6fXDhqcaza1sCKl3dz+0U1aZsEQKePikjIRXti/Gj1O3zz6W1EOqPcunQ6yy+dTU1FwYj229TaxZX3rqJ4Qja//PQlgQ4/hYFOHxWRMSsrM4OPXFjDdWdN5VvPbOPh1XtZufod3r+gij+7dDa1M0tPesjI3fnSz9+kqa2LB25fOu6TwFCUCERkTCgtyOH/3rCIT10xhwdf2cMPXtvDU5sOsXhaMbU1Zcwsz2d6WT4zy+I/Bypx0d7Vw3+8spvH3zzI31x1Oouqi5PbkBDS0JCIjEltXVEeWVvHw6/v5e2GCB3dseOv5edkcuHsci6dW8Fl8yqpKS/g9d2NPPJGHY+/eZBIZ5RL5lTwHx8/L21Kaqds8fogKBGIyIncnYaWTvY0trHnSBvr9x5l1fYG9hxpA6AgJ5PWrh7yczK5etEUPnBuNRfMKk+LU0N7aY5ARMY1M2NSUR6TivJYWlPGB8+dBsCeI628sP0wb9Yd4/zZZVy1aDL5OfqzdyL9RkRk3JpZXsDM8pGdXZQOVGtIRCTNKRGIiKQ5JQIRkTSnRCAikuaUCERE0pwSgYhImlMiEBFJc0oEIiJpbsyVmDCzBmBPn6eKgWP9bHri84M9Huh+BXB4BOEOFNvJbKf2Df04Hds30rYNFtvJbKf2Df04LO2b6e79L7rg7mP6Btw/nOcHezzI/TVBxHYy26l9al9/90faNrVP7et7Gw9DQ78c5vODPR7o/kgNd1+Dbaf2Df1Y7Ts1at/Q24339gFjcGgomcxsjQ9QrW88UPvGrvHcNlD7km089AiCdH+qAwiY2jd2jee2gdqXVOoRiIikOfUIRETSXFokAjN7wMzqzWzjKbz3XDN708x2mNk/W2KVbDO728z2mdm6xO2a0Y982DGOevv6vP7XZuZmVjF6EZ90jEF8fl81sw2Jz+5pM5s6+pEPO8Yg2vcNM9uSaOPPzKxk9CMfdoxBtO9mM9tkZjEzS8lY+0jaNcD+Pmpm2xO3j/Z5ftD/R0fFSE9hGgs34DLgHGDjKbx3NXABYMATwNWJ5+8GPpfqtgXVvsRr04GniF+3UTGe2gcU9dnmM8B946x97weyEvfvAe4ZZ+07AzgdeA6oHUvtSsRcc8JzZcDOxM/SxP3SwX4Ho3lLix6Bu68CGvs+Z2anmdmTZrbWzF4ws/knvs/MphD/g/Gqxz+RB4EbkxP18AXYvm8BfwOkdCIpiPa5e3OfTQtIYRsDat/T7h5NbPoqMC3YVgwsoPa95e5bkxH/QE61XQO4Evi1uze6exPwa+CqZP0NSuelKu8HPuHu283sfOA7wHtO2KYaqOvzuC7xXK9PmdlHgDXAXyc+wLAYUfvM7AZgn7uvD6InOgpG/PmZ2deAjxC/6vKKYMM9aaPx77PXx4EfBxLlqRvN9oXJcNrVn2pgb5/HvW1Nyu8gLROBmU0ELgL+q88fudyT3M2/AV8l/k3yq8A3if8Pl3IjbZ+Z5QP/i/jwQuiM0ueHu38J+JKZfRH4FHDXqAU5AqPVvsS+vgREgR+OTnQjN5rtC5PB2mVmHwP+IvHcHOBxM+sCdrn7TcmO9URpmQiIT5IfdfclfZ80s0xgbeLho8T/2PftUk8D9gG4+6E+7/se8KsgAz5JI23facAsoLc3MA14w8zOc/eDAcc+HCP+/E7wQ+BxQpIIGKX2mdntwLXAexPDCmEx2p9fWPTbLgB3/z7wfQAzew643d1399lkH7Csz+NpxOcS9pGM30EqJllScQNq6DOpA7wM3Jy4b8DiAd534kTNNYnnp/TZ5i+Bh8dT+07YZjcpnCwO6POb27SZrXoAAANySURBVGebTwM/GWftuwrYDFSmsl1B//skhZPFp9ouBp4s3kV8org0cb9sOL+DUWlHqv+BJOnDWgkcALqJj7HdQfwb75PA+sT/MF8e4L21wEbgbeDbvHsR3kPAm8AG4t9epiSrPclo3wnb7Ca1Zw0F8fk9knh+A/FaLNXjrH07iI85r0vcUnlWVBDtuymxr07gEPDUWGkX/SSCxPMfT3xuO4CPDfU7GM2briwWEUlzaXH6qIiIDEyJQEQkzSkRiIikOSUCEZE0p0QgIpLmlAhkXDCzSJKP9/Io7WeZmR1LVEHdYmb/MIz33GhmC0bj+CKgRCDSLzMb9Kp7d79oFA/3gsevRj0buNbMLh5i+xsBJQIZNUoEMm4NVAnSzK4zs9fM7Hdm9oyZVSWev9vMHjKzl4CHEo8fMLPnzGynmX2mz74jiZ/LEq//JPGN/od9auZfk3hubaKO/KBlSNy9nfjFX72F/+40s9fNbL2ZPWJm+WZ2EXA98I1EL+K0EVS8FAGUCGR8ux/4tLufC3yOeCVIgBeBC9z9bOBh4qW2ey0A/sjdb0s8nk+8RPB5wF1mlt3Pcc4GPpt472zgYjPLA75LvHb8uUDlUMGaWSkwF1iVeOqn7r7U3RcDbwF3uPvLxK9k/7y7L3H3twdpp8iwpGvRORnnhqhwOQ34caLWew7xui69Hk18M+/1mLt3Ap1mVg9U8ftlgQFWu3td4rjriNefiQA73b133yuB5QOEe6mZrSeeBO71dwv7LTKzvwNKgInEFwk6mXaKDIsSgYxXA1aCBP4F+Ed3f9TMlhFfba5X6wnbdva530P//88MZ5vBvODu15rZLOBVM/tPd18HrABu9PiaELfz+9Upew3WTpFh0dCQjEseX4Fsl5ndDGBxixMvF/NuKd+P9vf+UbAVmG1mNYnHtw71hkTv4e+BLySeKgQOJIajPtxn05bEa0O1U2RYlAhkvMg3s7o+t78i/sfzjsSwyybghsS2dxMfSlkLHA4imMTw0ieBJxPHaSG+EtpQ7gMuSySQ/wO8BrwEbOmzzcPA5xOT3acxcDtFhkXVR0UCYmYT3T2SOIvoX4Ht7v6tVMclciL1CESCc2di8ngT8eGo76Y4HpF+qUcgIpLm1CMQEUlzSgQiImlOiUBEJM0pEYiIpDklAhGRNKdEICKS5v4/dC6/bqNGQcUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b_46Jv3vTGqk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 611
        },
        "outputId": "1eb2165a-ccfb-46e0-d400-f637940d8d31"
      },
      "source": [
        "#Train the Classifier by Gradually Unfreezing the Layers\n",
        "learn.fit_one_cycle(cyc_len=1, max_lr=1e-3, moms=(0.8, 0.7))\n",
        "    #Unfreeze Layer\n",
        "learn.freeze_to(-2)\n",
        "learn.fit_one_cycle(3, slice(1e-4,1e-2), moms=(0.8,0.7))\n",
        "    #Unfreeze Layer\n",
        "learn.freeze_to(-3)\n",
        "learn.fit_one_cycle(3, slice(1e-5,5e-3), moms=(0.8,0.7))\n",
        "    #Unfreeze All Layers\n",
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(3, slice(1e-5,1e-3), moms=(0.8,0.7))\n",
        "    # get predictions\n",
        "preds, targets = learn.get_preds()\n",
        "predictions = np.argmax(preds, axis=1)\n",
        "pd.crosstab(predictions, targets)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.108155</td>\n",
              "      <td>1.081742</td>\n",
              "      <td>0.432636</td>\n",
              "      <td>00:08</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.925905</td>\n",
              "      <td>0.871612</td>\n",
              "      <td>0.606272</td>\n",
              "      <td>00:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.869928</td>\n",
              "      <td>0.852567</td>\n",
              "      <td>0.606852</td>\n",
              "      <td>00:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.838431</td>\n",
              "      <td>0.851427</td>\n",
              "      <td>0.606272</td>\n",
              "      <td>00:10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.841839</td>\n",
              "      <td>0.862219</td>\n",
              "      <td>0.607433</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.826976</td>\n",
              "      <td>0.875148</td>\n",
              "      <td>0.600465</td>\n",
              "      <td>00:16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.785392</td>\n",
              "      <td>0.886886</td>\n",
              "      <td>0.595819</td>\n",
              "      <td>00:16</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.758142</td>\n",
              "      <td>0.912891</td>\n",
              "      <td>0.582462</td>\n",
              "      <td>00:24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.752605</td>\n",
              "      <td>0.927439</td>\n",
              "      <td>0.577816</td>\n",
              "      <td>00:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.726572</td>\n",
              "      <td>0.929626</td>\n",
              "      <td>0.577816</td>\n",
              "      <td>00:23</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>935</td>\n",
              "      <td>97</td>\n",
              "      <td>496</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>21</td>\n",
              "      <td>3</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>88</td>\n",
              "      <td>12</td>\n",
              "      <td>57</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0    0   1    2\n",
              "row_0              \n",
              "0      935  97  496\n",
              "1       21   3   13\n",
              "2       88  12   57"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwZz5DgSUjaL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_orig=pd.read_csv('/content/validation.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "neMrBlM4Udp-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "stance=test_orig['Processed_tweets'].apply(lambda row: str(learn.predict(row)[0]))\n",
        "pd.DataFrame(stance).to_csv('res3.csv')\n",
        "d=pd.DataFrame(stance)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsjgJbqEWe_n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Validation Set\n",
        "target=pd.read_csv('/content/true_st.csv')['Truth']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nbvSdtxf7Ksq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "4c631ba0-96cd-4df2-f30d-4256789b1e21"
      },
      "source": [
        "# AUC CAlculation on validation SET\n",
        "from sklearn.metrics import roc_auc_score\n",
        "print(roc_auc_score(truth['Support'],d['Support']))\n",
        "#0\n",
        "print(roc_auc_score(truth['Oppose'],d['Oppose']))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5762153759647981\n",
            "0.725813602908575\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o2SODQFOWyCW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233
        },
        "outputId": "436213bb-1629-4550-e31c-3206057183c9"
      },
      "source": [
        "#predictions for submission\n",
        "stance=testatable['Processed_tweets'].apply(lambda row: str(learn.predict(row)[0]))\n",
        "\n",
        "stance.to_csv"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0       none\n",
              "1       none\n",
              "2       none\n",
              "3       none\n",
              "4       none\n",
              "        ... \n",
              "1732    none\n",
              "1733    none\n",
              "1734    none\n",
              "1735    none\n",
              "1736    none\n",
              "Name: Processed_tweets, Length: 1737, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7cGfksJtZ66i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# converting multiclass to multilabel ; \n",
        "# none maps to -> 0,0 ; Against maps to 0,1; \n",
        "# Favour maps to 1,0 for Support & Oppose Respectively\n",
        "\n",
        "support=[]\n",
        "oppose=[]\n",
        "\n",
        "for i in stance:\n",
        "  if (i=='support'):\n",
        "    support.append(1)\n",
        "    pass\n",
        "  if (i=='oppose'):\n",
        "    oppose.append(1)\n",
        "    pass\n",
        "  else:\n",
        "    support.append(0)\n",
        "    oppose.append(0)\n",
        "    pass\n",
        "  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qYnwsqLNa29I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "a53d5605-4164-4894-d3ce-b7d5bfbb01c9"
      },
      "source": [
        "#submission predictions\n",
        "pd.DataFrame(stance)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Processed_tweets</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1732</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1733</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1734</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1735</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1736</th>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1737 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     Processed_tweets\n",
              "0                none\n",
              "1                none\n",
              "2                none\n",
              "3                none\n",
              "4                none\n",
              "...               ...\n",
              "1732             none\n",
              "1733             none\n",
              "1734             none\n",
              "1735             none\n",
              "1736             none\n",
              "\n",
              "[1737 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 146
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PL0aC1J_gVFb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "f8f69028-4135-4bfa-f6d5-cf8f6fb32997"
      },
      "source": [
        "#saving the model\n",
        "!tar -zcvf kaggle_lmodel.tar.gz '/content/models'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tar: Removing leading `/' from member names\n",
            "/content/models/\n",
            "/content/models/ft_enc_kaggle_v1.pth\n",
            "/content/models/tmp.pth\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}