D2L_Seq_Models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "D2L_Seq_Models.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPNmbPH4tbZpnYEQ7nyVhVo",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rajlm10/D2L-Torch/blob/main/D2L_Seq_Models.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "-UytPVNO74rC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "76cb8776-4c47-4eb1-93f6-5e7221141302"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting d2l\n",
            "  Downloading d2l-0.17.4-py3-none-any.whl (82 kB)\n",
            "\u001b[?25l\r\u001b[K     |████                            | 10 kB 26.3 MB/s eta 0:00:01\r\u001b[K     |████████                        | 20 kB 8.4 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 30 kB 7.5 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 40 kB 3.6 MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 51 kB 3.6 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 61 kB 4.2 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 71 kB 4.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 81 kB 4.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 82 kB 584 kB/s \n",
            "\u001b[?25hRequirement already satisfied: jupyter==1.0.0 in /usr/local/lib/python3.7/dist-packages (from d2l) (1.0.0)\n",
            "Collecting pandas==1.2.4\n",
            "  Downloading pandas-1.2.4-cp37-cp37m-manylinux1_x86_64.whl (9.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.9 MB 22.3 MB/s \n",
            "\u001b[?25hCollecting d2l\n",
            "  Downloading d2l-0.17.3-py3-none-any.whl (82 kB)\n",
            "\u001b[K     |████████████████████████████████| 82 kB 618 kB/s \n",
            "\u001b[?25hCollecting numpy==1.18.5\n",
            "  Downloading numpy-1.18.5-cp37-cp37m-manylinux1_x86_64.whl (20.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 20.1 MB 1.2 MB/s \n",
            "\u001b[?25hCollecting matplotlib==3.3.3\n",
            "  Downloading matplotlib-3.3.3-cp37-cp37m-manylinux1_x86_64.whl (11.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.6 MB 19.6 MB/s \n",
            "\u001b[?25hCollecting requests==2.25.1\n",
            "  Downloading requests-2.25.1-py2.py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 4.2 MB/s \n",
            "\u001b[?25hCollecting pandas==1.2.2\n",
            "  Downloading pandas-1.2.2-cp37-cp37m-manylinux1_x86_64.whl (9.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.9 MB 26.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: ipywidgets in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l) (7.6.5)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l) (5.6.1)\n",
            "Requirement already satisfied: notebook in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l) (5.3.1)\n",
            "Requirement already satisfied: qtconsole in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l) (5.2.2)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l) (5.2.0)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.7/dist-packages (from jupyter==1.0.0->d2l) (4.10.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.3->d2l) (2.8.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.3->d2l) (0.11.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.3->d2l) (3.0.7)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.3->d2l) (7.1.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.3->d2l) (1.3.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.2.2->d2l) (2018.9)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l) (2021.10.8)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests==2.25.1->d2l) (1.24.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib==3.3.3->d2l) (1.15.0)\n",
            "Requirement already satisfied: traitlets>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l) (5.1.1)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l) (5.1.1)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l) (5.5.0)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter==1.0.0->d2l) (5.3.5)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (0.7.5)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (2.6.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (4.4.2)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (57.4.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (1.0.18)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (0.8.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (4.8.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipykernel->jupyter==1.0.0->d2l) (0.2.5)\n",
            "Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l) (0.2.0)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l) (3.5.2)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l) (5.1.3)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter==1.0.0->d2l) (1.0.2)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (4.9.2)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (4.3.3)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (21.4.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (3.10.0.2)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (0.18.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (4.11.2)\n",
            "Requirement already satisfied: importlib-resources>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (5.4.0)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources>=1.4.0->jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter==1.0.0->d2l) (3.7.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l) (2.11.3)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l) (0.13.3)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter==1.0.0->d2l) (1.8.0)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->jupyter==1.0.0->d2l) (22.3.0)\n",
            "Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook->jupyter==1.0.0->d2l) (0.7.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook->jupyter==1.0.0->d2l) (2.0.1)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l) (0.8.4)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l) (0.6.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l) (0.7.1)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l) (1.5.0)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l) (0.4)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter==1.0.0->d2l) (4.1.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter==1.0.0->d2l) (0.5.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter==1.0.0->d2l) (21.3)\n",
            "Requirement already satisfied: qtpy in /usr/local/lib/python3.7/dist-packages (from qtconsole->jupyter==1.0.0->d2l) (2.0.1)\n",
            "Installing collected packages: numpy, requests, pandas, matplotlib, d2l\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.21.5\n",
            "    Uninstalling numpy-1.21.5:\n",
            "      Successfully uninstalled numpy-1.21.5\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 1.3.5\n",
            "    Uninstalling pandas-1.3.5:\n",
            "      Successfully uninstalled pandas-1.3.5\n",
            "  Attempting uninstall: matplotlib\n",
            "    Found existing installation: matplotlib 3.2.2\n",
            "    Uninstalling matplotlib-3.2.2:\n",
            "      Successfully uninstalled matplotlib-3.2.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.8.0 requires tf-estimator-nightly==2.8.0.dev2021122109, which is not installed.\n",
            "tensorflow 2.8.0 requires numpy>=1.20, but you have numpy 1.18.5 which is incompatible.\n",
            "tables 3.7.0 requires numpy>=1.19.0, but you have numpy 1.18.5 which is incompatible.\n",
            "jaxlib 0.3.0+cuda11.cudnn805 requires numpy>=1.19, but you have numpy 1.18.5 which is incompatible.\n",
            "jax 0.3.1 requires numpy>=1.19, but you have numpy 1.18.5 which is incompatible.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.25.1 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed d2l-0.17.3 matplotlib-3.3.3 numpy-1.18.5 pandas-1.2.2 requests-2.25.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "matplotlib",
                  "mpl_toolkits",
                  "numpy"
                ]
              }
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "!pip install d2l\n",
        "#Restart runtime"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Utils "
      ],
      "metadata": {
        "id": "QgcP5OCd7_3X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import collections\n",
        "import re\n",
        "from d2l import torch as d2l\n",
        "import torch\n",
        "import math\n",
        "import random\n",
        "from torch import nn\n",
        "from torch.nn import functional as F \n"
      ],
      "metadata": {
        "id": "4jOaiehj79WQ"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "d2l.DATA_HUB['time_machine'] = (d2l.DATA_URL + 'timemachine.txt', '090b5e7e70c295757f55df93cb0a180b9691891a')\n",
        "\n",
        "def read_time_machine():\n",
        "  \"\"\"Load the time machine dataset into a list of text lines.\"\"\" \n",
        "  with open(d2l.download('time_machine'), 'r') as f:\n",
        "    lines = f.readlines()\n",
        "  return [re.sub('[^A-Za-z]+', ' ', line).strip().lower() for line in lines]\n",
        "\n",
        "lines = read_time_machine() \n",
        "print(f'# text lines: {len(lines)}') \n",
        "print(lines[0])\n",
        "print(lines[10])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5mnRy3aD8Cym",
        "outputId": "9faad38d-3321-42b1-c98a-d90d415a4c2b"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading ../data/timemachine.txt from http://d2l-data.s3-accelerate.amazonaws.com/timemachine.txt...\n",
            "# text lines: 3221\n",
            "the time machine by h g wells\n",
            "twinkled and his usually pale face was flushed and animated the\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Tokenization\n",
        "def tokenize(lines,token='word'):\n",
        "  if token=='word':\n",
        "    return [line.split() for line in lines]\n",
        "  if token=='char':\n",
        "    return [list(line) for line in lines]\n",
        "  else:\n",
        "    print('ERROR: unknown token type: ' + token)"
      ],
      "metadata": {
        "id": "InxcMOkH8FKr"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Vocab:\n",
        "  def __init__(self, tokens=None, min_freq=0, reserved_tokens=None):\n",
        "    if tokens is None:\n",
        "      tokens = []\n",
        "    if reserved_tokens is None: \n",
        "      reserved_tokens = []\n",
        "    # Sort according to frequencies\n",
        "    counter = count_corpus(tokens)\n",
        "    self._token_freqs = sorted(counter.items(), key=lambda x: x[1],reverse=True)\n",
        "\n",
        "    #Build vocab on init\n",
        "    self.idx_to_token=['<unk>']+reserved_tokens #List\n",
        "    self.token_to_idx={token:idx for idx,token in enumerate(self.idx_to_token)} #Dict\n",
        "\n",
        "    for token,freq in self._token_freqs:\n",
        "      #Don't include tokens with freq<min_freq in the vocab\n",
        "      if freq<min_freq:\n",
        "        break\n",
        "      if token not in self.idx_to_token:\n",
        "        self.idx_to_token.append(token)\n",
        "        self.token_to_idx[token]=len(self.idx_to_token)-1\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.idx_to_token)\n",
        "\n",
        "  @property\n",
        "  def unk(self):\n",
        "    # Index for the unknown token\n",
        "    return 0\n",
        "  \n",
        "  @property\n",
        "  def token_freqs(self):\n",
        "    # Counter object \n",
        "    return self._token_freqs\n",
        "  \n",
        "  def __getitem__(self,tokens):\n",
        "    if not isinstance(tokens, (list, tuple)):\n",
        "      return self.token_to_idx.get(tokens, self.unk) \n",
        "    #If tokens is a list \n",
        "    return [self.__getitem__(token) for token in tokens]\n",
        "\n",
        "  def to_tokens(self, indices):\n",
        "    if not isinstance(indices, (list, tuple)):\n",
        "      return self.idx_to_token[indices]\n",
        "    #If indices is a list\n",
        "    return [self.idx_to_token[index] for index in indices]\n",
        "\n",
        "\n",
        "def count_corpus(tokens):\n",
        "  \"\"\"Count token frequencies.\"\"\"\n",
        "  # Here `tokens` is a 1D list or 2D list\n",
        "  if len(tokens) == 0 or isinstance(tokens[0], list):\n",
        "    # Flatten a list of token lists into a 1D list\n",
        "    tokens = [token for line in tokens for token in line]\n",
        "  #If its already a 1D list,return  \n",
        "  return collections.Counter(tokens)\n"
      ],
      "metadata": {
        "id": "c3HtkOeo8Hh1"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def load_corpus_time_machine(max_tokens=-1,level='char'):\n",
        "  \"\"\"Return token indices and the vocabulary of the time machine dataset.\"\"\" \n",
        "  lines = read_time_machine()\n",
        "  tokens = tokenize(lines, level)\n",
        "  vocab = Vocab(tokens)\n",
        "  # Call getitem on a single token and append it to a list\n",
        "  corpus = [vocab[token] for line in tokens for token in line]\n",
        "  if max_tokens > 0:\n",
        "    corpus = corpus[:max_tokens] #First max_tokens tokens\n",
        "  return corpus, vocab\n"
      ],
      "metadata": {
        "id": "AA35AEJz8LhN"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "def seq_data_iter_random(corpus, batch_size, num_steps):\n",
        "  \"\"\"Generate a minibatch of subsequences using random sampling.\"\"\"\n",
        "  # Start with a random offset (inclusive of `num_steps - 1`) to partition a # sequence\n",
        "  corpus = corpus[random.randint(0, num_steps - 1):]\n",
        "  # Subtract 1 since we need to account for labels\n",
        "  num_subseqs = (len(corpus) - 1) // num_steps\n",
        "  # The starting indices for subsequences of length `num_steps` \n",
        "  initial_indices = list(range(0, num_subseqs * num_steps, num_steps))\n",
        "  # In random sampling, the subsequences from two adjacent random\n",
        "  # minibatches during iteration are not necessarily adjacent on the\n",
        "  # original sequence\n",
        "  random.shuffle(initial_indices)\n",
        "\n",
        "  def data(pos):\n",
        "  # Return a sequence of length `num_steps` starting from `pos` \n",
        "    return corpus[pos: pos + num_steps]\n",
        "  \n",
        "  num_batches = num_subseqs // batch_size\n",
        "  for i in range(0, batch_size * num_batches, batch_size):\n",
        "    # Here, `initial_indices` contains randomized starting indices for # subsequences\n",
        "    initial_indices_per_batch = initial_indices[i: i + batch_size]\n",
        "    X = [data(j) for j in initial_indices_per_batch]\n",
        "    Y = [data(j + 1) for j in initial_indices_per_batch] \n",
        "    yield torch.tensor(X), torch.tensor(Y)"
      ],
      "metadata": {
        "id": "GDVb8wX3CNHN"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def seq_data_iter_sequential(corpus, batch_size, num_steps):\n",
        "  \"\"\"Generate a minibatch of subsequences using sequential partitioning.\"\"\" \n",
        "  # Start with a random offset to partition a sequence\n",
        "  offset = random.randint(0, num_steps)\n",
        "  num_tokens = ((len(corpus) - offset - 1) // batch_size) * batch_size\n",
        "  Xs = torch.tensor(corpus[offset: offset + num_tokens])\n",
        "  Ys = torch.tensor(corpus[offset + 1: offset + 1 + num_tokens])\n",
        "  Xs, Ys = Xs.reshape(batch_size, -1), Ys.reshape(batch_size, -1) \n",
        "  num_batches = Xs.shape[1] // num_steps\n",
        "  for i in range(0, num_steps * num_batches, num_steps):\n",
        "    X = Xs[:, i: i + num_steps]\n",
        "    Y = Ys[:, i: i + num_steps] \n",
        "    yield X, Y"
      ],
      "metadata": {
        "id": "oYjCO3_bCNyZ"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SeqDataLoader:\n",
        "  \"\"\"An iterator to load sequence data.\"\"\"\n",
        "  def __init__(self, batch_size, num_steps, use_random_iter, max_tokens):\n",
        "    if use_random_iter:\n",
        "      self.data_iter_fn = seq_data_iter_random\n",
        "    else:\n",
        "      self.data_iter_fn = seq_data_iter_sequential\n",
        "    \n",
        "    self.corpus, self.vocab = load_corpus_time_machine(max_tokens) \n",
        "    self.batch_size, self.num_steps = batch_size, num_steps\n",
        "\n",
        "  def __iter__(self):\n",
        "    return self.data_iter_fn(self.corpus, self.batch_size, self.num_steps)"
      ],
      "metadata": {
        "id": "99-DMl6vCQdW"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def load_data_time_machine(batch_size, num_steps, use_random_iter=False, max_tokens=10000):\n",
        "  \"\"\"Return the iterator and the vocabulary of the time machine dataset.\"\"\"\n",
        "  data_iter = SeqDataLoader(batch_size, num_steps, use_random_iter, max_tokens)\n",
        "  return data_iter, data_iter.vocab"
      ],
      "metadata": {
        "id": "shaVo2-VCSH3"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RNNs "
      ],
      "metadata": {
        "id": "q3XKakb28N9m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(prefix, num_preds, net, vocab, device): \n",
        "  \"\"\"Generate new characters following the `prefix`.\"\"\"\n",
        "  state = net.begin_state(batch_size=1, device=device) \n",
        "  outputs = [vocab[prefix[0]]] # gives [token]\n",
        "  get_input = lambda: torch.tensor([outputs[-1]], device=device).reshape((1, 1)) \n",
        "  for y in prefix[1:]: # Warm-up period\n",
        "    _, state = net(get_input(), state)\n",
        "    outputs.append(vocab[y]) #Append input text i.e is prefix\n",
        "\n",
        "  for _ in range(num_preds): # Predict `num_preds` steps\n",
        "    y, state = net(get_input(), state) #Keep updating state and input\n",
        "    outputs.append(int(y.argmax(dim=1).reshape(1))) \n",
        "  return ''.join([vocab.idx_to_token[i] for i in outputs])"
      ],
      "metadata": {
        "id": "0zxuyVv2CVtE"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def grad_clipping(net, theta):\n",
        "  \"\"\"Clip the gradient.\"\"\"\n",
        "  if isinstance(net, nn.Module):\n",
        "    params = [p for p in net.parameters() if p.requires_grad] \n",
        "  else:\n",
        "    params = net.params\n",
        "  norm = torch.sqrt(sum(torch.sum((p.grad ** 2)) for p in params)) \n",
        "  if norm > theta:\n",
        "    for param in params: \n",
        "      param.grad[:] *= theta / norm"
      ],
      "metadata": {
        "id": "2AQsYKBRCXM-"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_epoch(net, train_iter, loss, optimizer, device, use_random_iter): \n",
        "  \"\"\"Train a net within one epoch\"\"\"\n",
        "  state, timer = None, d2l.Timer()\n",
        "  metric = d2l.Accumulator(2) # Sum of training loss, no. of tokens\n",
        "  for X, Y in train_iter:\n",
        "    if state is None or use_random_iter:\n",
        "      # Initialize `state` when either it is the first iteration or \n",
        "      # using random sampling\n",
        "      state = net.begin_state(batch_size=X.shape[0], device=device)\n",
        "    else:\n",
        "      if isinstance(net, nn.Module) and not isinstance(state, tuple):\n",
        "        # `state` is a tensor for `nn.GRU`\n",
        "        state.detach_() \n",
        "      else:\n",
        "        # `state` is a tuple of tensors for `nn.LSTM` and # for our custom scratch implementation\n",
        "        for s in state:\n",
        "          s.detach_() \n",
        "    y = Y.T.reshape(-1)\n",
        "    X, y = X.to(device), y.to(device)\n",
        "    y_hat, state = net(X, state)\n",
        "    l = loss(y_hat, y.long()).mean()\n",
        "    \n",
        "    if isinstance(optimizer, torch.optim.Optimizer):\n",
        "      optimizer.zero_grad() \n",
        "      l.backward() \n",
        "      grad_clipping(net, 1) \n",
        "      optimizer.step()\n",
        "\n",
        "    metric.add(l * y.numel(), y.numel())\n",
        "  return math.exp(metric[0] / metric[1]), metric[1] / timer.stop()"
      ],
      "metadata": {
        "id": "IpGYF6t48Pzw"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train(net, train_iter, vocab,loss, optimizer, num_epochs, device, use_random_iter=False):\n",
        "  \"\"\"Train a model\"\"\"\n",
        "  for epoch in range(num_epochs):\n",
        "    ppl, speed = train_epoch(net, train_iter, loss, optimizer, device, use_random_iter)\n",
        "    if (epoch + 1) % 50 == 0:\n",
        "      print(f'perplexity {ppl:.3f}, {speed:.1f} tokens/sec on {str(device)}') "
      ],
      "metadata": {
        "id": "C319Ke2xA34M"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size, num_steps = 32, 35\n",
        "train_iter, vocab = load_data_time_machine(batch_size, num_steps)"
      ],
      "metadata": {
        "id": "O0IB_sXe9wUH"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class RNNModel(nn.Module):\n",
        "  \"\"\"The RNN model.\"\"\"\n",
        "  def __init__(self, rnn_layer, vocab_size, **kwargs):\n",
        "    super(RNNModel, self).__init__(**kwargs)\n",
        "    self.rnn = rnn_layer #Can be a RNN,GRU or LSTM\n",
        "    self.vocab_size = vocab_size\n",
        "    self.num_hiddens = self.rnn.hidden_size\n",
        "\n",
        "    # If the RNN is bidirectional (to be introduced later), \n",
        "    # `num_directions` should be 2, else it should be 1.\n",
        "    if not self.rnn.bidirectional:\n",
        "      self.num_directions = 1\n",
        "      self.linear = nn.Linear(self.num_hiddens, self.vocab_size)\n",
        "    else:\n",
        "      self.num_directions = 2\n",
        "      self.linear = nn.Linear(self.num_hiddens * 2, self.vocab_size)\n",
        "\n",
        "  def forward(self, inputs, state):\n",
        "    X = F.one_hot(inputs.T.long(), self.vocab_size) #inputs.T=num_steps X batch_size\n",
        "    X = X.to(torch.float32)\n",
        "    Y, state = self.rnn(X, state)\n",
        "    # The fully connected layer will first change the shape of `Y` to # (`num_steps` * `batch_size`, `num_hiddens`). \n",
        "    #Its output shape after the linear layer is # (`num_steps` * `batch_size`, `vocab_size`).\n",
        "    output = self.linear(Y.reshape((-1, Y.shape[-1])))\n",
        "    return output, state\n",
        "\n",
        "  def begin_state(self, device, batch_size=1): \n",
        "    if not isinstance(self.rnn, nn.LSTM):\n",
        "    # `nn.GRU` takes a tensor as hidden state\n",
        "      return torch.zeros((self.num_directions * self.rnn.num_layers, batch_size, self.num_hiddens),device=device)\n",
        "    \n",
        "    else:\n",
        "    # `nn.LSTM` takes a tuple of hidden states return for C and H\n",
        "      ( torch.zeros((self.num_directions * self.rnn.num_layers, batch_size, self.num_hiddens), device=device),\n",
        "       torch.zeros((self.num_directions * self.rnn.num_layers, batch_size, self.num_hiddens), device=device) )\n"
      ],
      "metadata": {
        "id": "4oavyOCB-Ajz"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = d2l.try_gpu()\n",
        "num_hiddens=256\n",
        "net = RNNModel(nn.RNN(len(vocab), num_hiddens), vocab_size=len(vocab))\n",
        "net = net.to(device)\n",
        "predict('time traveller', 10, net, vocab, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "jDAvSJhkABHc",
        "outputId": "f30e231a-996b-4e48-dbca-69e8b8f127e7"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travellerbbbbpppppp'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "num_epochs, lr = 500, 1\n",
        "\n",
        "loss=nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.SGD(net.parameters(), lr)\n",
        "train(net, train_iter, vocab,loss,optimizer, num_epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lzud5PTQAihp",
        "outputId": "a8a92ac9-1d3c-405b-d862-ceaff9dbe805"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "perplexity 7.522, 112556.6 tokens/sec on cuda:0\n",
            "perplexity 3.596, 105103.9 tokens/sec on cuda:0\n",
            "perplexity 2.056, 107354.2 tokens/sec on cuda:0\n",
            "perplexity 1.592, 109778.3 tokens/sec on cuda:0\n",
            "perplexity 1.518, 107961.8 tokens/sec on cuda:0\n",
            "perplexity 1.445, 107909.1 tokens/sec on cuda:0\n",
            "perplexity 1.324, 106752.0 tokens/sec on cuda:0\n",
            "perplexity 1.418, 107126.3 tokens/sec on cuda:0\n",
            "perplexity 1.346, 107646.7 tokens/sec on cuda:0\n",
            "perplexity 1.299, 107497.7 tokens/sec on cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict('time traveller', 50, net, vocab, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "GVCu4UbWERew",
        "outputId": "6263b33d-eb12-4fa1-b8fe-b408b17d4d0a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travellerit would be remarkably convenient for the time som'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# GRU"
      ],
      "metadata": {
        "id": "GASZ5XPdErXa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gru_layer = nn.GRU(len(vocab), num_hiddens)\n",
        "model = RNNModel(gru_layer, len(vocab))\n",
        "model = model.to(device)\n",
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "koq2FD7OEswF",
        "outputId": "a061c3bf-402d-47f7-aaad-b1c875c2bd8a"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travellerxxowwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loss=nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr)\n",
        "train(model, train_iter, vocab,loss,optimizer, num_epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KRHc7H8wFHll",
        "outputId": "73ce727e-cbb8-4afd-bbd4-cc9468d90cc2"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "perplexity 9.854, 124438.6 tokens/sec on cuda:0\n",
            "perplexity 7.817, 121533.1 tokens/sec on cuda:0\n",
            "perplexity 6.028, 120107.3 tokens/sec on cuda:0\n",
            "perplexity 3.765, 122578.9 tokens/sec on cuda:0\n",
            "perplexity 1.568, 120465.7 tokens/sec on cuda:0\n",
            "perplexity 1.105, 122215.3 tokens/sec on cuda:0\n",
            "perplexity 1.064, 117252.2 tokens/sec on cuda:0\n",
            "perplexity 1.051, 121475.4 tokens/sec on cuda:0\n",
            "perplexity 1.055, 121212.1 tokens/sec on cuda:0\n",
            "perplexity 1.044, 120371.6 tokens/sec on cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "0YUnKyWHFwY2",
        "outputId": "c4660e77-851d-4c3c-f295-067785f61a04"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time traveller for so it will be convenient to speak of himwas e'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# LSTM"
      ],
      "metadata": {
        "id": "uwZALi3CFyeB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "lstm_layer = nn.LSTM(len(vocab), num_hiddens)\n",
        "model = RNNModel(lstm_layer, len(vocab))\n",
        "model = model.to(device)\n",
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "l1fH8BfTF0Ij",
        "outputId": "1a8d1f09-24f7-4c50-f38f-ad88f64ca86d"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travellerbbeeeneneeneeneeneeneeneeneeneeneeneeneeneeneeneen'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loss=nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr)\n",
        "train(model, train_iter, vocab,loss,optimizer, num_epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ETfAG5ptGJyL",
        "outputId": "18699d29-9db9-42af-d6b0-17c942c0596e"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "perplexity 11.172, 100491.4 tokens/sec on cuda:0\n",
            "perplexity 8.553, 99860.4 tokens/sec on cuda:0\n",
            "perplexity 6.553, 100810.6 tokens/sec on cuda:0\n",
            "perplexity 4.553, 100541.7 tokens/sec on cuda:0\n",
            "perplexity 2.436, 99911.6 tokens/sec on cuda:0\n",
            "perplexity 1.348, 99873.9 tokens/sec on cuda:0\n",
            "perplexity 1.132, 100040.1 tokens/sec on cuda:0\n",
            "perplexity 1.066, 101240.2 tokens/sec on cuda:0\n",
            "perplexity 1.042, 101301.9 tokens/sec on cuda:0\n",
            "perplexity 1.042, 99982.3 tokens/sec on cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "vgvTohwnGLY7",
        "outputId": "28cd2744-93ea-4831-bb9d-b01a7fa3e285"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time traveller for so it will be convenient to speak of himwas e'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Stacked LSTM "
      ],
      "metadata": {
        "id": "8o4lXV-NIrZW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_stacks=2\n",
        "lstm_layer = nn.LSTM(len(vocab), num_hiddens,num_stacks)\n",
        "model = RNNModel(lstm_layer, len(vocab))\n",
        "model = model.to(device)\n",
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "dNgsEN2cItI8",
        "outputId": "1c4a4682-0a67-435a-b933-3fb7c5ddd5d5"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travelleraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loss=nn.CrossEntropyLoss()\n",
        "lr=2 #complex architecture\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr)\n",
        "train(model, train_iter, vocab,loss,optimizer, num_epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qJXMyTkyI1s2",
        "outputId": "3573f9b2-ae67-424a-92d6-45eaae7f9a36"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "perplexity 14.547, 79689.4 tokens/sec on cuda:0\n",
            "perplexity 9.229, 80013.5 tokens/sec on cuda:0\n",
            "perplexity 5.512, 79797.5 tokens/sec on cuda:0\n",
            "perplexity 1.752, 79467.6 tokens/sec on cuda:0\n",
            "perplexity 1.108, 79289.1 tokens/sec on cuda:0\n",
            "perplexity 1.053, 78335.3 tokens/sec on cuda:0\n",
            "perplexity 1.039, 79101.7 tokens/sec on cuda:0\n",
            "perplexity 1.035, 79150.1 tokens/sec on cuda:0\n",
            "perplexity 1.024, 79693.8 tokens/sec on cuda:0\n",
            "perplexity 1.029, 79261.3 tokens/sec on cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "zWnXaLw6I3gc",
        "outputId": "c877f70f-5cbd-4a7b-b973-e46b29417d15"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travelleryou can show black is white by argument said filby'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Bidirectional LSTM"
      ],
      "metadata": {
        "id": "FTLIZKNkI5OM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Why you shouldn't use them for language modelling\n",
        "One of the key features of a bidirectional RNN is that information from both ends of the sequence is used to estimate the output. That is, we use information from both future and past observations to predict the current one. In the case of next token prediction this is not quite what we want.\n",
        "\n",
        "\n",
        "If we were to ignore all advice regarding the fact that bidirectional RNNs use past and future data and simply apply it to language models, we will get estimates with acceptable perplexity. Nonetheless, the ability of the model to predict future tokens is severely compromised as the experiment below illustrates. **Despite reasonable perplexity, it only generates gibberish even after many iterations. We include the code below as a cautionary example against using them in the wrong context.**"
      ],
      "metadata": {
        "id": "8IIsff3ejIp_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#To be added\n",
        "num_stacks=2\n",
        "lstm_layer = nn.LSTM(len(vocab), num_hiddens,num_stacks,bidirectional=True)\n",
        "model = RNNModel(lstm_layer, len(vocab))\n",
        "model = model.to(device)\n",
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "id": "Z5youEa7I8zq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8be2aab7-f4ad-40d9-d824-14f17ed2d781"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travellerkdkdddkdddkdddkdddkdddkdddkdddkdddkdddkdddkdddkddd'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loss=nn.CrossEntropyLoss()\n",
        "lr=1\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr)\n",
        "train(model, train_iter, vocab,loss,optimizer, num_epochs, device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ci3pmOL2j3U7",
        "outputId": "f1c3060c-0a74-47b9-e598-e90f95e3561e"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "perplexity 3.377, 41536.6 tokens/sec on cuda:0\n",
            "perplexity 1.290, 42050.4 tokens/sec on cuda:0\n",
            "perplexity 1.202, 41665.1 tokens/sec on cuda:0\n",
            "perplexity 1.175, 41928.9 tokens/sec on cuda:0\n",
            "perplexity 1.158, 42151.3 tokens/sec on cuda:0\n",
            "perplexity 1.138, 41602.1 tokens/sec on cuda:0\n",
            "perplexity 1.134, 41856.9 tokens/sec on cuda:0\n",
            "perplexity 1.138, 41990.1 tokens/sec on cuda:0\n",
            "perplexity 1.113, 41987.4 tokens/sec on cuda:0\n",
            "perplexity 1.107, 41585.2 tokens/sec on cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict('time traveller', 50, model, vocab, device)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "ZiQSS-UHj8tH",
        "outputId": "fe1ba3a7-6400-49c4-916f-87e46fd1dbc9"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'time travellerererererererererererererererererererererererererer'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    }
  ]
}