single-layer-nn/single-layer-nn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e95ff11e-3d6a-4da9-835f-ed9770358225",
   "metadata": {},
   "source": [
    "# Single Layer Name-like Word Generator\n",
    "This is a python script that tries to mimic a bigram model using a single layer neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cfc0526a-f1ed-4b65-91fd-74b0235898aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports and loading the dataset\n",
    "from pathlib import Path\n",
    "import torch\n",
    "\n",
    "data_file = Path(\"data/names.txt\")\n",
    "data = data_file.read_text().split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "81ae3faf-2459-41f3-9015-1b2a61450805",
   "metadata": {},
   "outputs": [],
   "source": [
    "special_char = \".\"\n",
    "chars = sorted(list(set(special_char.join(data))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "70e111ed-79a6-4975-9869-d45d3f433335",
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = {s:i for i,s in enumerate(chars)}\n",
    "itos = {i:s for i,s in enumerate(chars)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c0ba8ab1-f991-4d9a-b858-477d2d269589",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(228146,\n",
       " tensor([ 0,  5, 13,  ..., 25, 26, 24]),\n",
       " tensor([ 5, 13, 13,  ..., 26, 24,  0]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating the dataset\n",
    "xs, ys = [], []\n",
    "for name in data:\n",
    "    word = special_char + name + special_char\n",
    "    for ch1, ch2 in zip(word, word[1:]):\n",
    "        ix1, ix2 = stoi[ch1], stoi[ch2]\n",
    "        xs.append(ix1)\n",
    "        ys.append(ix2)\n",
    "# create tensors\n",
    "xs, ys = torch.tensor(xs), torch.tensor(ys)\n",
    "num = xs.nelement()\n",
    "num, xs, ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "802d03e8-070d-432b-9149-147252a24a3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# xs should rather be an encoding, than an integer\n",
    "# we'll use one-hot encodings for the inputs\n",
    "import torch.nn.functional as F\n",
    "xenc = F.one_hot(xs, num_classes=27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e406d354-3c47-4c74-8398-30af9b839cea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the Layer\n",
    "g = torch.Generator().manual_seed(10)\n",
    "W = torch.randn((27, 27), generator=g, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4e9c0c95-dd28-4aed-ab44-9a54a497b9c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.454429864883423\n",
      "2.4544167518615723\n",
      "2.454404830932617\n",
      "2.454393148422241\n",
      "2.45438289642334\n",
      "2.4543724060058594\n",
      "2.4543628692626953\n",
      "2.4543540477752686\n",
      "2.454345464706421\n",
      "2.4543375968933105\n"
     ]
    }
   ],
   "source": [
    "# learn\n",
    "learning_rate = 10\n",
    "\n",
    "for k in range(5000):\n",
    "\n",
    "    # forward prop\n",
    "\n",
    "    # encode the input xs\n",
    "    xenc = F.one_hot(xs, num_classes=27).float()\n",
    "    # get the logits\n",
    "    logits = xenc @ W\n",
    "    # get the counts\n",
    "    counts = logits.exp()\n",
    "    # get the probs\n",
    "    probs = counts / counts.sum(1, keepdim=True)\n",
    "    # get neg log likelihood or loss\n",
    "    loss = -probs[torch.arange(num), ys].log().mean()\n",
    "    # loss += 0.01*(W**2).mean() # regularization for more smoothened probabilities\n",
    "    # equivalent to laplacian correction in bigram approach\n",
    "\n",
    "\n",
    "    # reset w.grad and prop back\n",
    "    W.grad = None\n",
    "    loss.backward()\n",
    "\n",
    "    # update\n",
    "    W.data += -(learning_rate)*W.grad\n",
    "    if not k%500:\n",
    "        print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "540758f7-a458-4982-b218-f9d1d80db493",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss=tensor(2.4543, grad_fn=<NegBackward0>)\n",
      "benamaha.\n",
      "kynilalos.\n",
      "stanoustiei.\n",
      "labrein.\n",
      "del.\n"
     ]
    }
   ],
   "source": [
    "# sampling from the neaural network\n",
    "\n",
    "g = torch.Generator().manual_seed(46)\n",
    "print(f\"{loss=}\")\n",
    "for i in range(5):\n",
    "\n",
    "    out = []\n",
    "    ix = 0\n",
    "\n",
    "    while True:\n",
    "        # get encodings\n",
    "        xenc = F.one_hot(torch.tensor([ix]), num_classes=27).float()\n",
    "\n",
    "        # get prob\n",
    "        logits = xenc @ W\n",
    "        counts = logits.exp()\n",
    "        p = counts / counts.sum(1, keepdim=True)\n",
    "\n",
    "        ix = torch.multinomial(p, num_samples=1, replacement=True, generator=g).item()\n",
    "        out.append(itos[ix])\n",
    "        if ix == 0:\n",
    "            break\n",
    "    print(\"\".join(out))\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f7b28ae-a4ba-4172-ba05-a7dfd9333713",
   "metadata": {},
   "source": [
    "## Concluding\n",
    "With the neural network approach, we reached at the same result as that of bigram model with a more flexible solution."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}