GPT_Trainer_for_Malayalam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!rm -rf mygpt\n",
        "!git clone https://github.com/aloshdenny/mygpt.git\n",
        "!pip install tiktoken\n",
        "%cd mygpt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PjvEClKKAmTl",
        "outputId": "ed90fb73-dda5-4a7e-ea9f-acc76b495f3d"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'mygpt'...\n",
            "remote: Enumerating objects: 16, done.\u001b[K\n",
            "remote: Counting objects: 100% (16/16), done.\u001b[K\n",
            "remote: Compressing objects: 100% (12/12), done.\u001b[K\n",
            "remote: Total 16 (delta 6), reused 13 (delta 3), pack-reused 0 (from 0)\u001b[K\n",
            "Receiving objects: 100% (16/16), 18.97 KiB | 18.97 MiB/s, done.\n",
            "Resolving deltas: 100% (6/6), done.\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2024.9.11)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.32.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2024.8.30)\n",
            "Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.8.0\n",
            "/content/mygpt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sPD69l8Y7XQr",
        "outputId": "9a808a13-1689-4937-9e5e-7d40cc5bd85c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-12-15 06:34:57--  https://www.kaggle.com/api/v1/datasets/download/arjungravi/ultimate-malayalam-dataset\n",
            "Resolving www.kaggle.com (www.kaggle.com)... 35.244.233.98\n",
            "Connecting to www.kaggle.com (www.kaggle.com)|35.244.233.98|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://storage.googleapis.com:443/kaggle-data-sets/4818708/8169935/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20241215%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241215T063458Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=5aef952186ad52bbe61bfb6fa850c403999445489202ccdb10c211f911d5926ffb06029447434a82c1ff9b8fa6e9b3079b6b4184460d670984644ec14600e9836021fcde5847a549072ecb1b51a017070940f64ab16b7b24081a3beca65cd32d68e25098d8a62bfcaed09eb635445a92432091e8aaab75def70c54b506ff80f0ec50d23c80fd7c53fd3e93aa01335f8fe7519834ba88188c48048cfd06acbfd996673435afc1423d3e685322f5308acceaf6225c96bdbcc839d787d7cc8fe6f88c268a5b71aa0c83bc7fd65b3fc46e7a0965cbfd993a5051e5c782009170c3481b8f5355bf56a191ec7e5a22ff94dfd98344040aff43cd0294cade0180e32b75 [following]\n",
            "--2024-12-15 06:34:58--  https://storage.googleapis.com/kaggle-data-sets/4818708/8169935/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20241215%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20241215T063458Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=5aef952186ad52bbe61bfb6fa850c403999445489202ccdb10c211f911d5926ffb06029447434a82c1ff9b8fa6e9b3079b6b4184460d670984644ec14600e9836021fcde5847a549072ecb1b51a017070940f64ab16b7b24081a3beca65cd32d68e25098d8a62bfcaed09eb635445a92432091e8aaab75def70c54b506ff80f0ec50d23c80fd7c53fd3e93aa01335f8fe7519834ba88188c48048cfd06acbfd996673435afc1423d3e685322f5308acceaf6225c96bdbcc839d787d7cc8fe6f88c268a5b71aa0c83bc7fd65b3fc46e7a0965cbfd993a5051e5c782009170c3481b8f5355bf56a191ec7e5a22ff94dfd98344040aff43cd0294cade0180e32b75\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.68.207, 142.251.175.207, 64.233.170.207, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.68.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 342223774 (326M) [application/zip]\n",
            "Saving to: ‘ultimate-malayalam-dataset’\n",
            "\n",
            "ultimate-malayalam- 100%[===================>] 326.37M  22.5MB/s    in 15s     \n",
            "\n",
            "2024-12-15 06:35:13 (21.2 MB/s) - ‘ultimate-malayalam-dataset’ saved [342223774/342223774]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "#!/bin/bash\n",
        "!wget https://www.kaggle.com/api/v1/datasets/download/arjungravi/ultimate-malayalam-dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip ./ultimate-malayalam-dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tk7Dkq_o74Jg",
        "outputId": "39ac36c6-f42e-42e7-c029-2cbc81b62e02"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  ./ultimate-malayalam-dataset\n",
            "  inflating: output.csv              \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv(\"./output.csv\")\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "k04i9QGu8I_l",
        "outputId": "6af20210-4cc9-407b-9a3d-83cd4386b888"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                                               Text\n",
              "0           0  മറ്റുള്ളവരെ കുറിച്ചു ചിന്തിക്കുന്നത്‌ മറ്റൊരു ...\n",
              "1           1  ഓസ്ട്രേലിയൻ സർവീസ് ടീമിനെതിരെ ബാറ്റ് ചെയ്ത ബ്ര...\n",
              "2           2                                  പുറത്ത്‌ മഞ്ഞുമഴ.\n",
              "3           3                         അതിന് ശേഷം ഇലക്ട്രിക് ബസ്.\n",
              "4           4  മലയാളം, കന്നഡ, തമിഴ് തുടങ്ങിയ ഭാഷകളിൽ ശ്രദ്ധേയ..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-834d5f38-9c78-4415-a4bb-efeb64487db2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>മറ്റുള്ളവരെ കുറിച്ചു ചിന്തിക്കുന്നത്‌ മറ്റൊരു ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>ഓസ്ട്രേലിയൻ സർവീസ് ടീമിനെതിരെ ബാറ്റ് ചെയ്ത ബ്ര...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>പുറത്ത്‌ മഞ്ഞുമഴ.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>അതിന് ശേഷം ഇലക്ട്രിക് ബസ്.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>മലയാളം, കന്നഡ, തമിഴ് തുടങ്ങിയ ഭാഷകളിൽ ശ്രദ്ധേയ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-834d5f38-9c78-4415-a4bb-efeb64487db2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-834d5f38-9c78-4415-a4bb-efeb64487db2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-834d5f38-9c78-4415-a4bb-efeb64487db2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-38a5436d-8eb3-4af1-9800-9f470f1cc3b8\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-38a5436d-8eb3-4af1-9800-9f470f1cc3b8')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-38a5436d-8eb3-4af1-9800-9f470f1cc3b8 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "# Function to preprocess text (you can adjust the preprocessing steps as needed)\n",
        "def preprocess_text(text):\n",
        "    # Normalize spaces and remove unwanted characters (you can refine the pattern as needed)\n",
        "    text = re.sub(r'\\s+', ' ', text)  # Replace multiple spaces with a single space\n",
        "    text = re.sub(r'[^\\u0D00-\\u0D7F\\s]', '', text)  # Keep only Malayalam characters and spaces\n",
        "    return text\n",
        "\n",
        "# Function to split text into sentences using regex\n",
        "def split_into_sentences(text):\n",
        "    # Split on common sentence-ending punctuation marks (periods, question marks, exclamation points)\n",
        "    # We also handle cases like abbreviations, so we add a space after punctuation marks\n",
        "    sentences = re.split(r'(?<=[.!?])\\s+', text)\n",
        "    return sentences\n",
        "\n",
        "# Load the CSV file\n",
        "df = pd.read_csv(\"./output.csv\")\n",
        "\n",
        "# Initialize an empty list to store the sentences\n",
        "sentences = []\n",
        "\n",
        "# Loop over the 'Text' column, preprocess each sentence, and split them\n",
        "for text in df['Text'].dropna().tolist():\n",
        "    # Preprocess the entire text\n",
        "    processed_text = preprocess_text(text)\n",
        "\n",
        "    # Split the processed text into sentences\n",
        "    text_sentences = split_into_sentences(processed_text)\n",
        "\n",
        "    # Add each sentence to the list\n",
        "    sentences.extend(text_sentences)\n",
        "\n",
        "# Save the preprocessed sentences to a text file, one sentence per line\n",
        "with open('input.txt', 'w', encoding='utf-8') as f:\n",
        "    for sentence in sentences:\n",
        "        f.write(sentence.strip() + '\\n')  # Strip extra spaces and write each sentence on a new line\n",
        "\n",
        "print(f\"Saved {len(sentences)} sentences to 'input.txt'\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l9oAiJgd8zoQ",
        "outputId": "88745fc9-b096-47da-e670-0ec041e1e123"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saved 6050956 sentences to 'input.txt'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BIMWQx1_EMhn",
        "outputId": "964d8777-3aa4-4dcd-8da7-44ee06962c26"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.2.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd mygpt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m0gDF9rvHoXW",
        "outputId": "3ce93605-7a93-4eca-d03c-d0f512372cb1"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/mygpt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sentencepiece as spm\n",
        "\n",
        "# Specify the input file (processed Malayalam text)\n",
        "input_file = 'input.txt'\n",
        "\n",
        "# Read the first 10000 characters from the input file\n",
        "with open(input_file, 'r', encoding='utf-8') as f:\n",
        "    text = f.read(10000)\n",
        "\n",
        "# Save the subset to a temporary file\n",
        "subset_file = 'input_subset.txt'\n",
        "with open(subset_file, 'w', encoding='utf-8') as f:\n",
        "    f.write(text)\n",
        "\n",
        "# Train the SentencePiece model on the subset of text\n",
        "spm.SentencePieceTrainer.train(\n",
        "    f'--input={subset_file} --model_prefix=malayalam_tokenizer --vocab_size=6993 --character_coverage=1.0 --model_type=bpe'\n",
        ")\n",
        "\n",
        "# This will create two files:\n",
        "# - malayalam_tokenizer.model (the model file)\n",
        "# - malayalam_tokenizer.vocab (the vocabulary file)"
      ],
      "metadata": {
        "id": "kgGkzb6_EPfc"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 main.py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qXnhLd427bsO",
        "outputId": "13ac01c4-a7ed-413d-b0a7-ac07397d4db1"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using device cuda\n",
            "Loading model...\n",
            "/content/mygpt/main.py:161: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  m.load_state_dict(torch.load(\"mine.pt\"))\n",
            "പുറത്ത്‌ മഞ്ഞുമഴകോടതിയിൽസ്ഥാപനങ്ങളുംരിഭക്ഷണങ്ങളാണ്പാലത്തിലൂടെത്തിലാണ്ഇത്അതിനപ്പുറംഡയാലിസിസിനുംകടകളില്വല്ലാത്തകലാപംഹര്മന്പ്രീത്പൊട്ടിപ്രതിഫലമായിമഠത്തിലെതരത്തില്ബാധകമാക്കുംശര്മപ്ലാനുകൾവൈറലാണ്പ്രധാനമന്ത്രിനരേന്ദ്രണ്ടതുണ്ട്എന്നിവസംബന്ധിച്ച്എഴുത്തുകുത്തുകളൊന്നുംകണ്ടെത്തിയിട്ടില്ലെന്ന്അയാളെഹിനോക്കരുത്വിക്കറ്റ്ചെയ്തിരുന്നത്സ്ഥാനത്തുംഎന്നിവസംബന്ധിച്ച്എന്നിവരെക്കൂടാതെജീവന്ജീവന്രക്ഷിക്കുവാനായിയിട്ടില്ലെന്ന്രാജ്യദ്രോഹഒരുപാലത്തിലൂടെഇവിടെപതിക്കുന്നത്പദ്ധതികള്മജിസ്ട്രേറ്റ്മെഗാന്കൊല്ലപ്പെട്ടത്ഉപകരണത്തിന്റെമെഗാന്വാണിജ്യവുംഎന്നിവസംബന്ധിച്ച്പിൻഗാമിയായതണുപ്പജീവന്എന്ന്മഠാധിപതിക്കണംഎന്നാല്ഹര്മന്പ്രീത്ഒരുമാസത്തിനിടെഎന്നിവരെക്കൂടാതെവളരെഎളുപ്പമായിചെറുപ്പംമധ്യപ്രദേശ്സംബന്ധിച്ച്ജീവന്രക്ഷിക്കുവാനായിചിലപ്പോഴൊക്കെദിവസംനിയോഗിതനായനേടിയത്പാലത്തിലൂടെജീവന്എന്നിവരെക്കൂടാതെബാനര്ജിഇത്സഹായിക്കുംപുതിയസർക്കാരുംകാണേണ്ടസർക്കാരുംരുന്നുതന്നെഅത്ഭുതമായിജീവന്ഫലമായുംഇസ്രായേല്യർദേശംകാംപിനാകിവല്ലാത്തപറഞ്ഞു\n",
            "\n"
          ]
        }
      ]
    }
  ]
}