BERT-based-QA-preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT-based-QA-preprocessing.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMHRrYmxcdvRekCFt8WoVLh"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hKfvNlxGgDtB"
      },
      "source": [
        "# Libraries\n",
        "\n",
        "Installing HuggingFace Transformers (https://github.com/huggingface/transformers)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fb8BGEtXgAyA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bed2c54e-c9d3-4584-b0c1-1184e9a26392"
      },
      "source": [
        "!pip install transformers\r\n",
        "\r\n",
        "import json\r\n",
        "from pathlib import Path\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "from transformers import BertTokenizerFast \r\n",
        "import torch"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/0c/7d5950fcd80b029be0a8891727ba21e0cd27692c407c51261c3c921f6da3/transformers-4.1.1-py3-none-any.whl (1.5MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5MB 9.8MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 30.2MB/s \n",
            "\u001b[?25hCollecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 22.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=9925cfdd1ac6977b09ae70070864ef982c0677ba96088ca7aedf90fbc5b32396\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tRuKCDbpfQd7"
      },
      "source": [
        "# Datset processing\n",
        "\n",
        "Uploading the dataset, splitting the data into train and validation sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ywVh-BXxfPAD"
      },
      "source": [
        "path = Path('qa_dataset.json')\n",
        "data = json.loads(path.read_text(encoding='utf-8'))\n",
        "\n",
        "train, val = train_test_split(all_data, test_size=0.2, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VkXTOmn7gyee"
      },
      "source": [
        "Getting contexts, questions and answers from the the train and validation set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bqqh_4GtfQ44"
      },
      "source": [
        "def read_set(set):\n",
        "    \n",
        "    contexts = []\n",
        "    questions = []\n",
        "    answers = []\n",
        "\n",
        "    for group in set:\n",
        "        context = group['context']\n",
        "        for qa in group['qas']:\n",
        "            question = qa['question']\n",
        "            for answer in qa['answers']:\n",
        "                contexts.append(context)\n",
        "                questions.append(question)\n",
        "                answers.append(answer)\n",
        "\n",
        "    return contexts, questions, answers\n",
        "\n",
        "train_contexts, train_questions, train_answers = read_set(train)\n",
        "val_contexts, val_questions, val_answers = read_set(val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pw-lw_ewhlP4"
      },
      "source": [
        "Adding span tags to answers and contexts"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-mfvs56EfUWM"
      },
      "source": [
        "def add_tags(answers, contexts):\n",
        "    for answer, context in zip(answers, contexts):\n",
        "        gold_text = answer['text']\n",
        "        start_tags = answer['answer_start']\n",
        "        end_tags = answer['answer_end']\n",
        "\n",
        "add_tags(train_answers, train_contexts)\n",
        "add_tags(val_answers, val_contexts)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fyykpM2RjjRJ"
      },
      "source": [
        "# Tokenization and vectorization\n",
        "\n",
        "Initializating BertTokenizerFast from HuggingFace for BERT base multilingual cased pre-trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lpJsubljfdRd"
      },
      "source": [
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-multilingual-cased')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "62BjfF2-kJ2y"
      },
      "source": [
        "Tokenizing and vectorizing questions and contexts with BertTokenizerFast"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8vJcVP_2kKLV"
      },
      "source": [
        "train_encodings = tokenizer(train_contexts, train_questions, truncation=True, padding=True)\n",
        "val_encodings = tokenizer(val_contexts, val_questions, truncation=True, padding=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0LLl8Dj9kZ_r"
      },
      "source": [
        "# Preparing the data for training\n",
        "\n",
        "Adding token positions to answers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4d2Xs9nfqmr"
      },
      "source": [
        "def add_positions(encodings, answers):\n",
        "    start_positions = []\n",
        "    end_positions = []\n",
        "    for i in range(len(answers)):\n",
        "        start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))\n",
        "        end_positions.append(encodings.char_to_token(i, answers[i]['answer_end']))\n",
        "    encodings.update({'start_positions': start_positions, 'end_positions': end_positions})\n",
        "\n",
        "add_positions(train_encodings, train_answers)\n",
        "add_positions(val_encodings, val_answers)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A25yijMWk4Wj"
      },
      "source": [
        "Adapting the data for training with PyTorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mav7cNayfuPo"
      },
      "source": [
        "class Dataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings):\n",
        "        self.encodings = encodings\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.encodings.input_ids)\n",
        "\n",
        "train_dataset = Dataset(train_encodings)\n",
        "val_dataset = Dataset(val_encodings)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}