05_SQuAD_in_Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SQuAD in Tensorflow 2.0 (Question and Answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This time, we are going to solve SQuAD problem with Tensorflow by fine-tuning pretrained BERT-Large model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### For donwloading SQuAD dataset, please refer to '03_SQuAD_in_Keras' notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from transformers import *\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import re\n",
    "import pickle\n",
    "import codecs\n",
    "from tqdm import tqdm\n",
    "import shutil\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data Preprocessing and load tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 11:55:00.262751  6888 tokenization_utils.py:1015] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt from cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\9b3c03a36e83b13d5ba95ac965c9f9074a99e14340c523ab405703179e79fc46.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'bert',\n",
       " 'dev-v1.1.json',\n",
       " 'glove',\n",
       " 'News_Category_Dataset_v2.json',\n",
       " 'ratings_test.txt',\n",
       " 'ratings_train.txt',\n",
       " 'train-v1.1.json']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A function to make SQuAD JSON file to Pandas Dataframe\n",
    "> Reference: https://www.kaggle.com/sanjay11100/squad-stanford-q-a-json-to-pandas-dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squad_json_to_dataframe_train(input_file_path, record_path = ['data','paragraphs','qas','answers'],\n",
    "                           verbose = 1):\n",
    "    \"\"\"\n",
    "    input_file_path: path to the squad json file.\n",
    "    record_path: path to deepest level in json file default value is\n",
    "    ['data','paragraphs','qas','answers']\n",
    "    verbose: 0 to suppress it default is 1\n",
    "    \"\"\"\n",
    "    if verbose:\n",
    "        print(\"Reading the json file\")    \n",
    "    file = json.loads(open(input_file_path).read())\n",
    "    if verbose:\n",
    "        print(\"processing...\")\n",
    "    # parsing different level's in the json file\n",
    "    js = pd.io.json.json_normalize(file , record_path )\n",
    "    m = pd.io.json.json_normalize(file, record_path[:-1] )\n",
    "    r = pd.io.json.json_normalize(file,record_path[:-2])\n",
    "    \n",
    "    #combining it into single dataframe\n",
    "    idx = np.repeat(r['context'].values, r.qas.str.len())\n",
    "    ndx  = np.repeat(m['id'].values,m['answers'].str.len())\n",
    "    m['context'] = idx\n",
    "    js['q_idx'] = ndx\n",
    "    main = pd.concat([ m[['id','question','context']].set_index('id'),js.set_index('q_idx')],1,sort=False).reset_index()\n",
    "    main['c_id'] = main['context'].factorize()[0]\n",
    "    if verbose:\n",
    "        print(\"shape of the dataframe is {}\".format(main.shape))\n",
    "        print(\"Done\")\n",
    "    return main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading the json file\n",
      "processing...\n",
      "shape of the dataframe is (87599, 6)\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Convert Train JSON data to Dataframe\n",
    "train = squad_json_to_dataframe_train(\"./data/train-v1.1.json\")\n",
    "\n",
    "# New column that shows the length of the column 'context'\n",
    "train['context_len'] = train['context'].str.len()\n",
    "\n",
    "# Check the length of 'question' oolumn, which is hightly likely abnormal if it's less than 10\n",
    "train.loc[train['question'].str.len() <= 10].head(10)\n",
    "# Delete the ones less than 10\n",
    "train = train.loc[train['question'].str.len() >= 10].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same hyperparatmer setting as the KERAS version\n",
    "SEQ_LEN = 384\n",
    "DATA_COLUMN = \"context\"\n",
    "QUESTION_COLUMN = \"question\"\n",
    "TEXT = \"text\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_data(data_df):\n",
    "    global tokenizer\n",
    "    indices, segments, masks, target_start, target_end = [], [], [], [], []\n",
    "    \n",
    "    for i in tqdm(range(len(data_df))):\n",
    "        # que : tokenized variable of 'question' column to be used as an BERT input. 'tokenizer.encode' give a tokenized list and segment, but here we only use list\n",
    "        # doc : tokenized variable of 'context' column to be used as an BERT input.\n",
    "        \n",
    "        que = tokenizer.encode(data_df[QUESTION_COLUMN][i])\n",
    "        doc = tokenizer.encode(data_df[DATA_COLUMN][i])\n",
    "        \n",
    "        # delete the [CLS] token, which is at the very first of each 'context' \n",
    "        doc.pop(0)\n",
    "\n",
    "        # que_len, doc_len : length of question , length of context\n",
    "        que_len = len(que)\n",
    "        doc_len = len(doc)\n",
    "\n",
    "        # If the length of question is over 64, it make it to 64 \n",
    "        if que_len > 64:\n",
    "            que = que[:63]\n",
    "            # for any cut question, we want to have [SEP] at the last of the question \n",
    "            que.append(102)\n",
    " \n",
    "\n",
    "\n",
    "        # If the length of context is over 384, it make it to 384 \n",
    "        if len(que+doc) > SEQ_LEN:\n",
    "            while len(que+doc) != SEQ_LEN:\n",
    "                doc.pop(-1)\n",
    "            doc.pop(-1)\n",
    "            # for any cut context, we want to have [SEP] at the last of the context \n",
    "            doc.append(102)\n",
    "\n",
    "        \n",
    "        ############################\n",
    "        ###### Segment example #####\n",
    "        ############################\n",
    "        \n",
    "        # question, context, padding\n",
    "        # 00000000, 1111111, 0000000\n",
    "        \n",
    "        segment = [0]*len(que) + [1]*len(doc) + [0]*(SEQ_LEN-len(que)-len(doc))\n",
    "        if len(que + doc) <= SEQ_LEN:\n",
    "            mask = [1]*len(que+doc) + [0]*(SEQ_LEN-len(que+doc))\n",
    "        else:\n",
    "            mask = [1]*len(que+doc)\n",
    "        \n",
    "        # If the sum lenght of question and context is less than 384, fill remaining with 0 \n",
    "        if len(que + doc) <= SEQ_LEN:\n",
    "            while len(que+doc) != SEQ_LEN:\n",
    "                doc.append(0)\n",
    "\n",
    "        # ids : Acutal input that is the sum of length of question and context\n",
    "        ids = que + doc\n",
    "        \n",
    "        # slide the context to the length of the text, and if answer is found inside context,\n",
    "        # show the first and last word of the text inside context \n",
    "        \n",
    "        text = tokenizer.encode(data_df[TEXT][i])\n",
    "        text_slide_len = len(text[1:-1])\n",
    "        \n",
    "        # exist_flag : if text is found inside context, convert from 0 to 1 \n",
    "        for j in range(0,(len(doc))):  \n",
    "            exist_flag = 0\n",
    "            if text[1:-1] == doc[j:j+text_slide_len]:\n",
    "                ans_start = j + len(que)\n",
    "                ans_end = j + text_slide_len - 1 + len(que)\n",
    "                exist_flag = 1\n",
    "                break\n",
    "        \n",
    "        # if text is NOT found inside context, the put the first and last value as 384\n",
    "        # (this would be deleted later) \n",
    "        if exist_flag == 0:\n",
    "            ans_start = SEQ_LEN\n",
    "            ans_end = SEQ_LEN\n",
    "\n",
    "        # Add ids and segments to indices and segments, which is going to be BERT input \n",
    "        indices.append(ids)\n",
    "        segments.append(segment)\n",
    "        masks.append(mask)\n",
    "        \n",
    "        # Starting point of answer is ans_start and finishing point of answer is ans_end \n",
    "        target_start.append(ans_start)\n",
    "        target_end.append(ans_end)\n",
    "\n",
    "    # Save all \"indices, segments, ans_start, ans_end\" as numpy array     \n",
    "    indices_x = np.array(indices)\n",
    "    segments = np.array(segments)\n",
    "    masks = np.array(masks)\n",
    "    target_start = np.array(target_start)\n",
    "    target_end = np.array(target_end)\n",
    "    \n",
    "    # del_list would be a list that should be deleted because ans_start and ans_end is not actually answer \n",
    "    del_list = np.where(target_start!=SEQ_LEN)[0]\n",
    "    not_del_list = np.where(target_start==SEQ_LEN)[0]\n",
    "    indices_x = indices_x[del_list]\n",
    "    segments = segments[del_list]\n",
    "    masks = masks[del_list]\n",
    "    target_start = target_start[del_list]\n",
    "    target_end = target_end[del_list]\n",
    "\n",
    "    return [indices_x, masks, segments], [target_start, target_end], not_del_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataframe and split it into train/test\n",
    "\n",
    "def load_data(df):\n",
    "    data_df = df\n",
    "    data_df[DATA_COLUMN] = data_df[DATA_COLUMN].astype(str)\n",
    "    data_df[QUESTION_COLUMN] = data_df[QUESTION_COLUMN].astype(str)\n",
    "    data_df[TEXT] = data_df[TEXT].astype(str)\n",
    "    data_x, data_y, del_list = convert_data(data_df)\n",
    "\n",
    "    return data_x, data_y, del_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|██▊                                                                         | 3247/87589 [00:14<05:00, 280.83it/s]W0626 11:56:05.136086  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (718 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:56:05.150048  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (718 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:56:05.164046  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (718 > 512). Running this sequence through the model will result in indexing errors\n",
      "  4%|██▊                                                                         | 3276/87589 [00:14<05:49, 241.33it/s]W0626 11:56:05.179003  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (718 > 512). Running this sequence through the model will result in indexing errors\n",
      " 23%|█████████████████▌                                                         | 20492/87589 [01:13<03:57, 282.44it/s]W0626 11:57:04.300811  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (591 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:04.312781  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (591 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:04.324747  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (591 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:04.335717  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (591 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:04.348683  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (591 > 512). Running this sequence through the model will result in indexing errors\n",
      " 30%|██████████████████████▎                                                    | 26061/87589 [01:31<04:24, 232.98it/s]W0626 11:57:22.517556  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (523 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:22.528527  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (523 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:22.538494  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (523 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:22.549470  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (523 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:22.561438  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (523 > 512). Running this sequence through the model will result in indexing errors\n",
      " 34%|█████████████████████████▌                                                 | 29875/87589 [01:46<04:46, 201.22it/s]W0626 11:57:37.475661  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (556 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:37.486632  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (556 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:37.498601  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (556 > 512). Running this sequence through the model will result in indexing errors\n",
      " 34%|█████████████████████████▌                                                 | 29896/87589 [01:46<04:59, 192.74it/s]W0626 11:57:37.510568  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (556 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:57:37.521538  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (556 > 512). Running this sequence through the model will result in indexing errors\n",
      " 41%|██████████████████████████████▌                                            | 35668/87589 [02:10<03:36, 239.32it/s]W0626 11:58:01.295731  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (622 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:58:01.308698  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (622 > 512). Running this sequence through the model will result in indexing errors\n",
      " 41%|██████████████████████████████▌                                            | 35693/87589 [02:10<03:58, 217.37it/s]W0626 11:58:01.322650  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (622 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:58:01.334651  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (622 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:58:01.346619  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (622 > 512). Running this sequence through the model will result in indexing errors\n",
      " 47%|███████████████████████████████████▏                                       | 41130/87589 [02:34<03:08, 246.75it/s]W0626 11:58:25.848202  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (518 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:58:25.858175  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (518 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:58:25.869147  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (518 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:58:25.879119  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (518 > 512). Running this sequence through the model will result in indexing errors\n",
      " 56%|██████████████████████████████████████████                                 | 49079/87589 [03:10<02:55, 218.83it/s]W0626 11:59:01.460972  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (549 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:01.472940  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (549 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:01.483908  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (549 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:01.494883  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (549 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:01.505850  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (549 > 512). Running this sequence through the model will result in indexing errors\n",
      " 57%|██████████████████████████████████████████▉                                | 50198/87589 [03:15<02:41, 230.87it/s]W0626 11:59:06.012021  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (532 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:06.023336  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (532 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:06.033309  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (532 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:06.044296  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (532 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:06.056264  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (532 > 512). Running this sequence through the model will result in indexing errors\n",
      " 58%|███████████████████████████████████████████▍                               | 50740/87589 [03:17<02:33, 240.79it/s]W0626 11:59:08.254386  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (587 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:08.266354  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (587 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:08.278322  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (587 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:08.291287  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (587 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:08.305250  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (587 > 512). Running this sequence through the model will result in indexing errors\n",
      " 62%|██████████████████████████████████████████████▍                            | 54299/87589 [03:31<02:08, 258.76it/s]W0626 11:59:22.527154  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (639 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:22.541114  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (639 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:22.555059  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (639 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:22.568043  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (639 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:22.582005  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (639 > 512). Running this sequence through the model will result in indexing errors\n",
      " 66%|█████████████████████████████████████████████████▎                         | 57559/87589 [03:45<02:20, 213.86it/s]W0626 11:59:36.458875  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (767 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:36.472873  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (767 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:36.487831  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (767 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:36.503788  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (767 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:36.518825  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (767 > 512). Running this sequence through the model will result in indexing errors\n",
      " 68%|███████████████████████████████████████████████████▎                       | 59996/87589 [03:54<02:06, 218.42it/s]W0626 11:59:45.692293  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (853 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.709241  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (853 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.724207  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (853 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.742162  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (853 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.765098  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (853 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.780058  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (565 > 512). Running this sequence through the model will result in indexing errors\n",
      " 69%|███████████████████████████████████████████████████▍                       | 60019/87589 [03:54<02:30, 183.76it/s]W0626 11:59:45.795021  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (565 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.808996  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (565 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 11:59:45.821913  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (565 > 512). Running this sequence through the model will result in indexing errors\n",
      " 74%|███████████████████████████████████████████████████████▎                   | 64557/87589 [04:13<01:29, 258.35it/s]W0626 12:00:04.359809  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (693 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:04.373773  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (693 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:04.388300  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (693 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:04.402261  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (693 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:04.416224  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (693 > 512). Running this sequence through the model will result in indexing errors\n",
      " 75%|████████████████████████████████████████████████████████▎                  | 65709/87589 [04:18<01:24, 260.42it/s]W0626 12:00:09.102911  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (536 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:09.109896  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (536 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:09.118869  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (536 > 512). Running this sequence through the model will result in indexing errors\n",
      " 76%|████████████████████████████████████████████████████████▋                  | 66237/87589 [04:20<01:28, 241.20it/s]W0626 12:00:11.277145  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (527 > 512). Running this sequence through the model will result in indexing errors\n",
      " 76%|████████████████████████████████████████████████████████▋                  | 66269/87589 [04:20<01:22, 258.12it/s]W0626 12:00:11.289150  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (527 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:11.300222  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (527 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:11.312191  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (527 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:11.323165  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (527 > 512). Running this sequence through the model will result in indexing errors\n",
      " 82%|█████████████████████████████████████████████████████████████▍             | 71751/87589 [04:42<01:07, 233.00it/s]W0626 12:00:33.599433  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (605 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:33.612399  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (605 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:33.635304  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (605 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:33.650290  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (605 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:00:33.666253  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (605 > 512). Running this sequence through the model will result in indexing errors\n",
      " 94%|██████████████████████████████████████████████████████████████████████▊    | 82682/87589 [05:27<00:22, 214.51it/s]W0626 12:01:18.361309  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (530 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:18.372286  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (530 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:18.384247  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (530 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:18.395218  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (530 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:18.407186  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (530 > 512). Running this sequence through the model will result in indexing errors\n",
      "100%|██████████████████████████████████████████████████████████████████████████▋| 87168/87589 [05:46<00:01, 264.91it/s]W0626 12:01:37.236936  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:37.249902  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:37.264888  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:37.278857  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:37.292819  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:37.305784  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "100%|██████████████████████████████████████████████████████████████████████████▋| 87195/87589 [05:46<00:01, 220.95it/s]W0626 12:01:37.320711  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "W0626 12:01:37.334675  6888 tokenization_utils.py:1934] Token indices sequence length is longer than the specified maximum sequence length for this model (573 > 512). Running this sequence through the model will result in indexing errors\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 87589/87589 [05:47<00:00, 251.77it/s]\n"
     ]
    }
   ],
   "source": [
    "train_x, train_y, z = load_data(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[  101,  2000,  3183, ...,     0,     0,     0],\n",
       "        [  101,  2054,  2003, ...,     0,     0,     0],\n",
       "        [  101,  1996, 13546, ...,     0,     0,     0],\n",
       "        ...,\n",
       "        [  101,  2007,  2054, ...,     0,     0,     0],\n",
       "        [  101,  1999,  2054, ...,     0,     0,     0],\n",
       "        [  101,  2054,  2003, ...,     0,     0,     0]]),\n",
       " array([[1, 1, 1, ..., 0, 0, 0],\n",
       "        [1, 1, 1, ..., 0, 0, 0],\n",
       "        [1, 1, 1, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [1, 1, 1, ..., 0, 0, 0],\n",
       "        [1, 1, 1, ..., 0, 0, 0],\n",
       "        [1, 1, 1, ..., 0, 0, 0]]),\n",
       " array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]])]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Load Pre-trained BERT model and fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MyModel():\n",
    "  \n",
    "    model = TFBertModel.from_pretrained('bert-large-uncased')\n",
    "\n",
    "    token_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_word_ids')\n",
    "    seg_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_segments')\n",
    "    mask_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_masks')\n",
    "\n",
    "    seq_output, _ = model([token_inputs, mask_inputs, seg_inputs])\n",
    "    \n",
    "    x = tf.keras.layers.Dense(2, kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02))(seq_output)\n",
    "    start, end = tf.split(x, 2, axis=-1)\n",
    "    start = tf.squeeze(start, axis=-1)\n",
    "    end = tf.squeeze(end, axis=-1)\n",
    "    bert_model2 = tf.keras.Model([token_inputs, mask_inputs, seg_inputs], [start, end])\n",
    "    \n",
    "    import tensorflow_addons as tfa\n",
    "    # https://github.com/tensorflow/addons/blob/master/docs/tutorials/losses_triplet.ipynb\n",
    "    \n",
    "    opt = tf.keras.optimizers.Adam(lr=0.01) # or 1.5e-5\n",
    "    \n",
    "    bert_model2.compile(\n",
    "          optimizer = opt,\n",
    "          loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "          metrics=['sparse_categorical_accuracy'])\n",
    "    bert_model2.summary()\n",
    "    \n",
    "    del model\n",
    "    \n",
    "    return bert_model2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Start Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 12:11:01.370805  6888 configuration_utils.py:285] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-config.json from cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\6dfaed860471b03ab5b9acb6153bea82b6632fb9bbe514d3fff050fe1319ee6d.788fed32bb8481a9b15ce726d41c53d5d5066b04c667e34ce3a7a3826d1573d8\n",
      "I0626 12:11:01.371837  6888 configuration_utils.py:321] Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 1024,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 4096,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "I0626 12:12:21.703265  6888 modeling_tf_utils.py:393] loading weights file https://cdn.huggingface.co/bert-large-uncased-tf_model.h5 from cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\0e94058889449694767c52e013d258f5f6223218e0af7897a6e7ce38dbb775da.606c3f554f24fc83ef5ab05b02561dc75cb37da9ebf736c0519c46aa59c17271.h5\n",
      "I0626 12:12:29.496854  6888 modeling_tf_utils.py:435] Layers from pretrained model not used in TFBertModel: ['nsp___cls', 'mlm___cls']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_word_ids (InputLayer)     [(None, 384)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_masks (InputLayer)        [(None, 384)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_segments (InputLayer)     [(None, 384)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_bert_model_1 (TFBertModel)   ((None, 384, 1024),  335141888   input_word_ids[0][0]             \n",
      "                                                                 input_masks[0][0]                \n",
      "                                                                 input_segments[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 384, 2)       2050        tf_bert_model_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_split_1 (TensorFlow [(None, 384, 1), (No 0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Squeeze_2 (TensorFl [(None, 384)]        0           tf_op_layer_split_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Squeeze_3 (TensorFl [(None, 384)]        0           tf_op_layer_split_1[0][1]        \n",
      "==================================================================================================\n",
      "Total params: 335,143,938\n",
      "Trainable params: 335,143,938\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# create and view summary\n",
    "bert_model2 = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save weights for later use\n",
    "# bert_model2.save_weights(os.path.join(path, \"bert_large_2epoch.h5\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Check the loss "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "preds = bert_model2.predict(train_x)\n",
    "\n",
    "start_indexes = np.argmax(preds[0], axis=-1)\n",
    "end_indexes = np.argmax(preds[1], axis=-1)\n",
    "\n",
    "# f1_score of start_indexes\n",
    "print(classification_report(train_y[0], start_indexes))\n",
    "\n",
    "# f1_score of end_indexes\n",
    "print(classification_report(train_y[1], end_indexes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Inference on Test set \n",
    "### (same as Keras version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squad_json_to_dataframe_dev(input_file_path, record_path = ['data','paragraphs','qas','answers'],\n",
    "                           verbose = 1):\n",
    "    \"\"\"\n",
    "    input_file_path: path to the squad json file.\n",
    "    record_path: path to deepest level in json file default value is\n",
    "    ['data','paragraphs','qas','answers']\n",
    "    verbose: 0 to suppress it default is 1\n",
    "    \"\"\"\n",
    "    if verbose:\n",
    "        print(\"Reading the json file\")    \n",
    "    file = json.loads(open(input_file_path).read())\n",
    "    if verbose:\n",
    "        print(\"processing...\")\n",
    "    # parsing different level's in the json file\n",
    "    js = pd.io.json.json_normalize(file , record_path )\n",
    "    m = pd.io.json.json_normalize(file, record_path[:-1] )\n",
    "    r = pd.io.json.json_normalize(file,record_path[:-2])\n",
    "    \n",
    "    #combining it into single dataframe\n",
    "    idx = np.repeat(r['context'].values, r.qas.str.len())\n",
    "    m['context'] = idx\n",
    "    main = m[['id','question','context','answers']].set_index('id').reset_index()\n",
    "    main['c_id'] = main['context'].factorize()[0]\n",
    "    if verbose:\n",
    "        print(\"shape of the dataframe is {}\".format(main.shape))\n",
    "        print(\"Done\")\n",
    "    return main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file_path ='./data/dev-v1.1.json'\n",
    "record_path = ['data','paragraphs','qas','answers']\n",
    "verbose = 0\n",
    "dev = squad_json_to_dataframe_dev(input_file_path=input_file_path,record_path=record_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new column that's the number of the answer\n",
    "dev['answer_len'] = dev['answers'].map(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change answers to the list\n",
    "def get_text(text_len, answers):\n",
    "  # text_len : the number of answers in question and context\n",
    "  # answers : example: [{'answer_start': 177, 'text': 'Denver Broncos'}, {'answer_start': 177, 'text': 'Denver Broncos'}, {'answer_start': 177, 'text': 'Denver Broncos'}]\n",
    "    texts = []\n",
    "    for i in range(text_len):\n",
    "        texts.append(answers[i]['text'])\n",
    "    return texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply to all data in 'text' column\n",
    "dev['texts'] = dev.apply(lambda x: get_text(x['answer_len'], x['answers']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specifcy text column\n",
    "TEXT_COLUMN = 'texts'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same data converting step as training data\n",
    "def convert_data(data_df):\n",
    "    global tokenizer\n",
    "    indices, segments, masks, target_start, target_end = [], [], [], [], []\n",
    "\n",
    "    for i in tqdm(range(len(data_df))):\n",
    "        que = tokenizer.encode(data_df[QUESTION_COLUMN][i])\n",
    "        doc = tokenizer.encode(data_df[DATA_COLUMN][i])\n",
    "        doc.pop(0)\n",
    "\n",
    "        que_len = len(que)\n",
    "        doc_len = len(doc)\n",
    "\n",
    "        if que_len > 64:\n",
    "            que = que[:63]\n",
    "            que.append(102)\n",
    "        \n",
    "        if len(que+doc) > SEQ_LEN:\n",
    "            while len(que+doc) != SEQ_LEN:\n",
    "                doc.pop(-1)\n",
    "\n",
    "            doc.pop(-1)\n",
    "            doc.append(102)\n",
    "        \n",
    "        if len(que + doc) <= SEQ_LEN:\n",
    "            mask = [1]*len(que+doc) + [0]*(SEQ_LEN-len(que+doc))\n",
    "        else:\n",
    "            mask = [1]*len(que+doc)\n",
    "        segment = [0]*len(que) + [1]*len(doc) + [0]*(SEQ_LEN-len(que)-len(doc))\n",
    "        if len(que + doc) <= SEQ_LEN:\n",
    "            while len(que+doc) != SEQ_LEN:\n",
    "                doc.append(0)\n",
    "\n",
    "        ids = que + doc\n",
    "\n",
    "        texts = data_df[TEXT_COLUMN][i]\n",
    "        for text_element in texts:\n",
    "            text = tokenizer.encode(text_element)\n",
    "\n",
    "            text_slide_len = len(text[1:-1])\n",
    "            for j in range(0,(len(doc))):  \n",
    "                exist_flag = 0\n",
    "                if text[1:-1] == doc[j:j+text_slide_len]:\n",
    "                    ans_start = j + len(que)\n",
    "                    ans_end = j + text_slide_len - 1 + len(que)\n",
    "                    exist_flag = 1\n",
    "                    break\n",
    "        \n",
    "          if exist_flag == 0:\n",
    "            ans_start = SEQ_LEN\n",
    "            ans_end = SEQ_LEN\n",
    "\n",
    "        indices.append(ids)\n",
    "        segments.append(segment)\n",
    "        masks.append(mask)\n",
    "        target_start.append(ans_start)\n",
    "        target_end.append(ans_end)\n",
    "        \n",
    " \n",
    "    indices_x = np.array(indices)\n",
    "    segments = np.array(segments)\n",
    "    masks = np.array(masks)\n",
    "    target_start = np.array(target_start)\n",
    "    target_end = np.array(target_end)\n",
    "    \n",
    "\n",
    "    del_list = np.where(target_start!=SEQ_LEN)[0]\n",
    "    not_del_list = np.where(target_start==SEQ_LEN)[0]\n",
    "    indices_x = indices_x[del_list]\n",
    "    segments = segments[del_list]\n",
    "    masks = masks[del_list]\n",
    "    target_start = target_start[del_list]\n",
    "    target_end = target_end[del_list]\n",
    "\n",
    "    return [indices_x, masks, segments], del_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load and concert test data to dataframe\n",
    "dev_bert_input = convert_data(dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A bit cleaning\n",
    "dev_bert_input, del_list = dev_bert_input[0], dev_bert_input[1]\n",
    "dev = dev.iloc[del_list]\n",
    "dev = dev.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes = dev_bert_input[0]\n",
    "bert_predictions = bert_model2.predict(dev_bert_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_indexes = np.argmax(bert_predictions[0], axis=-1)\n",
    "end_indexes = np.argmax(bert_predictions[1], axis=-1)\n",
    "not_del_list = np.where(start_indexes <= end_indexes)[0]\n",
    "start_indexes = start_indexes[not_del_list]\n",
    "end_indexes = end_indexes[not_del_list]\n",
    "indexes = indexes[not_del_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev = dev.iloc[not_del_list].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.convert_ids_to_tokens(197)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# length :length of test dataset\n",
    "\n",
    "length = len(dev)\n",
    "\n",
    "sentences = []\n",
    "\n",
    "untokenized = []\n",
    "\n",
    "for j in range(len(start_indexes)):\n",
    "    sentence = []\n",
    "    for i in range(start_indexes[j], end_indexes[j]+1):\n",
    "        token_based_word = tokenizer.convert_ids_to_tokens(indexes[j][i].item())\n",
    "        sentence.append(token_based_word)\n",
    "        \n",
    "# save each tokenized word to 'sentence_string'   \n",
    "    sentence_string = \"\"\n",
    "  \n",
    "    for w in sentence:\n",
    "        \n",
    "        # If the token starts with ##, delete ## \n",
    "        if w.startswith(\"##\"):\n",
    "            w = w.replace(\"##\", \"\")\n",
    "        # If the token does NOT start with ##, it's the start of the new sentence, so add a space     \n",
    "        else:\n",
    "            w = \" \" + w\n",
    "        # concatanate all tokens    \n",
    "        sentence_string += w\n",
    "    # if sentence_string starts with \" \", delete space   \n",
    "    if sentence_string.startswith(\" \"):\n",
    "        sentence_string = \"\" + sentence_string[1:]\n",
    "        \n",
    "    # Append all tokens to 'untokenized'\n",
    "    untokenized.append(sentence_string)\n",
    "\n",
    "    sentences.append(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_answers = []\n",
    "for i in range(length):\n",
    "    dev_answer = []\n",
    "    texts_dict = dev['answers'][i]\n",
    "\n",
    "    # save each answer as a list \n",
    "    for j in range(len(texts_dict)):\n",
    "        dev_answer.append(texts_dict[j]['text'])\n",
    "\n",
    "    dev_answers.append(dev_answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_tokens = []\n",
    "for i in dev_answers:\n",
    "    dev_tokened = []\n",
    "    for j in i:\n",
    "        temp_token = tokenizer.tokenize(j)\n",
    "        #print(temp_token)\n",
    "        #temp_token.pop(0)\n",
    "        # DELETE [CLS] \n",
    "        #temp_token.pop(-1)\n",
    "        # DELETE [SEP] \n",
    "        dev_tokened.append(temp_token)\n",
    "    dev_tokens.append(dev_tokened)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONVERT tokenzied words into sentence and concetanate\n",
    "dev_answer_lists = []\n",
    "for dev_answers in dev_tokens:\n",
    "    dev_answer_list = []\n",
    "    for dev_answer in dev_answers:\n",
    "        dev_answer_string = \" \".join(dev_answer)\n",
    "        dev_answer_list.append(dev_answer_string)\n",
    "    dev_answer_lists.append(dev_answer_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# untokenizing\n",
    "dev_strings_end = []\n",
    "for dev_strings in dev_answer_lists:\n",
    "    dev_strings_processed = []\n",
    "    for dev_string in dev_strings:\n",
    "        dev_string = dev_string.replace(\" ##\", \"\")\n",
    "        dev_strings_processed.append(dev_string)\n",
    "    dev_strings_end.append(dev_strings_processed)\n",
    "\n",
    "dev_answers = dev_strings_end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import string, re\n",
    "\n",
    "def normalize_answer(s):\n",
    "    \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n",
    "    def remove_articles(text):\n",
    "        return re.sub(r'\\b(a|an|the)\\b', ' ', text)\n",
    "\n",
    "    def white_space_fix(text):\n",
    "        return ' '.join(text.split())\n",
    "\n",
    "    def remove_punc(text):\n",
    "        exclude = set(string.punctuation)\n",
    "        return ''.join(ch for ch in text if ch not in exclude)\n",
    "\n",
    "    def lower(text):\n",
    "        return text.lower()\n",
    "\n",
    "    return white_space_fix(remove_articles(remove_punc(lower(s))))\n",
    "\n",
    "def f1_score(prediction, ground_truth):\n",
    "    prediction_tokens = normalize_answer(prediction).split()\n",
    "    ground_truth_tokens = normalize_answer(ground_truth).split()\n",
    "    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)\n",
    "    num_same = sum(common.values())\n",
    "    if num_same == 0:\n",
    "        return 0\n",
    "    precision = 1.0 * num_same / len(prediction_tokens)\n",
    "    recall = 1.0 * num_same / len(ground_truth_tokens)\n",
    "    f1 = (2 * precision * recall) / (precision + recall)\n",
    "    return f1\n",
    "\n",
    "def exact_match_score(prediction, ground_truth):\n",
    "    return (normalize_answer(prediction) == normalize_answer(ground_truth))\n",
    "\n",
    "def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):\n",
    "    scores_for_ground_truths = []\n",
    "    for ground_truth in ground_truths:\n",
    "        score = metric_fn(prediction, ground_truth)\n",
    "        scores_for_ground_truths.append(score)\n",
    "    return max(scores_for_ground_truths)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_sum = 0\n",
    "\n",
    "for i in range(len(untokenized)):\n",
    "    f1 = metric_max_over_ground_truths(f1_score, untokenized[i], dev_answers[i])\n",
    "    f1_sum += f1\n",
    "print(\"f1 score : \", f1_sum / length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EM_sum = 0\n",
    "\n",
    "for i in range(len(untokenized)):\n",
    "    EM = metric_max_over_ground_truths(exact_match_score, untokenized[i], dev_answers[i])\n",
    "    EM_sum += EM\n",
    "print(\"EM Score : \", EM_sum / length)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}