03_Sentiment_Analysis_in_Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis (Classification) using BERT - Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This time, We will perform a sentiment Analysis using Google BERT model on the movie data with default tensorflow 2.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For detailed examplantaion on BERT and fine-tuning to tackle this problem, please refer to the previous notebook '02_Sentiment _Analysis_using BERT' that is performed in Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:05:38.525708 13176 file_utils.py:39] PyTorch version 1.5.0 available.\n",
      "I0626 14:05:38.527736 13176 file_utils.py:55] TensorFlow version 2.2.0 available.\n",
      "wandb: WARNING W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from transformers import *\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load the data\n",
    "##### For data loading and preprocessing step, it is the same as the Keras version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bert_config.json',\n",
       " 'bert_model.ckpt.data-00000-of-00001',\n",
       " 'bert_model.ckpt.index',\n",
       " 'bert_model.ckpt.meta',\n",
       " 'vocab.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./data/bert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.abspath('./data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_table(os.path.join(path,\"ratings_train.txt\"))\n",
    "test = pd.read_table(os.path.join(path,\"ratings_test.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150000, 3)\n",
      "(50000, 3)\n"
     ]
    }
   ],
   "source": [
    "print(train.shape)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9976970</td>\n",
       "      <td>아 더빙.. 진짜 짜증나네요 목소리</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3819312</td>\n",
       "      <td>흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10265843</td>\n",
       "      <td>너무재밓었다그래서보는것을추천한다</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9045019</td>\n",
       "      <td>교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6483659</td>\n",
       "      <td>사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5403919</td>\n",
       "      <td>막 걸음마 뗀 3세부터 초등학교 1학년생인 8살용영화.ㅋㅋㅋ...별반개도 아까움.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7797314</td>\n",
       "      <td>원작의 긴장감을 제대로 살려내지못했다.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9443947</td>\n",
       "      <td>별 반개도 아깝다 욕나온다 이응경 길용우 연기생활이몇년인지..정말 발로해도 그것보단...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>7156791</td>\n",
       "      <td>액션이 없는데도 재미 있는 몇안되는 영화</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5912145</td>\n",
       "      <td>왜케 평점이 낮은건데? 꽤 볼만한데.. 헐리우드식 화려함에만 너무 길들여져 있나?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                           document  label\n",
       "0   9976970                                아 더빙.. 진짜 짜증나네요 목소리      0\n",
       "1   3819312                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나      1\n",
       "2  10265843                                  너무재밓었다그래서보는것을추천한다      0\n",
       "3   9045019                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정      0\n",
       "4   6483659  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...      1\n",
       "5   5403919      막 걸음마 뗀 3세부터 초등학교 1학년생인 8살용영화.ㅋㅋㅋ...별반개도 아까움.      0\n",
       "6   7797314                              원작의 긴장감을 제대로 살려내지못했다.      0\n",
       "7   9443947  별 반개도 아깝다 욕나온다 이응경 길용우 연기생활이몇년인지..정말 발로해도 그것보단...      0\n",
       "8   7156791                             액션이 없는데도 재미 있는 몇안되는 영화      1\n",
       "9   5912145      왜케 평점이 낮은건데? 꽤 볼만한데.. 헐리우드식 화려함에만 너무 길들여져 있나?      1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Create a BERT input\n",
    "##### Through Huggingface, you can easily call 'bert-base-multilingual-cased' tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:05:50.342952 13176 filelock.py:274] Lock 1314914939912 acquired on C:\\Users\\bokhy/.cache\\torch\\transformers\\96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729.lock\n",
      "I0626 14:05:50.346936 13176 file_utils.py:436] https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt not found in cache or force_download set to True, downloading to C:\\Users\\bokhy\\.cache\\torch\\transformers\\tmp0g_0we9u\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34f90c5b168344838289722c338fd2c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=995526.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:05:51.235238 13176 file_utils.py:440] storing https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt in cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729\n",
      "I0626 14:05:51.236236 13176 file_utils.py:443] creating metadata file for C:\\Users\\bokhy/.cache\\torch\\transformers\\96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729\n",
      "I0626 14:05:51.238231 13176 filelock.py:318] Lock 1314914939912 released on C:\\Users\\bokhy/.cache\\torch\\transformers\\96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729.lock\n",
      "I0626 14:05:51.239228 13176 tokenization_utils.py:1015] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt from cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we get into BERT, a simple intro on how to use tokenizer would be: \n",
    "\n",
    "tokenizer.encode => Change the sentence to 'numerical' token (that could be used as an input to BERT) \n",
    "tokenizer.tokenize => Tokenize the sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 9004, 32537, 9659, 22458, 84177, 42428, 102]\n",
      "['너', '##무', '재', '##미', '##있는', '영화']\n"
     ]
    }
   ],
   "source": [
    "# 'This move is so much fun'\n",
    "print(tokenizer.encode(\"너무 재미있는 영화\"))\n",
    "print(tokenizer.tokenize(\"너무 재미있는 영화\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So we are bascially tokenizing all training data \n",
    "BERT's input takes 3 shapes: Token, Segment, Mask\n",
    "\n",
    "Token: Indexed numbers\n",
    "\n",
    "Segment: Numbers that tell whether it's a front sentence or back sentence\n",
    "\n",
    "Mask: Automatically assgined (shows whether sentence is valid or not). If it's valid, fill with 1, it not fill with 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 9004, 32537, 9659, 22458, 84177, 42428, 34776, 119, 8955, 9954, 35465, 25805, 98199, 119088, 10892, 42428, 102]\n",
      "['너', '##무', '재', '##미', '##있는', '영화', '##였다', '.', '꼭', '한', '##번', '다시', '보고', '##싶', '##은', '영화']\n",
      "[101, 9004, 32537, 9659, 22458, 84177, 42428, 34776, 119, 8955, 9954, 35465, 25805, 98199, 119088, 10892, 42428, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.encode(\"너무 재미있는 영화였다. 꼭 한번 다시 보고싶은 영화\"))\n",
    "print(tokenizer.tokenize(\"너무 재미있는 영화였다. 꼭 한번 다시 보고싶은 영화\"))\n",
    "print(tokenizer.encode(\"너무 재미있는 영화였다. 꼭 한번 다시 보고싶은 영화\", max_length=128, pad_to_max_length=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_data(data_df):\n",
    "    global tokenizer\n",
    "    \n",
    "    SEQ_LEN = 128 #SEQ_LEN : Length of input\n",
    "    \n",
    "    tokens, masks, segments, targets = [], [], [], []\n",
    "    \n",
    "    for i in tqdm(range(len(data_df))):\n",
    "        # token : Tokenize the sentence\n",
    "        token = tokenizer.encode(data_df[DATA_COLUMN][i], max_length=SEQ_LEN, pad_to_max_length=True)\n",
    "       \n",
    "        # If it's valid (not padding), fill with 1, it not (padding) fill with 0\n",
    "        num_zeros = token.count(0)\n",
    "        mask = [1]*(SEQ_LEN-num_zeros) + [0]*num_zeros\n",
    "        \n",
    "        # Segement is 0 (because we only input 1 sentence) \n",
    "        segment = [0]*SEQ_LEN\n",
    "\n",
    "        tokens.append(token)\n",
    "        masks.append(mask)\n",
    "        segments.append(segment)\n",
    "        \n",
    "        # target value: 1 or 0 \n",
    "        targets.append(data_df[LABEL_COLUMN][i])\n",
    "\n",
    "    # change tokens, masks, segments as numpy array  \n",
    "    tokens = np.array(tokens)\n",
    "    masks = np.array(masks)\n",
    "    segments = np.array(segments)\n",
    "    targets = np.array(targets)\n",
    "\n",
    "    return [tokens, masks, segments], targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataframe and split it into train/test\n",
    "\n",
    "def load_data(df):\n",
    "    data_df = df\n",
    "    data_df[DATA_COLUMN] = data_df[DATA_COLUMN].astype(str)\n",
    "    data_df[LABEL_COLUMN] = data_df[LABEL_COLUMN].astype(int)\n",
    "    data_x, data_y = convert_data(data_df)\n",
    "    return data_x, data_y\n",
    "\n",
    "SEQ_LEN = 128\n",
    "BATCH_SIZE = 20\n",
    "\n",
    "DATA_COLUMN = \"document\"\n",
    "LABEL_COLUMN = \"label\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 150000/150000 [00:50<00:00, 2974.36it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 50000/50000 [00:16<00:00, 3006.82it/s]\n"
     ]
    }
   ],
   "source": [
    "train_x, train_y = load_data(train)\n",
    "test_x, test_y = load_data(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Create a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:07:27.754609 13176 filelock.py:274] Lock 1315105674888 acquired on C:\\Users\\bokhy/.cache\\torch\\transformers\\45629519f3117b89d89fd9c740073d8e4c1f0a70f9842476185100a8afe715d1.65df3cef028a0c91a7b059e4c404a975ebe6843c71267b67019c0e9cfa8a88f0.lock\n",
      "I0626 14:07:27.757560 13176 file_utils.py:436] https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json not found in cache or force_download set to True, downloading to C:\\Users\\bokhy\\.cache\\torch\\transformers\\tmpeelszb2b\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03b4873c2d314cfca4ffffac22920bc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=625.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:07:28.171469 13176 file_utils.py:440] storing https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json in cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\45629519f3117b89d89fd9c740073d8e4c1f0a70f9842476185100a8afe715d1.65df3cef028a0c91a7b059e4c404a975ebe6843c71267b67019c0e9cfa8a88f0\n",
      "I0626 14:07:28.173465 13176 file_utils.py:443] creating metadata file for C:\\Users\\bokhy/.cache\\torch\\transformers\\45629519f3117b89d89fd9c740073d8e4c1f0a70f9842476185100a8afe715d1.65df3cef028a0c91a7b059e4c404a975ebe6843c71267b67019c0e9cfa8a88f0\n",
      "I0626 14:07:28.175475 13176 filelock.py:318] Lock 1315105674888 released on C:\\Users\\bokhy/.cache\\torch\\transformers\\45629519f3117b89d89fd9c740073d8e4c1f0a70f9842476185100a8afe715d1.65df3cef028a0c91a7b059e4c404a975ebe6843c71267b67019c0e9cfa8a88f0.lock\n",
      "I0626 14:07:28.182441 13176 configuration_utils.py:285] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json from cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\45629519f3117b89d89fd9c740073d8e4c1f0a70f9842476185100a8afe715d1.65df3cef028a0c91a7b059e4c404a975ebe6843c71267b67019c0e9cfa8a88f0\n",
      "I0626 14:07:28.183440 13176 configuration_utils.py:321] Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 119547\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:08:48.558371 13176 filelock.py:274] Lock 1315105652936 acquired on C:\\Users\\bokhy/.cache\\torch\\transformers\\273ed844d60ef1d5a4ea8f7857e3c3869d05d7b22296f4ae9bc56026ed40eeb7.1b4841f14bf42137fc7ecee17a46c1b2f22b417f636347e4b810bd06dd9c45ea.h5.lock\n",
      "I0626 14:08:48.560464 13176 file_utils.py:436] https://cdn.huggingface.co/bert-base-multilingual-cased-tf_model.h5 not found in cache or force_download set to True, downloading to C:\\Users\\bokhy\\.cache\\torch\\transformers\\tmpt_00t2o_\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bfb3e05645b48b18d20e0219949ec77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1083389348.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:13:07.895117 13176 file_utils.py:440] storing https://cdn.huggingface.co/bert-base-multilingual-cased-tf_model.h5 in cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\273ed844d60ef1d5a4ea8f7857e3c3869d05d7b22296f4ae9bc56026ed40eeb7.1b4841f14bf42137fc7ecee17a46c1b2f22b417f636347e4b810bd06dd9c45ea.h5\n",
      "I0626 14:13:07.899083 13176 file_utils.py:443] creating metadata file for C:\\Users\\bokhy/.cache\\torch\\transformers\\273ed844d60ef1d5a4ea8f7857e3c3869d05d7b22296f4ae9bc56026ed40eeb7.1b4841f14bf42137fc7ecee17a46c1b2f22b417f636347e4b810bd06dd9c45ea.h5\n",
      "I0626 14:13:07.901078 13176 filelock.py:318] Lock 1315105652936 released on C:\\Users\\bokhy/.cache\\torch\\transformers\\273ed844d60ef1d5a4ea8f7857e3c3869d05d7b22296f4ae9bc56026ed40eeb7.1b4841f14bf42137fc7ecee17a46c1b2f22b417f636347e4b810bd06dd9c45ea.h5.lock\n",
      "I0626 14:13:07.902076 13176 modeling_tf_utils.py:393] loading weights file https://cdn.huggingface.co/bert-base-multilingual-cased-tf_model.h5 from cache at C:\\Users\\bokhy/.cache\\torch\\transformers\\273ed844d60ef1d5a4ea8f7857e3c3869d05d7b22296f4ae9bc56026ed40eeb7.1b4841f14bf42137fc7ecee17a46c1b2f22b417f636347e4b810bd06dd9c45ea.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0626 14:13:11.355507 13176 modeling_tf_utils.py:435] Layers from pretrained model not used in TFBertModel: ['nsp___cls', 'mlm___cls']\n"
     ]
    }
   ],
   "source": [
    "model = TFBertModel.from_pretrained('bert-base-multilingual-cased')\n",
    "\n",
    "# define each input\n",
    "token_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_word_ids')\n",
    "mask_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_masks')\n",
    "segment_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_segment')\n",
    "\n",
    "# output shape of BERT is [batch_size, length of the sentence, 768]\n",
    "bert_outputs = model([token_inputs, mask_inputs, segment_inputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'tf_bert_model/Identity:0' shape=(None, 128, 768) dtype=float32>,\n",
       " <tf.Tensor 'tf_bert_model/Identity_1:0' shape=(None, 768) dtype=float32>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_outputs = bert_outputs[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_first = tf.keras.layers.Dense(1, activation='sigmoid', kernel_initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02))(bert_outputs)\n",
    "sentiment_model = tf.keras.Model([token_inputs, mask_inputs, segment_inputs], sentiment_first)\n",
    "\n",
    "# Let try using Rectified Adam optimizer\n",
    "import tensorflow_addons as tfa\n",
    "opt = tfa.optimizers.RectifiedAdam(lr=1.0e-5, weight_decay=0.0025)\n",
    "\n",
    "sentiment_model.compile(optimizer=opt, loss=tf.keras.losses.BinaryCrossentropy(), metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_word_ids (InputLayer)     [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_masks (InputLayer)        [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_segment (InputLayer)      [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_bert_model (TFBertModel)     ((None, 128, 768), ( 177853440   input_word_ids[0][0]             \n",
      "                                                                 input_masks[0][0]                \n",
      "                                                                 input_segment[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1)            769         tf_bert_model[0][1]              \n",
      "==================================================================================================\n",
      "Total params: 177,854,209\n",
      "Trainable params: 177,854,209\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sentiment_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_model.fit(train_x, train_y, epochs=1, shuffle=True, batch_size=100, validation_data=(test_x, test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save weights for later use\n",
    "# sentiment_model.save_weights(path+\"/huggingface_bert.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Model Testing with test-set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_convert_data(data_df):\n",
    "    global tokenizer\n",
    "    tokens, masks, segments = [], [], []\n",
    "    \n",
    "    for i in tqdm(range(len(data_df))):\n",
    "\n",
    "        token = tokenizer.encode(data_df[DATA_COLUMN][i], max_length=SEQ_LEN, pad_to_max_length=True)\n",
    "        num_zeros = token.count(0)\n",
    "        mask = [1]*(SEQ_LEN-num_zeros) + [0]*num_zeros\n",
    "        segment = [0]*SEQ_LEN\n",
    "\n",
    "        tokens.append(token)\n",
    "        segments.append(segment)\n",
    "        masks.append(mask)\n",
    "\n",
    "    tokens = np.array(tokens)\n",
    "    masks = np.array(masks)\n",
    "    segments = np.array(segments)\n",
    "    return [tokens, masks, segments]\n",
    "\n",
    "def predict_load_data(df):\n",
    "    data_df = df\n",
    "    data_df[DATA_COLUMN] = data_df[DATA_COLUMN].astype(str)\n",
    "    data_x = predict_convert_data(data_df)\n",
    "    return data_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = predict_load_data(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = sentiment_model.predict(test_set)\n",
    "\n",
    "# Negative: 0, Positive: 1\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check F1 Score\n",
    "from sklearn.metrics import classification_report\n",
    "y_true = test['label']\n",
    "# F1 Score 확인\n",
    "print(classification_report(y_true, np.round(preds,0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Check with actual sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_convert_data(data):\n",
    "    global tokenizer\n",
    "    tokens, masks, segments = [], [], []\n",
    "    token = tokenizer.encode(data, max_length=SEQ_LEN, pad_to_max_length=True)\n",
    "    \n",
    "    num_zeros = token.count(0) \n",
    "    mask = [1]*(SEQ_LEN-num_zeros) + [0]*num_zeros \n",
    "    segment = [0]*SEQ_LEN\n",
    "\n",
    "    tokens.append(token)\n",
    "    segments.append(segment)\n",
    "    masks.append(mask)\n",
    "\n",
    "    tokens = np.array(tokens)\n",
    "    masks = np.array(masks)\n",
    "    segments = np.array(segments)\n",
    "    return [tokens, masks, segments]\n",
    "\n",
    "def movie_evaluation_predict(sentence):\n",
    "    data_x = sentence_convert_data(sentence)\n",
    "    predict = sentiment_model.predict(data_x)\n",
    "    predict_value = np.ravel(predict)\n",
    "    predict_answer = np.round(predict_value,0).item()\n",
    "    \n",
    "    if predict_answer == 0:\n",
    "        print(\"(부정 확률 : %.2f) 부정적인 영화 평가입니다.\" % (1-predict_value))\n",
    "    elif predict_answer == 1:\n",
    "        print(\"(긍정 확률 : %.2f) 긍정적인 영화 평가입니다.\" % predict_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_evaluation_predict(\"보던거라 계속보고있는데 전개도 느리고 주인공인 은희는 한두컷 나오면서 소극적인모습에 \")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}