Lesson_7_BERT_on_外賣資料集_verbose.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lesson 7: BERT on 外賣資料集 verbose.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/UDICatNCHU/PyTorch-Tutorial/blob/master/Lesson_7_BERT_on_%E5%A4%96%E8%B3%A3%E8%B3%87%E6%96%99%E9%9B%86_verbose.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kTBPkhn3NwyT"
      },
      "source": [
        "### **環境設定**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EwDUPX_GAFoa"
      },
      "source": [
        ""
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3L-rpcqJJ6MQ"
      },
      "source": [
        ""
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "okpsnijCN5xr"
      },
      "source": [
        "### **取得 google drive 存取權限**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qu_53dSWe6vc",
        "outputId": "ec5e4c0b-1678-4345-f279-7dc9d2652b05",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ft9Jzub0X0iN",
        "outputId": "906a24b9-4a13-4d14-b7e9-299a864e15bd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "cd /content/drive/My Drive/Colab Notebooks/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/Colab Notebooks\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EV0IjAmsloF",
        "outputId": "fd89e518-d9d4-4beb-a857-0aa3810500a5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wget https://www.dropbox.com/s/ayihrt5n8kv4ofx/waimai_10k_zh_tw.csv?dl=0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-13 09:14:16--  https://www.dropbox.com/s/ayihrt5n8kv4ofx/waimai_10k_zh_tw.csv?dl=0\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.6.1, 2620:100:601b:1::a27d:801\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.6.1|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: /s/raw/ayihrt5n8kv4ofx/waimai_10k_zh_tw.csv [following]\n",
            "--2020-11-13 09:14:17--  https://www.dropbox.com/s/raw/ayihrt5n8kv4ofx/waimai_10k_zh_tw.csv\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://ucbcbb21cfd07337ba70735ea338.dl.dropboxusercontent.com/cd/0/inline/BDEe-daQpaadGDt5xPe_YibfYSbeLwQMUtY2G_i5SMS9XhapoMSk-941bpDBQ0_TaO_tWtYWrfS7UsyWegnaTWZSrY8Q080X4XCoJUU-3e2PYESAj3lDq7EQqN0ToOF1b4E/file# [following]\n",
            "--2020-11-13 09:14:17--  https://ucbcbb21cfd07337ba70735ea338.dl.dropboxusercontent.com/cd/0/inline/BDEe-daQpaadGDt5xPe_YibfYSbeLwQMUtY2G_i5SMS9XhapoMSk-941bpDBQ0_TaO_tWtYWrfS7UsyWegnaTWZSrY8Q080X4XCoJUU-3e2PYESAj3lDq7EQqN0ToOF1b4E/file\n",
            "Resolving ucbcbb21cfd07337ba70735ea338.dl.dropboxusercontent.com (ucbcbb21cfd07337ba70735ea338.dl.dropboxusercontent.com)... 162.125.6.15, 2620:100:601c:15::a27d:60f\n",
            "Connecting to ucbcbb21cfd07337ba70735ea338.dl.dropboxusercontent.com (ucbcbb21cfd07337ba70735ea338.dl.dropboxusercontent.com)|162.125.6.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 919380 (898K) [text/plain]\n",
            "Saving to: ‘waimai_10k_zh_tw.csv?dl=0’\n",
            "\n",
            "waimai_10k_zh_tw.cs 100%[===================>] 897.83K  --.-KB/s    in 0.009s  \n",
            "\n",
            "2020-11-13 09:14:17 (101 MB/s) - ‘waimai_10k_zh_tw.csv?dl=0’ saved [919380/919380]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eo8JiAvSr8l-"
      },
      "source": [
        "!mv 'waimai_10k_zh_tw.csv?dl=0' waimai_10k_zh_tw.csv"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8G-s_DPItZnY",
        "outputId": "1b3d86a6-ef9d-4b42-a17e-d70f9d441e54",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!head waimai_10k_zh_tw.csv"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "label,review\n",
            "1,很快，好吃，味道足，量大\n",
            "1,沒有送水沒有送水沒有送水\n",
            "1,非常快，態度好。\n",
            "1,方便，快捷，味道可口，快遞給力\n",
            "1,菜味道很棒！送餐很及時！\n",
            "1,今天師傅是不是手抖了，微辣格外辣！\n",
            "1,\"送餐快,態度也特別好,辛苦啦謝謝\"\n",
            "1,超級快就送到了，這麼冷的天氣騎士們辛苦了。謝謝你們。麻辣香鍋依然很好吃。\n",
            "1,經過上次晚了2小時，這次超級快，20分鐘就送到了……\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "__EAXsDQO8VA",
        "outputId": "4f6ed333-5335-4e03-8eb0-68434c210261",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wc waimai_10k_zh_tw.csv"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 11988  11988 919380 waimai_10k_zh_tw.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9A0d7j-4WYfk"
      },
      "source": [
        "### **切 training data 及 testing data**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S5q_BxSKO7hO"
      },
      "source": [
        "# 將要訓練的句子存起來\n",
        "def SaveSentence(filepath, sent_list):\n",
        "    f = open(filepath, 'w', encoding='UTF-8')\n",
        "    for sent in sent_list:\n",
        "        f.write(sent + '\\n')\n",
        "    f.close()\n",
        "\n",
        "# 分割train_data和test_data\n",
        "def data_Split(FileName):\n",
        "\n",
        "    fp = open(FileName, 'r', encoding='utf-8')\n",
        "    line = fp.readline()        # 第一行是label,review\n",
        "    line = fp.readline()\n",
        "\n",
        "    train_sent_num = 3000\n",
        "    test_sent_num = 1000\n",
        "    # 計算目前資料筆數\n",
        "    train_positive_num = 0 \n",
        "    train_negative_num = 0\n",
        "    test_positive_num = 0\n",
        "    test_negative_num = 0\n",
        "    train_data = []\n",
        "    test_data = []\n",
        "\n",
        "    # 用 while 逐行讀取檔案內容，直至檔案結尾\n",
        "    while line:\n",
        "        sent = ''\n",
        "        sent = line.replace('\\n', '')\n",
        "\n",
        "        if line[:2] == '1,':\n",
        "            if train_positive_num < train_sent_num:\n",
        "                train_data.append(sent)\n",
        "                train_positive_num += 1\n",
        "            elif test_positive_num < test_sent_num:\n",
        "                test_data.append(sent)\n",
        "                test_positive_num += 1\n",
        "        else:\n",
        "            if train_negative_num < train_sent_num:\n",
        "                train_data.append(sent)\n",
        "                train_negative_num += 1\n",
        "            elif test_negative_num < test_sent_num:\n",
        "                test_data.append(sent)\n",
        "                test_negative_num += 1\n",
        "        \n",
        "        line = fp.readline()\n",
        "    \n",
        "    fp.close()\n",
        "\n",
        "    SaveSentence('train_data.txt', train_data)\n",
        "    SaveSentence('test_data.txt', test_data)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PFWR9XNPWs-9"
      },
      "source": [
        "data_Split('waimai_10k_zh_tw.csv')"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kC-zNaj_t3bN",
        "outputId": "d04c0c37-7897-49b1-e28a-da4368642edc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wc train_data.txt"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  6000   6000 430529 train_data.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZKEBsX-9jVsl"
      },
      "source": [
        "### **安裝所需的函式庫**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pQNrfvGA1Tdb",
        "outputId": "aa251f6b-b92a-46b4-8a41-df7334eb13f5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.5.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: tokenizers==0.9.3 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: sentencepiece==0.1.91 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers) (3.12.4)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers) (50.3.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UM5pihYKZqxy"
      },
      "source": [
        "%tensorflow_version 2.x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W-ANDs98mfvE"
      },
      "source": [
        "import os\n",
        "import pickle\n",
        "import torch\n",
        "from transformers import BertConfig, BertTokenizer, BertForSequenceClassification, AdamW\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "import torch.nn.functional as F # 激勵函數"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAsW8-fPvMTW",
        "outputId": "4f125924-c5c6-4c9e-abc5-92c02155c83e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wget https://www.dropbox.com/s/fxa234s3de7k55q/bert-base-chinese-vocab.txt?dl=0"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-13 09:14:37--  https://www.dropbox.com/s/fxa234s3de7k55q/bert-base-chinese-vocab.txt?dl=0\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.6.1, 2620:100:601c:1::a27d:601\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.6.1|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: /s/raw/fxa234s3de7k55q/bert-base-chinese-vocab.txt [following]\n",
            "--2020-11-13 09:14:37--  https://www.dropbox.com/s/raw/fxa234s3de7k55q/bert-base-chinese-vocab.txt\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc0bd446a43b303b5f614a4aaf55.dl.dropboxusercontent.com/cd/0/inline/BDFlxMcPJ3UF1PQe_EHyElwCVfMSvmj9cRlyXbxT_NMv9CGRqjF8cGU4rstkgREluHOs6SKLVX22SBReh0Hz5nDiOExqPn9mN5ocTWoeU-3n16hNfSqlRb4lIfs-UrOGCvA/file# [following]\n",
            "--2020-11-13 09:14:38--  https://uc0bd446a43b303b5f614a4aaf55.dl.dropboxusercontent.com/cd/0/inline/BDFlxMcPJ3UF1PQe_EHyElwCVfMSvmj9cRlyXbxT_NMv9CGRqjF8cGU4rstkgREluHOs6SKLVX22SBReh0Hz5nDiOExqPn9mN5ocTWoeU-3n16hNfSqlRb4lIfs-UrOGCvA/file\n",
            "Resolving uc0bd446a43b303b5f614a4aaf55.dl.dropboxusercontent.com (uc0bd446a43b303b5f614a4aaf55.dl.dropboxusercontent.com)... 162.125.6.15, 2620:100:601c:15::a27d:60f\n",
            "Connecting to uc0bd446a43b303b5f614a4aaf55.dl.dropboxusercontent.com (uc0bd446a43b303b5f614a4aaf55.dl.dropboxusercontent.com)|162.125.6.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 109540 (107K) [text/plain]\n",
            "Saving to: ‘bert-base-chinese-vocab.txt?dl=0’\n",
            "\n",
            "bert-base-chinese-v 100%[===================>] 106.97K  --.-KB/s    in 0.003s  \n",
            "\n",
            "2020-11-13 09:14:38 (31.5 MB/s) - ‘bert-base-chinese-vocab.txt?dl=0’ saved [109540/109540]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qjG9viUvRnE"
      },
      "source": [
        "!mv 'bert-base-chinese-vocab.txt?dl=0' bert-base-chinese-vocab.txt"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fhvyxSugvjOj",
        "outputId": "f2fc1edf-8aba-462a-fae1-c937b8f20224",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!head bert-base-chinese-vocab.txt"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[PAD]\n",
            "[unused1]\n",
            "[unused2]\n",
            "[unused3]\n",
            "[unused4]\n",
            "[unused5]\n",
            "[unused6]\n",
            "[unused7]\n",
            "[unused8]\n",
            "[unused9]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dp7cT2IeBU8X"
      },
      "source": [
        "### **將資料轉換成輸入格式**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JIxND7_CcgSg"
      },
      "source": [
        "def convert_data_to_feature(FileName):\n",
        "    # 載入字典\n",
        "    tokenizer = BertTokenizer(vocab_file='bert-base-chinese-vocab.txt')\n",
        "\n",
        "    # 載入資料\n",
        "    Labels = []\n",
        "    Sentences = []\n",
        "    with open(FileName,'r',encoding='utf-8') as f:\n",
        "        data = f.read()\n",
        "    LS_pairs = data.split(\"\\n\")\n",
        "\n",
        "    for LS_pair in LS_pairs:\n",
        "        if LS_pair != \"\":\n",
        "            try:\n",
        "                L = LS_pair[:1]\n",
        "                S = LS_pair[2:]\n",
        "                Labels.append(int(L))\n",
        "                Sentences.append(S)\n",
        "            except:\n",
        "                continue\n",
        "    \n",
        "    assert len(Labels) == len(Sentences)\n",
        "\n",
        "    # BERT input embedding\n",
        "    max_seq_len = 0     # 紀錄最大長度\n",
        "    input_ids = []\n",
        "    original_length = []    # 紀錄原本長度\n",
        "    for S in Sentences:\n",
        "        # 將句子切割成一個個token\n",
        "        word_piece_list = tokenizer.tokenize(S)\n",
        "        # 將token轉成字典中的id\n",
        "        input_id = tokenizer.convert_tokens_to_ids(word_piece_list)\n",
        "        # 補上[CLS]和[SEP]\n",
        "        input_id = tokenizer.build_inputs_with_special_tokens(input_id)\n",
        "\n",
        "        if(len(input_id)>max_seq_len):\n",
        "            max_seq_len = len(input_id)\n",
        "        input_ids.append(input_id)\n",
        "\n",
        "    print(\"最長句子長度:\",max_seq_len)\n",
        "    assert max_seq_len <= 512 # 小於BERT-base長度限制\n",
        "    max_seq_len = 512\n",
        "\n",
        "    # 補齊長度\n",
        "    for c in input_ids:\n",
        "        # 紀錄原本長度\n",
        "        length = len(c)\n",
        "        original_length.append(length)\n",
        "        while len(c)<max_seq_len:\n",
        "            c.append(0)\n",
        "    \n",
        "    token_type_ids = [[0]*max_seq_len for i in range(len(Sentences))]         # token_type_ids # 儲存的是句子的id，id為0就是第一句，id為1就是第二句\n",
        "    attention_mask = []                                                      # attention_mask # 1代表是真實的單詞id，0代表補齊長度\n",
        "    for i in range(len(Sentences)):\n",
        "        attention_id = []\n",
        "        for j in range(original_length[i]):\n",
        "            attention_id.append(1)\n",
        "        while len(attention_id)<max_seq_len:\n",
        "            attention_id.append(0)\n",
        "        attention_mask.append(attention_id)\n",
        "\n",
        "    assert len(input_ids) == len(token_type_ids) and len(input_ids) == len(attention_mask) and len(input_ids) == len(Labels)\n",
        "\n",
        "    data_features = {'input_ids':input_ids,\n",
        "                    'token_type_ids':token_type_ids,\n",
        "                    'attention_mask':attention_mask,\n",
        "                    'labels':Labels}\n",
        "\n",
        "    return data_features"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ls8HGD6LCOdJ"
      },
      "source": [
        "### **輸入格式轉Dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vMZ8L2eKd-Gc"
      },
      "source": [
        "def makeDataset(data_feature):\n",
        "    input_ids = data_feature['input_ids']\n",
        "    token_type_ids = data_feature['token_type_ids']\n",
        "    attention_mask = data_feature['attention_mask']\n",
        "    labels = data_feature['labels']\n",
        "\n",
        "    all_input_ids = torch.tensor([input_id for input_id in input_ids], dtype=torch.long)\n",
        "    all_token_type_ids = torch.tensor([token_type_id for token_type_id in token_type_ids], dtype=torch.long)\n",
        "    all_attention_mask_ids = torch.tensor([attention_id for attention_id in attention_mask], dtype=torch.long)\n",
        "    all_labels = torch.tensor([label for label in labels], dtype=torch.long)\n",
        "    dataset = TensorDataset(all_input_ids, all_token_type_ids, all_attention_mask_ids, all_labels)\n",
        "\n",
        "    return dataset"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kfAZ7fBCYQO",
        "outputId": "f553486f-dcef-4553-fa70-22c694e03d1b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "tokenizer = BertTokenizer(vocab_file='bert-base-chinese-vocab.txt')\n",
        "data_features = convert_data_to_feature('train_data.txt')\n",
        "\n",
        "print(data_features['input_ids'][5999])\n",
        "print(tokenizer.convert_ids_to_tokens(data_features['input_ids'][5999]))\n",
        "print(data_features['token_type_ids'][5999])\n",
        "print(data_features['attention_mask'][5999])\n",
        "print(data_features['labels'][5999])"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "最長句子長度: 436\n",
            "[101, 7618, 2094, 6843, 6882, 889, 6963, 3893, 749, 8024, 2218, 1391, 749, 1060, 1366, 134, 134, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "['[CLS]', '餃', '子', '送', '過', '來', '都', '涼', '了', '，', '就', '吃', '了', '兩', '口', '=', '=', '[SEP]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jV3XNlQUbvBs"
      },
      "source": [
        "### **Fine-Tuning**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5SYWCMYbw7b"
      },
      "source": [
        "# 計算正確值\n",
        "def compute_accuracy(y_pred, y_target):\n",
        "    _, y_pred_indices = y_pred.max(dim=1)\n",
        "    n_correct = torch.eq(y_pred_indices, y_target).sum().item()\n",
        "    return n_correct / len(y_pred_indices) * 100"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BO8AFnQvdMzs",
        "outputId": "b601242a-699e-4a96-c1e8-7a6e5b1dcf72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        }
      },
      "source": [
        "# set device\n",
        "device = torch.device('cuda')\n",
        "\n",
        "train_data_feature = convert_data_to_feature('train_data.txt')\n",
        "test_data_feature = convert_data_to_feature('test_data.txt')\n",
        "train_dataset = makeDataset(train_data_feature)\n",
        "test_dataset = makeDataset(test_data_feature)\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=4, shuffle=True)\n",
        "test_dataloader = DataLoader(test_dataset, batch_size=4, shuffle=True)\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained('bert-base-chinese')\n",
        "model.to(device)\n",
        "\n",
        "# Prepare optimizer and schedule (linear warmup and decay)\n",
        "# no_decay = ['bias', 'LayerNorm.weight']\n",
        "# optimizer_grouped_parameters = [\n",
        "#     {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.0},\n",
        "#     {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
        "#     ]\n",
        "Learning_rate = 5e-6       # 學習率\n",
        "optimizer = AdamW(model.parameters(), lr=Learning_rate, eps=1e-8)\n",
        "\n",
        "for epoch in range(3):\n",
        "    # 訓練模式\n",
        "    model.train()\n",
        "    All_train_correct = 0.0\n",
        "    AllTrainLoss = 0.0\n",
        "    count = 0\n",
        "    for batch_dict in train_dataloader:\n",
        "        batch_dict = tuple(t.to(device) for t in batch_dict)\n",
        "\n",
        "        outputs = model(\n",
        "            input_ids = batch_dict[0],\n",
        "            token_type_ids = batch_dict[1],             \n",
        "            attention_mask = batch_dict[2],             \n",
        "            labels = batch_dict[3]\n",
        "            )\n",
        "        loss, logits = outputs[:2]\n",
        "            \n",
        "        train_correct = compute_accuracy(logits, batch_dict[3])       # 計算正確率\n",
        "        All_train_correct += train_correct\n",
        "        AllTrainLoss += loss.item()\n",
        "        count += 1\n",
        "\n",
        "        model.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "            \n",
        "    Average_train_correct = round(All_train_correct/count, 3)\n",
        "    Average_train_loss = round(AllTrainLoss/count, 3)\n",
        "\n",
        "    # 測試模式\n",
        "    model.eval()\n",
        "    All_test_correct = 0.0\n",
        "    AllTestLoss = 0.0\n",
        "    count = 0\n",
        "    for batch_dict in test_dataloader:\n",
        "        batch_dict = tuple(t.to(device) for t in batch_dict)\n",
        "\n",
        "        outputs = model(\n",
        "            input_ids = batch_dict[0],\n",
        "            token_type_ids = batch_dict[1],            \n",
        "            attention_mask = batch_dict[2],           \n",
        "            labels = batch_dict[3]\n",
        "            )\n",
        "        loss, logits = outputs[:2]\n",
        "\n",
        "        test_correct = compute_accuracy(logits, batch_dict[3])       # 計算正確率\n",
        "        All_test_correct += test_correct\n",
        "        AllTestLoss += loss.item()\n",
        "\n",
        "        count += 1\n",
        "        \n",
        "    Average_test_correct = round(All_test_correct/count, 3)\n",
        "    Average_test_loss = round(AllTestLoss/count, 3)\n",
        "\n",
        "    print('第' + str(epoch+1) + '次' + '訓練模式，loss為:' + str(Average_train_loss) + ' 正確率為' + str(Average_train_correct)+ '，測試模式，loss為:' + str(Average_test_loss) + ' 正確率為' + str(Average_test_correct))\n",
        "    \n",
        "# 模型存檔\n",
        "model_to_save = model.module if hasattr(model, 'module') else model\n",
        "model_to_save.save_pretrained('trained_model')"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "最長句子長度: 436\n",
            "最長句子長度: 307\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-chinese and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-6c5c5084d529>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m         \u001b[0mtrain_correct\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m       \u001b[0;31m# 計算正確率\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m         \u001b[0mAll_train_correct\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtrain_correct\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0mAllTrainLoss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-22-0830cc593eb9>\u001b[0m in \u001b[0;36mcompute_accuracy\u001b[0;34m(y_pred, y_target)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcompute_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mn_correct\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred_indices\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mn_correct\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred_indices\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P4KOyw5IhyhG"
      },
      "source": [
        "### **測試**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "87aXd79BhyJD"
      },
      "source": [
        "def to_input_id(sentence_input):\n",
        "    return tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentence_input)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3uAMmGFch6yc",
        "outputId": "add82f30-eea5-43b4-b015-98a69bc683ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "# load and init\n",
        "tokenizer = BertTokenizer(vocab_file='bert-base-chinese-vocab.txt')\n",
        "config = BertConfig.from_pretrained('trained_model/config.json')\n",
        "model = BertForSequenceClassification.from_pretrained('trained_model/pytorch_model.bin', from_tf=bool('.ckpt' in 'bert-base-chinese'), config=config)\n",
        "model.eval()\n",
        "\n",
        "print('請輸入句子')\n",
        "sentence = input()\n",
        "\n",
        "input_id = to_input_id(sentence)\n",
        "assert len(input_id) <= 512\n",
        "\n",
        "while len(input_id)<512:\n",
        "    input_id.append(0)\n",
        "\n",
        "input_ids = torch.LongTensor(input_id).unsqueeze(0)\n",
        "\n",
        "# predict時，因為沒有label所以沒有loss\n",
        "outputs = model(input_ids)\n",
        "predicts = outputs[:2]\n",
        "predicts = predicts[0]\n",
        "max_val = torch.max(predicts)\n",
        "predict_label = (predicts == max_val).nonzero().numpy()[0][1] # 在第1維度取最大值並返回索引值 \n",
        "\n",
        "if str(predict_label) == '1':\n",
        "    print('正面')\n",
        "else:\n",
        "    print('負面')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "請輸入句子\n",
            "好吃，很喜歡店面的裝飾，很棒的小店。\n",
            "正面\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}