NSMC_Classification_CNN_attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "NSMC_Classification_CNN_attention.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO9CgkXUTy5dHesHi22qPj4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/HyejinWon/pytorch-nsmc-classification/blob/test/NSMC_Classification_CNN_attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m8X6TCb-6h8v",
        "outputId": "6254ebe0-2ba8-4bd1-8a7c-bcda629e768c"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "shyy79L76Tlt"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n3cnoF4_zk6r",
        "outputId": "79bf0f51-08b9-44e7-930e-91499f2ac38a"
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7cJn0lKa6ePc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e1d9c82d-797b-4fbb-9619-d4af3f7a290d"
      },
      "source": [
        "!git clone https://github.com/SOMJANG/Mecab-ko-for-Google-Colab.git\n",
        "%cd Mecab-ko-for-Google-Colab\n",
        "!bash install_mecab-ko_on_colab190912.sh"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Mecab-ko-for-Google-Colab'...\n",
            "remote: Enumerating objects: 91, done.\u001b[K\n",
            "remote: Counting objects: 100% (91/91), done.\u001b[K\n",
            "remote: Compressing objects: 100% (85/85), done.\u001b[K\n",
            "remote: Total 91 (delta 43), reused 22 (delta 6), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (91/91), done.\n",
            "/content/Mecab-ko-for-Google-Colab\n",
            "Installing konlpy.....\n",
            "Collecting konlpy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/85/0e/f385566fec837c0b83f216b2da65db9997b35dd675e107752005b7d392b1/konlpy-0.5.2-py2.py3-none-any.whl (19.4MB)\n",
            "\u001b[K     |████████████████████████████████| 19.4MB 71.3MB/s \n",
            "\u001b[?25hCollecting JPype1>=0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/88/f817ef1af6f794e8f11313dcd1549de833f4599abcec82746ab5ed086686/JPype1-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (448kB)\n",
            "\u001b[K     |████████████████████████████████| 450kB 47.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.7/dist-packages (from konlpy) (1.19.5)\n",
            "Requirement already satisfied: tweepy>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (3.10.0)\n",
            "Collecting colorama\n",
            "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
            "Collecting beautifulsoup4==4.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/d4/10f46e5cfac773e22707237bfcd51bbffeaf0a576b0a847ec7ab15bd7ace/beautifulsoup4-4.6.0-py3-none-any.whl (86kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 14.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (4.2.6)\n",
            "Requirement already satisfied: typing-extensions; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from JPype1>=0.7.0->konlpy) (3.7.4.3)\n",
            "Requirement already satisfied: requests[socks]>=2.11.1 in /usr/local/lib/python3.7/dist-packages (from tweepy>=3.7.0->konlpy) (2.23.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tweepy>=3.7.0->konlpy) (1.15.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tweepy>=3.7.0->konlpy) (1.3.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (3.0.4)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6; extra == \"socks\" in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (1.7.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->tweepy>=3.7.0->konlpy) (3.1.1)\n",
            "Installing collected packages: JPype1, colorama, beautifulsoup4, konlpy\n",
            "  Found existing installation: beautifulsoup4 4.6.3\n",
            "    Uninstalling beautifulsoup4-4.6.3:\n",
            "      Successfully uninstalled beautifulsoup4-4.6.3\n",
            "Successfully installed JPype1-1.3.0 beautifulsoup4-4.6.0 colorama-0.4.4 konlpy-0.5.2\n",
            "Done\n",
            "Installing mecab-0.996-ko-0.9.2.tar.gz.....\n",
            "Downloading mecab-0.996-ko-0.9.2.tar.gz.......\n",
            "from https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz\n",
            "--2021-07-07 11:03:38--  https://bitbucket.org/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz\n",
            "Resolving bitbucket.org (bitbucket.org)... 104.192.141.1, 2406:da00:ff00::6b17:d1f5, 2406:da00:ff00::22c2:513, ...\n",
            "Connecting to bitbucket.org (bitbucket.org)|104.192.141.1|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://bbuseruploads.s3.amazonaws.com/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz?Signature=ibQjdPx%2F%2FTVR4iXFyL79kmQ3zrQ%3D&Expires=1625656899&AWSAccessKeyId=AKIA6KOSE3BNJRRFUUX6&versionId=null&response-content-disposition=attachment%3B%20filename%3D%22mecab-0.996-ko-0.9.2.tar.gz%22&response-content-encoding=None [following]\n",
            "--2021-07-07 11:03:38--  https://bbuseruploads.s3.amazonaws.com/eunjeon/mecab-ko/downloads/mecab-0.996-ko-0.9.2.tar.gz?Signature=ibQjdPx%2F%2FTVR4iXFyL79kmQ3zrQ%3D&Expires=1625656899&AWSAccessKeyId=AKIA6KOSE3BNJRRFUUX6&versionId=null&response-content-disposition=attachment%3B%20filename%3D%22mecab-0.996-ko-0.9.2.tar.gz%22&response-content-encoding=None\n",
            "Resolving bbuseruploads.s3.amazonaws.com (bbuseruploads.s3.amazonaws.com)... 52.217.141.185\n",
            "Connecting to bbuseruploads.s3.amazonaws.com (bbuseruploads.s3.amazonaws.com)|52.217.141.185|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1414979 (1.3M) [application/x-tar]\n",
            "Saving to: ‘mecab-0.996-ko-0.9.2.tar.gz’\n",
            "\n",
            "mecab-0.996-ko-0.9. 100%[===================>]   1.35M  3.67MB/s    in 0.4s    \n",
            "\n",
            "2021-07-07 11:03:39 (3.67 MB/s) - ‘mecab-0.996-ko-0.9.2.tar.gz’ saved [1414979/1414979]\n",
            "\n",
            "Done\n",
            "Unpacking mecab-0.996-ko-0.9.2.tar.gz.......\n",
            "Done\n",
            "Change Directory to mecab-0.996-ko-0.9.2.......\n",
            "installing mecab-0.996-ko-0.9.2.tar.gz........\n",
            "configure\n",
            "make\n",
            "make check\n",
            "make install\n",
            "ldconfig\n",
            "Done\n",
            "Change Directory to /content\n",
            "Downloading mecab-ko-dic-2.1.1-20180720.tar.gz.......\n",
            "from https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz\n",
            "--2021-07-07 11:04:55--  https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gz\n",
            "Resolving bitbucket.org (bitbucket.org)... 104.192.141.1, 2406:da00:ff00::22c2:513, 2406:da00:ff00::6b17:d1f5, ...\n",
            "Connecting to bitbucket.org (bitbucket.org)|104.192.141.1|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://bbuseruploads.s3.amazonaws.com/a4fcd83e-34f1-454e-a6ac-c242c7d434d3/downloads/b5a0c703-7b64-45ed-a2d7-180e962710b6/mecab-ko-dic-2.1.1-20180720.tar.gz?Signature=QTPh7UcuD5cd%2F2w%2BUdHK7CwfGbA%3D&Expires=1625656983&AWSAccessKeyId=AKIA6KOSE3BNJRRFUUX6&versionId=tzyxc1TtnZU_zEuaaQDGN4F76hPDpyFq&response-content-disposition=attachment%3B%20filename%3D%22mecab-ko-dic-2.1.1-20180720.tar.gz%22&response-content-encoding=None [following]\n",
            "--2021-07-07 11:04:55--  https://bbuseruploads.s3.amazonaws.com/a4fcd83e-34f1-454e-a6ac-c242c7d434d3/downloads/b5a0c703-7b64-45ed-a2d7-180e962710b6/mecab-ko-dic-2.1.1-20180720.tar.gz?Signature=QTPh7UcuD5cd%2F2w%2BUdHK7CwfGbA%3D&Expires=1625656983&AWSAccessKeyId=AKIA6KOSE3BNJRRFUUX6&versionId=tzyxc1TtnZU_zEuaaQDGN4F76hPDpyFq&response-content-disposition=attachment%3B%20filename%3D%22mecab-ko-dic-2.1.1-20180720.tar.gz%22&response-content-encoding=None\n",
            "Resolving bbuseruploads.s3.amazonaws.com (bbuseruploads.s3.amazonaws.com)... 52.216.143.132\n",
            "Connecting to bbuseruploads.s3.amazonaws.com (bbuseruploads.s3.amazonaws.com)|52.216.143.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 49775061 (47M) [application/x-tar]\n",
            "Saving to: ‘mecab-ko-dic-2.1.1-20180720.tar.gz’\n",
            "\n",
            "mecab-ko-dic-2.1.1- 100%[===================>]  47.47M  32.0MB/s    in 1.5s    \n",
            "\n",
            "2021-07-07 11:04:57 (32.0 MB/s) - ‘mecab-ko-dic-2.1.1-20180720.tar.gz’ saved [49775061/49775061]\n",
            "\n",
            "Done\n",
            "Unpacking  mecab-ko-dic-2.1.1-20180720.tar.gz.......\n",
            "Done\n",
            "Change Directory to mecab-ko-dic-2.1.1-20180720\n",
            "Done\n",
            "installing........\n",
            "configure\n",
            "make\n",
            "make install\n",
            "apt-get update\n",
            "apt-get upgrade\n",
            "apt install curl\n",
            "apt install git\n",
            "bash <(curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh)\n",
            "Done\n",
            "Successfully Installed\n",
            "Now you can use Mecab\n",
            "from konlpy.tag import Mecab\n",
            "mecab = Mecab()\n",
            "사용자 사전 추가 방법 : https://bit.ly/3k0ZH53\n",
            "NameError: name 'Tagger' is not defined 오류 발생 시 런타임을 재실행 해주세요\n",
            "블로그에 해결 방법을 남겨주신 tana님 감사합니다.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I64YvthHBI_0"
      },
      "source": [
        "from torchtext.legacy import data\n",
        "from konlpy.tag import Mecab"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PYCwdiGnA03M",
        "outputId": "3cdb48cc-bead-4040-861c-98e8a8638f06"
      },
      "source": [
        "cd /content/drive/MyDrive/torch_example/"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/torch_example\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ONSGndtPBMyr"
      },
      "source": [
        "tokenizer = Mecab()"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dQg-1dMbLVsU",
        "outputId": "8c8f32a9-fe9f-470d-fb11-2f4d34da5125"
      },
      "source": [
        "train = pd.read_csv(\"./data/nsmc/ratings_train.txt\", sep='\\t')\n",
        "test = pd.read_csv(\"./data/nsmc/ratings_test.txt\", sep='\\t')\n",
        "\n",
        "train = train.drop(columns=['id'])\n",
        "test = test.drop(columns=['id'])\n",
        "\n",
        "print(train.shape)\n",
        "print(test.shape)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(150000, 2)\n",
            "(50000, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_EL3Fhl5bDdU",
        "outputId": "10c3730c-3b93-4bba-d2cc-10c9f89f4bad"
      },
      "source": [
        "train_data = train.dropna() #말뭉치에서 nan 값을 제거함\n",
        "test_data = test.dropna()\n",
        "\n",
        "print(train_data.shape)\n",
        "print(test_data.shape)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(149995, 2)\n",
            "(49997, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f7ATQyWmMMCF"
      },
      "source": [
        "train_data, valid_data = train_test_split(train_data, test_size=0.3, random_state=32)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CWEE8eTSMhqh",
        "outputId": "59668d7b-1aca-48ce-ae16-a64786ae7cd6"
      },
      "source": [
        "print(len(train_data))\n",
        "print(len(valid_data))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "104996\n",
            "44999\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NU4u-h4ZBO-6"
      },
      "source": [
        "TEXT = data.Field(sequential=True, use_vocab=True, tokenize=tokenizer.morphs, lower=False, batch_first=True, fix_length=20)\n",
        "LABEL = data.LabelField(dtype = torch.float) "
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8b4ymeVtNPl_"
      },
      "source": [
        "def convert_dataset(input_data, text, label):\n",
        "    list_of_example = [data.Example.fromlist(row.tolist(), fields=[('text', text), ('label', label)])  for _, row in input_data.iterrows()]\n",
        "    dataset = data.Dataset(examples=list_of_example, fields=[('text', text), ('label', label)])\n",
        "    return dataset"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IR41A4m9OBJM"
      },
      "source": [
        "train_data = convert_dataset(train_data,TEXT,LABEL)\n",
        "valid_data = convert_dataset(valid_data, TEXT, LABEL)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3CLzUr8DDkE"
      },
      "source": [
        "TEXT.build_vocab(train_data, max_size=10000)\n",
        "LABEL.build_vocab(train_data)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nFQaqoPnDLaE",
        "outputId": "8e945924-05a2-4f70-fc14-8fccd975a010"
      },
      "source": [
        "print('단어 집합의 크기 : {}'.format(len(TEXT.vocab)))\n",
        "print('label 의 크기 : {}'.format(len(LABEL.vocab)))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "단어 집합의 크기 : 10002\n",
            "label 의 크기 : 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yeoY2_b4DS6E"
      },
      "source": [
        "10000개 단어 + \\<unk> + \\<pad> 토큰이 들어감\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kz5XeWKeEkwG",
        "outputId": "54b67cbc-7e70-4072-8d1c-bac4660f8049"
      },
      "source": [
        "print(LABEL.vocab.stoi) #라벨 값이 "
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "defaultdict(None, {0: 0, 1: 1})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQhMoHCg0ZmN"
      },
      "source": [
        "test_data = convert_dataset(test_data, TEXT, LABEL)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9gGUTpZ2DNvz"
      },
      "source": [
        "batch_size = 5\n",
        "train_iter, valid_iter, test_iter = data.Iterator.splits((train_data, valid_data, test_data), batch_size=batch_size, sort=False, device=device)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Q8mve6nDg7_",
        "outputId": "468608df-668c-4201-dd04-8ad4ddfd0b98"
      },
      "source": [
        "print('훈련 데이터의 미니 배치 수 : {}'.format(len(train_iter)))\n",
        "print('평가 데이터의 미니 배치 수 : {}'.format(len(valid_iter)))\n",
        "print('테스트 데이터의 미니 배치 수 : {}'.format(len(test_iter)))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "훈련 데이터의 미니 배치 수 : 21000\n",
            "평가 데이터의 미니 배치 수 : 9000\n",
            "테스트 데이터의 미니 배치 수 : 10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "06D_jCqWe0Zp"
      },
      "source": [
        "text를 cnn으로 돌릴때, conv2d와 conv1d로 나눠서 돌릴 수 있다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O70kcjWUKJCE"
      },
      "source": [
        "CNN을 1d 로 볼 때, input_chanel은 embed dim이다. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqD-Bk0-dJLU"
      },
      "source": [
        "class CNN1D(nn.Module):\n",
        "  def __init__(self, vocab_size, embedding_dim, n_filters, filter_sizes, output_dim, \n",
        "                 dropout, pad_idx):\n",
        "    super(CNN1D, self).__init__()\n",
        "    self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)\n",
        "\n",
        "    # set n_filters == embeding_dim\n",
        "    self.conv = nn.Conv1d(in_channels=embedding_dim, out_channels=n_filters, kernel_size=filter_sizes, padding='same')\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    self.attention = Self_attn(embedding_dim, n_filters, dropout)\n",
        "    self.layernorm = nn.LayerNorm(n_filters)\n",
        "\n",
        "    self.fc = nn.Linear(n_filters, output_dim)\n",
        "\n",
        "  def forward(self, x):\n",
        "    embedded = self.dropout(self.embedding(x))\n",
        "    #embedded : batch x seq len x embed dim\n",
        "\n",
        "    embedded = embedded.permute(0,2,1).contiguous()\n",
        "    #embedded : batch x embed dim x seq len\n",
        "\n",
        "    conv_out = F.relu(self.conv(embedded))\n",
        "    #conv_out : batch x feature map x (sent len - filter_size[n] + 1)\n",
        "    # == batch * feature map * seq len\n",
        "\n",
        "    conv_out = conv_out.permute(2,0,1)\n",
        "    # conv_out = seq len x batch x feature map\n",
        "\n",
        "    attn, _ =  self.attention(conv_out)\n",
        "    # attn = seq len x batch x channel\n",
        "    output = self.layernorm(attn) + conv_out\n",
        "    # output = seq len x batch x channel\n",
        "\n",
        "    output = output.permute(1, 2, 0).contiguous()\n",
        "    # output = batch x channel x seq len\n",
        "\n",
        "    pooled = F.max_pool1d(output, output.shape[2]).squeeze(2)\n",
        "    # pooled = batch x channel\n",
        "\n",
        "    output = self.fc(pooled)\n",
        "\n",
        "    return output"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0NwaIefVWHD7"
      },
      "source": [
        "class Self_attn(nn.Module):\n",
        "  def __init__(self, emb_dim, out_channels, dropout):\n",
        "    super(Self_attn, self).__init__()\n",
        "    \n",
        "    self.embed_dim = emb_dim\n",
        "    self.out_channels = out_channels\n",
        "\n",
        "    self.in_proj_q = nn.Linear(self.out_channels, self.embed_dim, bias=True)\n",
        "    self.in_proj_k = nn.Linear(self.out_channels, self.embed_dim, bias=True)\n",
        "    self.in_proj_v = nn.Linear(self.out_channels, self.embed_dim, bias=True)\n",
        "\n",
        "    self.out_proj = nn.Linear(self.embed_dim, self.out_channels, bias=True)\n",
        "\n",
        "    self.dropout = nn.Dropout(p=dropout)\n",
        "    self.scaling = self.embed_dim ** -0.5\n",
        "    \n",
        "  def forward(self, x):\n",
        "    '''\n",
        "    inputs :\n",
        "      x : input feature map (seq len x batch x feature map)\n",
        "    '''\n",
        "    query = self.in_proj_q(x)\n",
        "    key = self.in_proj_k(x)\n",
        "    value = self.in_proj_v(x)\n",
        "    # q,k,v = seq len x batch x channel(channel == embed_dim)\n",
        "    \n",
        "    query *= self.scaling\n",
        "\n",
        "    query = query.transpose(0,1)\n",
        "    key = key.transpose(0,1)\n",
        "    value = value.transpose(0,1)\n",
        "\n",
        "    attn_weight = torch.bmm(query, key.transpose(1,2))\n",
        "    # attn_weight = batch x seq len x seq len\n",
        "\n",
        "    attn_weight = F.softmax(attn_weight, dim=-1)\n",
        "    # attn_weight = batch x seq len x seq len\n",
        "\n",
        "    attn_weight = self.dropout(attn_weight)\n",
        "    # attn_weight = batch x seq len x seq len\n",
        "\n",
        "    attn = torch.bmm(attn_weight, value)\n",
        "    # attn = batch x seq len x channel\n",
        "\n",
        "    attn = attn.transpose(0,1).contiguous()\n",
        "    # attn = seq len x batch x channel\n",
        "\n",
        "    attn = self.out_proj(attn)\n",
        "    # attn = seq len x batch x channel\n",
        "\n",
        "    return attn, attn_weight\n"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hH0XYn65xqt5"
      },
      "source": [
        "def binary_accuracy(prediction, target):\n",
        "  '''\n",
        "from https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/1%20-%20Simple%20Sentiment%20Analysis.ipynb\n",
        "  '''\n",
        "  # round predictions to the closest integer (0 or 1)\n",
        "  rounded_preds = torch.round(torch.sigmoid(prediction))\n",
        "  \n",
        "  #convert into float for division\n",
        "  correct = (rounded_preds == target).float()\n",
        "\n",
        "  # rounded_preds = [ 1   0   0   1   1   1   0   1   1   1]\n",
        "  # targets       = [ 1   0   1   1   1   1   0   1   1   0]\n",
        "  # correct       = [1.0 1.0 0.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0]\n",
        "  acc = correct.sum() / len(correct)\n",
        "  return acc"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9MgSjQr3sa5i"
      },
      "source": [
        "def train(model, train_iter):\n",
        "  model.train()\n",
        "\n",
        "  epoch_loss, epoch_acc = 0, 0\n",
        "  for batch in train_iter:\n",
        "    optimizer.zero_grad()\n",
        "    x, y = batch.text.to(device), batch.label.to(device)\n",
        "    y_hat = model(x).squeeze(1)\n",
        "    \n",
        "    loss = criterion(y_hat, y)\n",
        "    acc = binary_accuracy(y_hat, y)\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    epoch_loss += loss.item()\n",
        "    epoch_acc += acc.item()\n",
        "\n",
        "  return epoch_loss / len(train_iter), epoch_acc / len(train_iter)\n"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZkuKVVyCzTF8"
      },
      "source": [
        "def evaluate(model, valid_iter):\n",
        "  model.eval()\n",
        "  with torch.no_grad():\n",
        "    eval_loss, eval_acc = 0, 0\n",
        "    for batch in valid_iter:\n",
        "      x,y = batch.text.to(device), batch.label.to(device)\n",
        "      y_hat = model(x).squeeze(1)\n",
        "      \n",
        "      loss = criterion(y_hat, y)\n",
        "      acc = binary_accuracy(y_hat, y)\n",
        "\n",
        "      eval_loss += loss.item()\n",
        "      eval_acc += acc.item()\n",
        "      \n",
        "  return eval_loss / len(valid_iter), eval_acc / len(valid_iter)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ss8RYLuN3QWN"
      },
      "source": [
        "def inference(model, test_iter):\n",
        "  model.eval()\n",
        "  with torch.no_grad():\n",
        "    test_loss, test_acc = 0, 0\n",
        "    for batch in test_iter:\n",
        "      x, y = batch.text.to(device), batch.label.to(device)\n",
        "      y_hat = model(x).squeeze(1)\n",
        "\n",
        "      loss = criterion(y_hat, y)\n",
        "      acc = binary_accuracy(y_hat, y)\n",
        "      \n",
        "      test_loss += loss.item()\n",
        "      test_acc += acc.item()\n",
        "\n",
        "  return test_loss / len(test_iter), test_acc / len(test_iter)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ilZJ-N2M1dcD",
        "outputId": "0f93d1e6-8589-4342-9de4-1818fef1ea56"
      },
      "source": [
        "#  vocab_size, embedding_dim, n_filters, filter_sizes, output_dim, dropout, pad_idx\n",
        "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
        "\n",
        "model = CNN1D(len(TEXT.vocab), 128, 128, 3, 1, 0.2, PAD_IDX)\n",
        "model.to(device)\n",
        "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
        "criterion = nn.BCEWithLogitsLoss()\n",
        "criterion.to(device)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BCEWithLogitsLoss()"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DShy6pep2BIO",
        "outputId": "4b6475aa-a530-4a5b-85be-3fd23fc48538"
      },
      "source": [
        "best_val_loss = float('inf')\n",
        "for _epoch in range(1,6): # 5 epoch\n",
        "  train_loss, train_acc = train(model, train_iter)\n",
        "  valid_loss, valid_acc = evaluate(model, valid_iter)\n",
        "  print(\"[Epoch: %d] train loss : %5.2f | train accuracy : %5.2f\" % (_epoch, train_loss, train_acc))\n",
        "  print(\"[Epoch: %d] val loss : %5.2f | val accuracy : %5.2f\" % (_epoch, valid_loss, valid_acc))\n",
        "\n",
        "    # 검증 오차가 가장 적은 최적의 모델을 저장\n",
        "  if valid_loss < best_val_loss:\n",
        "    best_val_loss = valid_loss"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[Epoch: 1] train loss :  0.45 | train accuracy :  0.79\n",
            "[Epoch: 1] val loss :  0.39 | val accuracy :  0.82\n",
            "[Epoch: 2] train loss :  0.37 | train accuracy :  0.84\n",
            "[Epoch: 2] val loss :  0.37 | val accuracy :  0.84\n",
            "[Epoch: 3] train loss :  0.34 | train accuracy :  0.86\n",
            "[Epoch: 3] val loss :  0.36 | val accuracy :  0.84\n",
            "[Epoch: 4] train loss :  0.31 | train accuracy :  0.87\n",
            "[Epoch: 4] val loss :  0.38 | val accuracy :  0.84\n",
            "[Epoch: 5] train loss :  0.29 | train accuracy :  0.88\n",
            "[Epoch: 5] val loss :  0.37 | val accuracy :  0.84\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZP8Lqf7n5CgO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ed4c7f2-d902-4400-d01d-8c13bc0009b7"
      },
      "source": [
        "test_loss, test_acc = inference(model, test_iter)\n",
        "print('Test Loss: %5.2f | Test Acc: %5.2f '%(test_loss, test_acc*100))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Loss:  0.37 | Test Acc: 84.28 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFeqcwhpeIoV"
      },
      "source": [
        ""
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0yqmiLYRgVfO"
      },
      "source": [
        ""
      ],
      "execution_count": 29,
      "outputs": []
    }
  ]
}