POS Tagging modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import re\n",
    "import string\n",
    "\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.probability import FreqDist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"txt/dilan.txt\", \"r\")\n",
    "text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def case_folding(text):\n",
    "    return text.lower()\n",
    "\n",
    "def strip_number(text):\n",
    "    return re.sub(r\"\\d+\", \"\", text)\n",
    "\n",
    "def strip_whitespace(text):\n",
    "    return re.sub('\\s+',' ',text.strip())\n",
    "\n",
    "def remove_punc_num(text):\n",
    "#     text = re.sub(r'([.*]){2,}','.',text)\n",
    "#     return re.sub(r'([^a-z\\s])(([\\-,.*]){2,})','',text)\n",
    "    return text.translate(str.maketrans('','',string.punctuation))\n",
    "\n",
    "def sentence_tokenizing(text):\n",
    "    return sent_tokenize(text)\n",
    "\n",
    "def word_tokenizing(text):\n",
    "    return word_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['dari', 'kantin', 'sebelum', 'mau', 'masuk', 'ke', 'kelas', 'aku', 'berpapasan', 'dengan', 'dilan', 'dia', 'jalan', 'bersama', 'kawankawannya', 'pasti', 'baru', 'datang', 'dari', 'warung', 'bi', 'eem', 'milea', 'dia', 'memanggil', 'dan', 'mendekat', 'kuhentikan', 'langkahku', 'nandan', 'hadi', 'dan', 'rani', 'terus', 'berjalan', 'karena', 'aku', 'minta', 'untuk', 'duluan', 'boleh', 'gak', 'aku', 'ikut', 'pelajaran', 'di', 'kelasmu', 'lagi', 'dia', 'senyum', 'aku', 'juga', 'kamu', 'mau', 'bikin', 'aku', 'senang', 'gak', 'kupandang', 'matanya', 'hampirhampir', 'gak', 'percaya', 'bahwa', 'aku', 'bisa', 'nanya', 'seberani', 'itu', 'kepadanya']\n"
     ]
    }
   ],
   "source": [
    "case_folded = case_folding(text)\n",
    "filtered = strip_whitespace(case_folded)\n",
    "filtered = remove_punc_num(filtered)\n",
    "sent_token = sentence_tokenizing(filtered)\n",
    "word_token = word_tokenizing(filtered)\n",
    "\n",
    "print(word_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tag import CRFTagger\n",
    "\n",
    "jumSample = 500000\n",
    "namaFile = \"B:\\Data\\koding\\python\\datasets/Indonesian_Manually_Tagged_Corpus.tsv\"\n",
    "\n",
    "with open(namaFile, 'r', encoding='utf-8') as f:\n",
    "    lines = f.read().split('\\n')\n",
    "\n",
    "pasangan = []\n",
    "allPasangan = []\n",
    "\n",
    "for line in lines[: min(jumSample, len(lines))]:\n",
    "    if line == '':\n",
    "        allPasangan.append(pasangan)\n",
    "        pasangan = []\n",
    "    else:\n",
    "        kata, tag = line.split('\\t')\n",
    "        p = (kata,tag)\n",
    "        pasangan.append(p)\n",
    "\n",
    "# train data model\n",
    "ct = CRFTagger()\n",
    "ct.train(allPasangan,'all_indo_man_tag_corpus_model.crf.tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('aku', 'NN'), ('makan', 'VB'), ('nasi', 'NN'), ('goreng', 'NN')], [('beli', 'VB'), ('buku', 'NN'), ('baru', 'JJ')]]\n"
     ]
    }
   ],
   "source": [
    "#test\n",
    "hasil = ct.tag_sents([['aku','makan','nasi','goreng'],['beli','buku','baru']])\n",
    "print(hasil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('aku', 'NN'), ('menggoreng', 'NN'), ('nasi', 'NN'), ('kemarin', 'NN')], [('dia', 'PRP'), ('pergi', 'VB'), ('besok', 'NN')]]\n"
     ]
    }
   ],
   "source": [
    "#test\n",
    "hasil = ct.tag_sents([['aku','menggoreng','nasi','kemarin'],['dia','pergi','besok']])\n",
    "print(hasil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('kursimu', 'VB'), ('mobilmu', 'NN'), ('sepeda', 'VB')]]\n"
     ]
    }
   ],
   "source": [
    "x = ct.tag_sents([['kursimu','mobilmu','sepeda']])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}