Project Final Solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final NLP project (University of Guilan)\n",
    "In this repository, we implemented a Statistical NLP model to predict news agency, news tags, etc as final project of NLP course in university of Guilan."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Contents\n",
    "* Libraries and Constants\n",
    "* Importing Data\n",
    "* Preprocessing\n",
    "* Creating Model for the First Task\n",
    "* Creating Model for the Second Task\n",
    "* Creating Model for the Third Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals\n",
    "\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.optimize as opt\n",
    "from functools import reduce\n",
    "from operator import add\n",
    "from hazm import *\n",
    "from utils.preprocessing import *\n",
    "from utils.models import *\n",
    "from copy import deepcopy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data root path\n",
    "data_root = 'data'\n",
    "fars_file = 'farsnews.jsonl'\n",
    "asriran_file = 'asriran.jsonl'\n",
    "# Dataset dataframe column names\n",
    "keys = None\n",
    "\n",
    "# News headline tags\n",
    "not_valid_labels = None\n",
    "\n",
    "# News agencies\n",
    "news_agencies = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Datapoints: 15000\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(data_root, asriran_file), encoding='utf-8') as jd:\n",
    "    asriran = [json.loads(line) for line in jd]\n",
    "    asriran = pd.DataFrame(asriran)\n",
    "print('Number of Datapoints: {}'.format(len(asriran)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Datapoints: 15000\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(data_root, fars_file), encoding='utf-8') as jd:\n",
    "    fars = [json.loads(line) for line in jd]\n",
    "    fars = pd.DataFrame(fars)\n",
    "print('Number of Datapoints: {}'.format(len(fars)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding Valid Labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "asr_labels = list(set(reduce(np.append, asriran.newsPathLinks.apply(lambda x: tuple(x.keys())))))\n",
    "fars_labels = list(set(reduce(np.append, fars.newsPathLinks.apply(lambda x: list(x.keys())))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "set((list(asr_labels) + list(fars_labels)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some labels are not valid so:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_valid_labels = [\n",
    "     'دانلود',\n",
    "     'ساير حوزه ها',\n",
    "     'سایر حوزه ها',\n",
    "     'دیگر رسانه ها',\n",
    "     'نامشخص',\n",
    "     'پیامک',\n",
    "     'صفحه نخست',\n",
    "     'عصرايران دو',\n",
    "]\n",
    "valid_labels = list(filter(lambda x: x not in not_valid_labels, list(set((list(asr_labels) + list(fars_labels))))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating Documents & Labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "asriran_tags = asriran['tags'].apply(lambda x: ' '.join(list(x.keys())))\n",
    "fars_tags = fars['tags'].apply(lambda x: ' '.join(list(x.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_count = 2\n",
    "tag_count = 10\n",
    "documents = np.append(asriran['body'] + ' ' + asriran['title'] * title_count + asriran_tags*tag_count,\n",
    "                        fars['body'] + ' ' + fars['title'] * title_count + fars_tags*tag_count)\n",
    "raw_labels = np.append(asriran.newsPathLinks.apply(lambda x: tuple(x.keys())),\n",
    "                        fars.newsPathLinks.apply(lambda x: tuple(x.keys())))\n",
    "org_labels = np.append( ['AsrIran'] * len(asriran), ['Fars'] * len(fars)) # For the third task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing Documents which are emtpy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "none_zero_docs = list(map(lambda x: len(x) > 1, documents))\n",
    "documents = documents[none_zero_docs]\n",
    "raw_labels = cleans_labels(raw_labels[none_zero_docs], valid_labels)\n",
    "org_labels = org_labels[none_zero_docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Duplicating documents for each of their labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "proc_documents, proc_labels = extend_labels(documents, raw_labels, valid_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalizing & Tokenizing & Removing Stopwords Documents:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer = Normalizer()\n",
    "word_filter = WordFilter()\n",
    "documents = list(pd.Series(documents).apply(normalizer.normalize).apply(tokenize).apply(word_filter.filter_words))\n",
    "proc_documents = list(proc_documents.apply(normalizer.normalize).apply(tokenize).apply(word_filter.filter_words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Replacing words with less than 2 occurances with unknown word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "created\n",
      "created\n"
     ]
    }
   ],
   "source": [
    "documents = make_unknown(documents)\n",
    "proc_documents = make_unknown(proc_documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making documents one hot encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_set, proc_labels = one_hot_encoder(proc_labels)\n",
    "label_set_th, org_labels = one_hot_encoder(org_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deviding document to train and test datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, y_train, x_test, y_test = train_test_split(proc_documents , proc_labels, train_size = 0.80, random_state=85)\n",
    "x_train_th, y_train_th, x_test_th, y_test_th = train_test_split(documents , org_labels, train_size = 0.80, random_state=85)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Model for the First Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab created\n",
      "P(c) calculated\n",
      "93\n",
      "%0.0 continue...\n",
      "%0.010752688172043012 continue...\n",
      "%0.021505376344086023 continue...\n",
      "%0.03225806451612903 continue...\n",
      "%0.043010752688172046 continue...\n",
      "%0.053763440860215055 continue...\n",
      "%0.06451612903225806 continue...\n",
      "%0.07526881720430108 continue...\n",
      "%0.08602150537634409 continue...\n",
      "%0.0967741935483871 continue...\n",
      "%0.10752688172043011 continue...\n",
      "%0.11827956989247312 continue...\n",
      "%0.12903225806451613 continue...\n",
      "%0.13978494623655913 continue...\n",
      "%0.15053763440860216 continue...\n",
      "%0.16129032258064516 continue...\n",
      "%0.17204301075268819 continue...\n",
      "%0.1827956989247312 continue...\n",
      "%0.1935483870967742 continue...\n",
      "%0.20430107526881722 continue...\n",
      "%0.21505376344086022 continue...\n",
      "%0.22580645161290322 continue...\n",
      "%0.23655913978494625 continue...\n",
      "%0.24731182795698925 continue...\n",
      "%0.25806451612903225 continue...\n",
      "%0.26881720430107525 continue...\n",
      "%0.27956989247311825 continue...\n",
      "%0.2903225806451613 continue...\n",
      "%0.3010752688172043 continue...\n",
      "%0.3118279569892473 continue...\n",
      "%0.3225806451612903 continue...\n",
      "%0.3333333333333333 continue...\n",
      "%0.34408602150537637 continue...\n",
      "%0.3548387096774194 continue...\n",
      "%0.3655913978494624 continue...\n",
      "%0.3763440860215054 continue...\n",
      "%0.3870967741935484 continue...\n",
      "%0.3978494623655914 continue...\n",
      "%0.40860215053763443 continue...\n",
      "%0.41935483870967744 continue...\n",
      "%0.43010752688172044 continue...\n",
      "%0.44086021505376344 continue...\n",
      "%0.45161290322580644 continue...\n",
      "%0.46236559139784944 continue...\n",
      "%0.4731182795698925 continue...\n",
      "%0.4838709677419355 continue...\n",
      "%0.4946236559139785 continue...\n",
      "%0.5053763440860215 continue...\n",
      "%0.5161290322580645 continue...\n",
      "%0.5268817204301075 continue...\n",
      "%0.5376344086021505 continue...\n",
      "%0.5483870967741935 continue...\n",
      "%0.5591397849462365 continue...\n",
      "%0.5698924731182796 continue...\n",
      "%0.5806451612903226 continue...\n",
      "%0.5913978494623656 continue...\n",
      "%0.6021505376344086 continue...\n",
      "%0.6129032258064516 continue...\n",
      "%0.6236559139784946 continue...\n",
      "%0.6344086021505376 continue...\n",
      "%0.6451612903225806 continue...\n",
      "%0.6559139784946236 continue...\n",
      "%0.6666666666666666 continue...\n",
      "%0.6774193548387096 continue...\n",
      "%0.6881720430107527 continue...\n",
      "%0.6989247311827957 continue...\n",
      "%0.7096774193548387 continue...\n",
      "%0.7204301075268817 continue...\n",
      "%0.7311827956989247 continue...\n",
      "%0.7419354838709677 continue...\n",
      "%0.7526881720430108 continue...\n",
      "%0.7634408602150538 continue...\n",
      "%0.7741935483870968 continue...\n",
      "%0.7849462365591398 continue...\n",
      "%0.7956989247311828 continue...\n",
      "%0.8064516129032258 continue...\n",
      "%0.8172043010752689 continue...\n",
      "%0.8279569892473119 continue...\n",
      "%0.8387096774193549 continue...\n",
      "%0.8494623655913979 continue...\n",
      "%0.8602150537634409 continue...\n",
      "%0.8709677419354839 continue...\n",
      "%0.8817204301075269 continue...\n",
      "%0.8924731182795699 continue...\n",
      "%0.9032258064516129 continue...\n",
      "%0.9139784946236559 continue...\n",
      "%0.9247311827956989 continue...\n",
      "%0.9354838709677419 continue...\n",
      "%0.946236559139785 continue...\n",
      "%0.956989247311828 continue...\n",
      "%0.967741935483871 continue...\n",
      "%0.978494623655914 continue...\n",
      "%0.989247311827957 continue...\n",
      "P(w|c) calculated\n"
     ]
    }
   ],
   "source": [
    "nb = NaiveBayes()\n",
    "nb.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%0 continue...\n",
      "%1000 continue...\n",
      "%2000 continue...\n",
      "%3000 continue...\n",
      "%4000 continue...\n",
      "%5000 continue...\n",
      "%6000 continue...\n",
      "%7000 continue...\n",
      "%8000 continue...\n",
      "%9000 continue...\n",
      "%10000 continue...\n",
      "%11000 continue...\n",
      "%12000 continue...\n",
      "%13000 continue...\n",
      "%14000 continue...\n",
      "%15000 continue...\n",
      "%16000 continue...\n",
      "%17000 continue...\n",
      "%18000 continue...\n",
      "%19000 continue...\n",
      "%20000 continue...\n",
      "%21000 continue...\n",
      "%22000 continue...\n",
      "%23000 continue...\n",
      "%24000 continue...\n",
      "%25000 continue...\n",
      "%26000 continue...\n",
      "%27000 continue...\n",
      "%28000 continue...\n",
      "%29000 continue...\n",
      "%30000 continue...\n",
      "%31000 continue...\n",
      "Label مسئولیت های اجتماعی: \n",
      "     Precision: 0.4745762711864407\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6436781609195402\n",
      "Label صنعت ، تجارت ، بازرگانی: \n",
      "     Precision: 0.5139664804469274\n",
      "     Recall: 0.9787234042553191\n",
      "     F1-Measure: 0.673992673992674\n",
      "Label ایران در جهان: \n",
      "     Precision: 0.5211267605633803\n",
      "     Recall: 0.9024390243902439\n",
      "     F1-Measure: 0.6607142857142856\n",
      "Label شهری: \n",
      "     Precision: 0.472\n",
      "     Recall: 0.9874476987447699\n",
      "     F1-Measure: 0.638700947225981\n",
      "Label غرب از نگاه غرب: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label خانواده: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label تور و توپ: \n",
      "     Precision: 0.49710982658959535\n",
      "     Recall: 0.9347826086956522\n",
      "     F1-Measure: 0.6490566037735849\n",
      "Label فوتبال ایران: \n",
      "     Precision: 0.5058721183123097\n",
      "     Recall: 0.917192429022082\n",
      "     F1-Measure: 0.6520885898514158\n",
      "Label علمی: \n",
      "     Precision: 0.8245614035087719\n",
      "     Recall: 0.7704918032786885\n",
      "     F1-Measure: 0.7966101694915254\n",
      "Label اجتماعی: \n",
      "     Precision: 0.9508361204013378\n",
      "     Recall: 0.5859439406430338\n",
      "     F1-Measure: 0.7250701351695996\n",
      "Label سرگرمی: \n",
      "     Precision: 0.07942238267148015\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.14715719063545152\n",
      "Label مسجد و هیئت: \n",
      "     Precision: 0.5\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6666666666666666\n",
      "Label فرهنگ و هنر: \n",
      "     Precision: 0.6198347107438017\n",
      "     Recall: 0.08269018743109151\n",
      "     F1-Measure: 0.14591439688715954\n",
      "Label احزاب و تشکل ها: \n",
      "     Precision: 0.5092592592592593\n",
      "     Recall: 0.990990990990991\n",
      "     F1-Measure: 0.672782874617737\n",
      "Label پاکستان: \n",
      "     Precision: 0.5179856115107914\n",
      "     Recall: 0.9113924050632911\n",
      "     F1-Measure: 0.6605504587155964\n",
      "Label بورس: \n",
      "     Precision: 0.47468354430379744\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6437768240343348\n",
      "Label گروههای توان خواه: \n",
      "     Precision: 0.41975308641975306\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.5913043478260869\n",
      "Label بازار: \n",
      "     Precision: 0.5483870967741935\n",
      "     Recall: 0.6891891891891891\n",
      "     F1-Measure: 0.6107784431137725\n",
      "Label حماسه و مقاومت: \n",
      "     Precision: 0.6666666666666666\n",
      "     Recall: 0.6666666666666666\n",
      "     F1-Measure: 0.6666666666666666\n",
      "Label خبر خوب: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label آفریقا: \n",
      "     Precision: 0.48878923766816146\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6566265060240963\n",
      "Label زنان و جوانان: \n",
      "     Precision: 0.5066666666666667\n",
      "     Recall: 0.95\n",
      "     F1-Measure: 0.6608695652173914\n",
      "Label مجلس: \n",
      "     Precision: 0.4743169398907104\n",
      "     Recall: 0.9455337690631809\n",
      "     F1-Measure: 0.6317321688500728\n",
      "Label تاریخ: \n",
      "     Precision: 0.6153846153846154\n",
      "     Recall: 0.8888888888888888\n",
      "     F1-Measure: 0.7272727272727274\n",
      "Label جنگ اقتصادی: \n",
      "     Precision: 0.5\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6666666666666666\n",
      "Label سینما و تئاتر: \n",
      "     Precision: 0.48604651162790696\n",
      "     Recall: 0.9766355140186916\n",
      "     F1-Measure: 0.6490683229813664\n",
      "Label داستان کوتاه: \n",
      "     Precision: 0.9\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.9473684210526316\n",
      "Label استانها: \n",
      "     Precision: 0.9779673063255153\n",
      "     Recall: 0.8380024360535931\n",
      "     F1-Measure: 0.9025910134470319\n",
      "Label انقلاب اسلامی: \n",
      "     Precision: nan\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label علم و فن آوری جهان: \n",
      "     Precision: 0.4959016393442623\n",
      "     Recall: 0.9758064516129032\n",
      "     F1-Measure: 0.6576086956521738\n",
      "Label اندیشه: \n",
      "     Precision: 0.4583333333333333\n",
      "     Recall: 0.9428571428571428\n",
      "     F1-Measure: 0.616822429906542\n",
      "Label امام و رهبری: \n",
      "     Precision: 0.53125\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6938775510204082\n",
      "Label شرق آسیا و اقیانوسیه: \n",
      "     Precision: 0.44285714285714284\n",
      "     Recall: 0.9393939393939394\n",
      "     F1-Measure: 0.6019417475728155\n",
      "Label تحلیل بین الملل: \n",
      "     Precision: nan\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label آسياي مرکزی و روسيه: \n",
      "     Precision: 0.5449591280653951\n",
      "     Recall: 0.8547008547008547\n",
      "     F1-Measure: 0.6655574043261231\n",
      "Label ورزش بانوان: \n",
      "     Precision: 0.5094339622641509\n",
      "     Recall: 0.7941176470588235\n",
      "     F1-Measure: 0.6206896551724137\n",
      "Label فرهنگی/هنری: \n",
      "     Precision: 0.8955512572533849\n",
      "     Recall: 0.954639175257732\n",
      "     F1-Measure: 0.9241516966067864\n",
      "Label فناوری و IT: \n",
      "     Precision: 0.86\n",
      "     Recall: 0.9662921348314607\n",
      "     F1-Measure: 0.9100529100529101\n",
      "Label حوادث: \n",
      "     Precision: 0.4929078014184397\n",
      "     Recall: 0.9686411149825784\n",
      "     F1-Measure: 0.6533490011750881\n",
      "Label آمریکا، اروپا: \n",
      "     Precision: 0.4913232104121475\n",
      "     Recall: 0.9476987447698745\n",
      "     F1-Measure: 0.6471428571428571\n",
      "Label ویژه نامه ها: \n",
      "     Precision: 0.6111111111111112\n",
      "     Recall: 0.4520547945205479\n",
      "     F1-Measure: 0.5196850393700787\n",
      "Label ورزش بین الملل: \n",
      "     Precision: 0.47424511545293074\n",
      "     Recall: 0.9501779359430605\n",
      "     F1-Measure: 0.6327014218009479\n",
      "Label آموزش و پرورش: \n",
      "     Precision: 0.4610951008645533\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6311637080867849\n",
      "Label محور مقاومت: \n",
      "     Precision: nan\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label حج و زیارت و وقف: \n",
      "     Precision: 0.48314606741573035\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6515151515151515\n",
      "Label اقتصادی: \n",
      "     Precision: 0.853990914990266\n",
      "     Recall: 0.548790658882402\n",
      "     F1-Measure: 0.6681898959126681\n",
      "Label قرآن و فعالیت های دینی: \n",
      "     Precision: 0.4769874476987448\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6458923512747875\n",
      "Label تشکل های دانشگاهی: \n",
      "     Precision: 0.48520710059171596\n",
      "     Recall: 0.9761904761904762\n",
      "     F1-Measure: 0.6482213438735178\n",
      "Label کتاب و ادبیات: \n",
      "     Precision: 0.5018450184501845\n",
      "     Recall: 0.9645390070921985\n",
      "     F1-Measure: 0.6601941747572815\n",
      "Label رسانه: \n",
      "     Precision: 0.5571428571428572\n",
      "     Recall: 0.8863636363636364\n",
      "     F1-Measure: 0.6842105263157894\n",
      "Label محیط زیست و گردشگری: \n",
      "     Precision: 0.5348837209302325\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6969696969696969\n",
      "Label عمومی: \n",
      "     Precision: 0.8790322580645161\n",
      "     Recall: 0.9316239316239316\n",
      "     F1-Measure: 0.904564315352697\n",
      "Label سلامت: \n",
      "     Precision: 0.6308186195826645\n",
      "     Recall: 0.8308668076109936\n",
      "     F1-Measure: 0.7171532846715328\n",
      "Label دفاتر منطقه ای: \n",
      "     Precision: 0.7094594594594594\n",
      "     Recall: 0.30973451327433627\n",
      "     F1-Measure: 0.4312114989733059\n",
      "Label تعاون و اشتغال: \n",
      "     Precision: 0.46511627906976744\n",
      "     Recall: 0.9302325581395349\n",
      "     F1-Measure: 0.6201550387596899\n",
      "Label کشاورزی و امور دام: \n",
      "     Precision: 0.4423076923076923\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6133333333333333\n",
      "Label ورزشی: \n",
      "     Precision: 0.877164056059357\n",
      "     Recall: 0.30201532784558616\n",
      "     F1-Measure: 0.4493243243243243\n",
      "Label رفاه و آسیب های اجتماعی: \n",
      "     Precision: 0.49056603773584906\n",
      "     Recall: 0.9512195121951219\n",
      "     F1-Measure: 0.6473029045643154\n",
      "Label راه و مسکن: \n",
      "     Precision: 0.4273972602739726\n",
      "     Recall: 0.968944099378882\n",
      "     F1-Measure: 0.5931558935361217\n",
      "Label غرب آسیا: \n",
      "     Precision: 0.4952015355086372\n",
      "     Recall: 0.9608938547486033\n",
      "     F1-Measure: 0.6535782140595312\n",
      "Label صنفی فرهنگی: \n",
      "     Precision: 0.46534653465346537\n",
      "     Recall: 0.9215686274509803\n",
      "     F1-Measure: 0.618421052631579\n",
      "Label انتظامی و حوادث: \n",
      "     Precision: 0.4698275862068966\n",
      "     Recall: 0.9478260869565217\n",
      "     F1-Measure: 0.6282420749279539\n",
      "Label آموزش: \n",
      "     Precision: 0.42786069651741293\n",
      "     Recall: 0.819047619047619\n",
      "     F1-Measure: 0.5620915032679739\n",
      "Label خواندنی ها و دیدنی ها: \n",
      "     Precision: 0.94140625\n",
      "     Recall: 0.9836734693877551\n",
      "     F1-Measure: 0.9620758483033932\n",
      "Label روانشناسی: \n",
      "     Precision: 0.7391304347826086\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.85\n",
      "Label موسیقی و هنرهای تجسمی: \n",
      "     Precision: 0.49514563106796117\n",
      "     Recall: 0.9902912621359223\n",
      "     F1-Measure: 0.6601941747572816\n",
      "Label مجله فارس پلاس: \n",
      "     Precision: 0.9558823529411765\n",
      "     Recall: 0.8783783783783784\n",
      "     F1-Measure: 0.9154929577464789\n",
      "Label دیدگاه: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label بین الملل: \n",
      "     Precision: 0.9417870036101083\n",
      "     Recall: 0.5981656635139009\n",
      "     F1-Measure: 0.7316389132340052\n",
      "Label ارتباطات و فن آوری اطلاعات: \n",
      "     Precision: 0.4858757062146893\n",
      "     Recall: 0.9885057471264368\n",
      "     F1-Measure: 0.6515151515151515\n",
      "Label رزمی: \n",
      "     Precision: 0.49693251533742333\n",
      "     Recall: 0.9642857142857143\n",
      "     F1-Measure: 0.6558704453441296\n",
      "Label فرهنگ عمومی: \n",
      "     Precision: 0.6666666666666666\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.8\n",
      "Label علمی و دانشگاهی: \n",
      "     Precision: 0.5789473684210527\n",
      "     Recall: 0.2515592515592516\n",
      "     F1-Measure: 0.35072463768115947\n",
      "Label سیاست خارجی: \n",
      "     Precision: 0.5814332247557004\n",
      "     Recall: 0.9248704663212435\n",
      "     F1-Measure: 0.714\n",
      "Label بیمه و بانک: \n",
      "     Precision: 0.5\n",
      "     Recall: 0.9692307692307692\n",
      "     F1-Measure: 0.6596858638743455\n",
      "Label حقوقی و قضایی: \n",
      "     Precision: 0.4411764705882353\n",
      "     Recall: 0.967741935483871\n",
      "     F1-Measure: 0.6060606060606061\n",
      "Label رادیو و تلویزیون: \n",
      "     Precision: 0.48562300319488816\n",
      "     Recall: 0.9806451612903225\n",
      "     F1-Measure: 0.6495726495726495\n",
      "Label امنیتی و دفاعی: \n",
      "     Precision: 0.4251968503937008\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.5966850828729281\n",
      "Label پژوهش: \n",
      "     Precision: 0.5\n",
      "     Recall: 0.2\n",
      "     F1-Measure: 0.28571428571428575\n",
      "Label اقتصاد بین الملل: \n",
      "     Precision: 0.430622009569378\n",
      "     Recall: 0.989010989010989\n",
      "     F1-Measure: 0.6000000000000001\n",
      "Label سفر و تفریح: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label سیاسی: \n",
      "     Precision: 0.8949720670391061\n",
      "     Recall: 0.5491943777853959\n",
      "     F1-Measure: 0.6806883365200764\n",
      "Label اطلاعات عمومی و دانستنی ها: \n",
      "     Precision: 0.5192307692307693\n",
      "     Recall: 0.8852459016393442\n",
      "     F1-Measure: 0.6545454545454545\n",
      "Label فارس من: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label آشپزی: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label نفت و انرژی: \n",
      "     Precision: 0.4876847290640394\n",
      "     Recall: 0.9801980198019802\n",
      "     F1-Measure: 0.6513157894736842\n",
      "Label دولت: \n",
      "     Precision: 0.45161290322580644\n",
      "     Recall: 0.8828828828828829\n",
      "     F1-Measure: 0.5975609756097561\n",
      "Label شبهه و شایعه: \n",
      "     Precision: 1.0\n",
      "     Recall: 0.9375\n",
      "     F1-Measure: 0.967741935483871\n",
      "Label فوتبال جهان: \n",
      "     Precision: 0.4669631512071156\n",
      "     Recall: 0.98\n",
      "     F1-Measure: 0.6325301204819277\n",
      "Label کشتی و وزنه برداری: \n",
      "     Precision: 0.484375\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6526315789473685\n",
      "Label علم و فن آوری ایران: \n",
      "     Precision: 0.5808823529411765\n",
      "     Recall: 0.79\n",
      "     F1-Measure: 0.6694915254237288\n",
      "Label اقتصاد کلان و بودجه: \n",
      "     Precision: 0.44029850746268656\n",
      "     Recall: 0.9833333333333333\n",
      "     F1-Measure: 0.6082474226804124\n",
      "Label تاریخ معاصر: \n",
      "     Precision: 0.6666666666666666\n",
      "     Recall: 0.2857142857142857\n",
      "     F1-Measure: 0.4\n",
      "Total Accuracy: 0.6632662759385395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Erfan\\AnacondaProjects\\github\\NLP-Fall18-UOG\\utils\\models.py:89: RuntimeWarning: invalid value encountered in true_divide\n",
      "  percision = confusion_matrix.diagonal()/np.sum(confusion_matrix, axis=1)\n"
     ]
    }
   ],
   "source": [
    "nb.evaluate(x_train, y_train, label_set=label_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%0 continue...\n",
      "%1000 continue...\n",
      "%2000 continue...\n",
      "%3000 continue...\n",
      "%4000 continue...\n",
      "%5000 continue...\n",
      "%6000 continue...\n",
      "%7000 continue...\n",
      "Label مسئولیت های اجتماعی: \n",
      "     Precision: 0.5384615384615384\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.7000000000000001\n",
      "Label صنعت ، تجارت ، بازرگانی: \n",
      "     Precision: 0.358974358974359\n",
      "     Recall: 0.7777777777777778\n",
      "     F1-Measure: 0.49122807017543857\n",
      "Label ایران در جهان: \n",
      "     Precision: 0.38461538461538464\n",
      "     Recall: 0.5555555555555556\n",
      "     F1-Measure: 0.4545454545454546\n",
      "Label شهری: \n",
      "     Precision: 0.46551724137931033\n",
      "     Recall: 0.8852459016393442\n",
      "     F1-Measure: 0.6101694915254237\n",
      "Label غرب از نگاه غرب: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label خانواده: \n",
      "     Precision: 1.0\n",
      "     Recall: 0.3333333333333333\n",
      "     F1-Measure: 0.5\n",
      "Label تور و توپ: \n",
      "     Precision: 0.4186046511627907\n",
      "     Recall: 0.6\n",
      "     F1-Measure: 0.49315068493150693\n",
      "Label فوتبال ایران: \n",
      "     Precision: 0.4357429718875502\n",
      "     Recall: 0.7331081081081081\n",
      "     F1-Measure: 0.5465994962216625\n",
      "Label علمی: \n",
      "     Precision: 0.16666666666666666\n",
      "     Recall: 0.09090909090909091\n",
      "     F1-Measure: 0.11764705882352942\n",
      "Label اجتماعی: \n",
      "     Precision: 0.7337748344370861\n",
      "     Recall: 0.4855390008764242\n",
      "     F1-Measure: 0.5843881856540084\n",
      "Label سرگرمی: \n",
      "     Precision: 0.15384615384615385\n",
      "     Recall: 0.9090909090909091\n",
      "     F1-Measure: 0.26315789473684215\n",
      "Label مسجد و هیئت: \n",
      "     Precision: 0.5\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6666666666666666\n",
      "Label فرهنگ و هنر: \n",
      "     Precision: 0.13114754098360656\n",
      "     Recall: 0.03755868544600939\n",
      "     F1-Measure: 0.058394160583941604\n",
      "Label احزاب و تشکل ها: \n",
      "     Precision: 0.4262295081967213\n",
      "     Recall: 0.9629629629629629\n",
      "     F1-Measure: 0.5909090909090909\n",
      "Label پاکستان: \n",
      "     Precision: 0.34782608695652173\n",
      "     Recall: 0.47058823529411764\n",
      "     F1-Measure: 0.39999999999999997\n",
      "Label بورس: \n",
      "     Precision: 0.5348837209302325\n",
      "     Recall: 0.92\n",
      "     F1-Measure: 0.6764705882352942\n",
      "Label گروههای توان خواه: \n",
      "     Precision: 0.5555555555555556\n",
      "     Recall: 0.8333333333333334\n",
      "     F1-Measure: 0.6666666666666667\n",
      "Label بازار: \n",
      "     Precision: 0.15\n",
      "     Recall: 0.1875\n",
      "     F1-Measure: 0.16666666666666663\n",
      "Label حماسه و مقاومت: \n",
      "     Precision: 0.0\n",
      "     Recall: nan\n",
      "     F1-Measure: nan\n",
      "Label خبر خوب: \n",
      "     Precision: 1.0\n",
      "     Recall: 0.6666666666666666\n",
      "     F1-Measure: 0.8\n",
      "Label آفریقا: \n",
      "     Precision: 0.42592592592592593\n",
      "     Recall: 0.7419354838709677\n",
      "     F1-Measure: 0.5411764705882354\n",
      "Label زنان و جوانان: \n",
      "     Precision: 0.46153846153846156\n",
      "     Recall: 0.75\n",
      "     F1-Measure: 0.5714285714285714\n",
      "Label مجلس: \n",
      "     Precision: 0.43243243243243246\n",
      "     Recall: 0.8495575221238938\n",
      "     F1-Measure: 0.573134328358209\n",
      "Label تاریخ: \n",
      "     Precision: 0.6666666666666666\n",
      "     Recall: 0.4\n",
      "     F1-Measure: 0.5\n",
      "Label جنگ اقتصادی: \n",
      "     Precision: 0.5\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6666666666666666\n",
      "Label سینما و تئاتر: \n",
      "     Precision: 0.532608695652174\n",
      "     Recall: 0.8448275862068966\n",
      "     F1-Measure: 0.6533333333333333\n",
      "Label داستان کوتاه: \n",
      "     Precision: 1.0\n",
      "     Recall: 0.3333333333333333\n",
      "     F1-Measure: 0.5\n",
      "Label استانها: \n",
      "     Precision: 0.9695121951219512\n",
      "     Recall: 0.7378190255220418\n",
      "     F1-Measure: 0.8379446640316206\n",
      "Label انقلاب اسلامی: \n",
      "     Precision: nan\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label علم و فن آوری جهان: \n",
      "     Precision: 0.5208333333333334\n",
      "     Recall: 0.7352941176470589\n",
      "     F1-Measure: 0.6097560975609756\n",
      "Label اندیشه: \n",
      "     Precision: 0.65\n",
      "     Recall: 0.7647058823529411\n",
      "     F1-Measure: 0.7027027027027027\n",
      "Label امام و رهبری: \n",
      "     Precision: 0.2\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.33333333333333337\n",
      "Label شرق آسیا و اقیانوسیه: \n",
      "     Precision: 0.5833333333333334\n",
      "     Recall: 0.6363636363636364\n",
      "     F1-Measure: 0.6086956521739131\n",
      "Label تحلیل بین الملل: \n",
      "     Precision: nan\n",
      "     Recall: nan\n",
      "     F1-Measure: nan\n",
      "Label آسياي مرکزی و روسيه: \n",
      "     Precision: 0.273972602739726\n",
      "     Recall: 0.4166666666666667\n",
      "     F1-Measure: 0.33057851239669417\n",
      "Label ورزش بانوان: \n",
      "     Precision: 0.42857142857142855\n",
      "     Recall: 0.5\n",
      "     F1-Measure: 0.4615384615384615\n",
      "Label فرهنگی/هنری: \n",
      "     Precision: 0.605\n",
      "     Recall: 0.8461538461538461\n",
      "     F1-Measure: 0.705539358600583\n",
      "Label فناوری و IT: \n",
      "     Precision: 0.4878048780487805\n",
      "     Recall: 0.6451612903225806\n",
      "     F1-Measure: 0.5555555555555556\n",
      "Label حوادث: \n",
      "     Precision: 0.3974358974358974\n",
      "     Recall: 0.7948717948717948\n",
      "     F1-Measure: 0.5299145299145299\n",
      "Label آمریکا، اروپا: \n",
      "     Precision: 0.47767857142857145\n",
      "     Recall: 0.8492063492063492\n",
      "     F1-Measure: 0.6114285714285714\n",
      "Label ویژه نامه ها: \n",
      "     Precision: 0.06666666666666667\n",
      "     Recall: 0.05263157894736842\n",
      "     F1-Measure: 0.058823529411764705\n",
      "Label ورزش بین الملل: \n",
      "     Precision: 0.4925373134328358\n",
      "     Recall: 0.7586206896551724\n",
      "     F1-Measure: 0.597285067873303\n",
      "Label آموزش و پرورش: \n",
      "     Precision: 0.5102040816326531\n",
      "     Recall: 0.9615384615384616\n",
      "     F1-Measure: 0.6666666666666667\n",
      "Label محور مقاومت: \n",
      "     Precision: nan\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label حج و زیارت و وقف: \n",
      "     Precision: 0.42105263157894735\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.5925925925925926\n",
      "Label اقتصادی: \n",
      "     Precision: 0.6202247191011236\n",
      "     Recall: 0.46779661016949153\n",
      "     F1-Measure: 0.5333333333333333\n",
      "Label قرآن و فعالیت های دینی: \n",
      "     Precision: 0.4067796610169492\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.5783132530120482\n",
      "Label تشکل های دانشگاهی: \n",
      "     Precision: 0.46153846153846156\n",
      "     Recall: 0.75\n",
      "     F1-Measure: 0.5714285714285714\n",
      "Label کتاب و ادبیات: \n",
      "     Precision: 0.42857142857142855\n",
      "     Recall: 0.6976744186046512\n",
      "     F1-Measure: 0.5309734513274337\n",
      "Label رسانه: \n",
      "     Precision: 0.2631578947368421\n",
      "     Recall: 0.5\n",
      "     F1-Measure: 0.3448275862068966\n",
      "Label محیط زیست و گردشگری: \n",
      "     Precision: 0.3548387096774194\n",
      "     Recall: 0.6470588235294118\n",
      "     F1-Measure: 0.4583333333333333\n",
      "Label عمومی: \n",
      "     Precision: 0.6153846153846154\n",
      "     Recall: 0.6153846153846154\n",
      "     F1-Measure: 0.6153846153846154\n",
      "Label سلامت: \n",
      "     Precision: 0.5657142857142857\n",
      "     Recall: 0.7333333333333333\n",
      "     F1-Measure: 0.6387096774193548\n",
      "Label دفاتر منطقه ای: \n",
      "     Precision: 0.28846153846153844\n",
      "     Recall: 0.18072289156626506\n",
      "     F1-Measure: 0.2222222222222222\n",
      "Label تعاون و اشتغال: \n",
      "     Precision: 0.4444444444444444\n",
      "     Recall: 0.7272727272727273\n",
      "     F1-Measure: 0.5517241379310345\n",
      "Label کشاورزی و امور دام: \n",
      "     Precision: 0.5\n",
      "     Recall: 0.7142857142857143\n",
      "     F1-Measure: 0.588235294117647\n",
      "Label ورزشی: \n",
      "     Precision: 0.6521739130434783\n",
      "     Recall: 0.2927669345579793\n",
      "     F1-Measure: 0.40412044374009504\n",
      "Label رفاه و آسیب های اجتماعی: \n",
      "     Precision: 0.4878048780487805\n",
      "     Recall: 0.7142857142857143\n",
      "     F1-Measure: 0.5797101449275363\n",
      "Label راه و مسکن: \n",
      "     Precision: 0.35714285714285715\n",
      "     Recall: 0.7692307692307693\n",
      "     F1-Measure: 0.48780487804878053\n",
      "Label غرب آسیا: \n",
      "     Precision: 0.3735408560311284\n",
      "     Recall: 0.8205128205128205\n",
      "     F1-Measure: 0.5133689839572193\n",
      "Label صنفی فرهنگی: \n",
      "     Precision: 0.36\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.5294117647058824\n",
      "Label انتظامی و حوادث: \n",
      "     Precision: 0.4375\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 0.6086956521739131\n",
      "Label آموزش: \n",
      "     Precision: 0.2553191489361702\n",
      "     Recall: 0.5217391304347826\n",
      "     F1-Measure: 0.3428571428571428\n",
      "Label خواندنی ها و دیدنی ها: \n",
      "     Precision: 0.6460176991150443\n",
      "     Recall: 0.6293103448275862\n",
      "     F1-Measure: 0.6375545851528385\n",
      "Label روانشناسی: \n",
      "     Precision: 0.3333333333333333\n",
      "     Recall: 0.2857142857142857\n",
      "     F1-Measure: 0.30769230769230765\n",
      "Label موسیقی و هنرهای تجسمی: \n",
      "     Precision: 0.48484848484848486\n",
      "     Recall: 0.8421052631578947\n",
      "     F1-Measure: 0.6153846153846154\n",
      "Label مجله فارس پلاس: \n",
      "     Precision: 1.0\n",
      "     Recall: 0.5\n",
      "     F1-Measure: 0.6666666666666666\n",
      "Label دیدگاه: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label بین الملل: \n",
      "     Precision: 0.8356164383561644\n",
      "     Recall: 0.5501691093573844\n",
      "     F1-Measure: 0.6634942216179469\n",
      "Label ارتباطات و فن آوری اطلاعات: \n",
      "     Precision: 0.40625\n",
      "     Recall: 0.8125\n",
      "     F1-Measure: 0.5416666666666666\n",
      "Label رزمی: \n",
      "     Precision: 0.4318181818181818\n",
      "     Recall: 0.7916666666666666\n",
      "     F1-Measure: 0.5588235294117647\n",
      "Label فرهنگ عمومی: \n",
      "     Precision: 0.0\n",
      "     Recall: nan\n",
      "     F1-Measure: nan\n",
      "Label علمی و دانشگاهی: \n",
      "     Precision: 0.2727272727272727\n",
      "     Recall: 0.168\n",
      "     F1-Measure: 0.2079207920792079\n",
      "Label سیاست خارجی: \n",
      "     Precision: 0.4064516129032258\n",
      "     Recall: 0.7777777777777778\n",
      "     F1-Measure: 0.5338983050847458\n",
      "Label بیمه و بانک: \n",
      "     Precision: 0.43243243243243246\n",
      "     Recall: 0.7272727272727273\n",
      "     F1-Measure: 0.5423728813559323\n",
      "Label حقوقی و قضایی: \n",
      "     Precision: 0.55\n",
      "     Recall: 0.9166666666666666\n",
      "     F1-Measure: 0.6874999999999999\n",
      "Label رادیو و تلویزیون: \n",
      "     Precision: 0.4\n",
      "     Recall: 0.7222222222222222\n",
      "     F1-Measure: 0.5148514851485149\n",
      "Label امنیتی و دفاعی: \n",
      "     Precision: 0.3137254901960784\n",
      "     Recall: 0.8888888888888888\n",
      "     F1-Measure: 0.46376811594202894\n",
      "Label پژوهش: \n",
      "     Precision: nan\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label اقتصاد بین الملل: \n",
      "     Precision: 0.5544554455445545\n",
      "     Recall: 0.9032258064516129\n",
      "     F1-Measure: 0.6871165644171778\n",
      "Label سفر و تفریح: \n",
      "     Precision: nan\n",
      "     Recall: nan\n",
      "     F1-Measure: nan\n",
      "Label سیاسی: \n",
      "     Precision: 0.7044967880085653\n",
      "     Recall: 0.4380825565912117\n",
      "     F1-Measure: 0.5402298850574713\n",
      "Label اطلاعات عمومی و دانستنی ها: \n",
      "     Precision: 0.4\n",
      "     Recall: 0.5263157894736842\n",
      "     F1-Measure: 0.45454545454545453\n",
      "Label فارس من: \n",
      "     Precision: 1.0\n",
      "     Recall: 1.0\n",
      "     F1-Measure: 1.0\n",
      "Label آشپزی: \n",
      "     Precision: 0.0\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Label نفت و انرژی: \n",
      "     Precision: 0.38461538461538464\n",
      "     Recall: 0.7894736842105263\n",
      "     F1-Measure: 0.5172413793103449\n",
      "Label دولت: \n",
      "     Precision: 0.5357142857142857\n",
      "     Recall: 0.8571428571428571\n",
      "     F1-Measure: 0.6593406593406593\n",
      "Label شبهه و شایعه: \n",
      "     Precision: 1.0\n",
      "     Recall: 0.4\n",
      "     F1-Measure: 0.5714285714285715\n",
      "Label فوتبال جهان: \n",
      "     Precision: 0.44086021505376344\n",
      "     Recall: 0.8864864864864865\n",
      "     F1-Measure: 0.5888689407540395\n",
      "Label کشتی و وزنه برداری: \n",
      "     Precision: 0.5294117647058824\n",
      "     Recall: 0.8181818181818182\n",
      "     F1-Measure: 0.6428571428571428\n",
      "Label علم و فن آوری ایران: \n",
      "     Precision: 0.10714285714285714\n",
      "     Recall: 0.16666666666666666\n",
      "     F1-Measure: 0.13043478260869565\n",
      "Label اقتصاد کلان و بودجه: \n",
      "     Precision: 0.35714285714285715\n",
      "     Recall: 0.7142857142857143\n",
      "     F1-Measure: 0.4761904761904762\n",
      "Label تاریخ معاصر: \n",
      "     Precision: 0.0\n",
      "     Recall: 0.0\n",
      "     F1-Measure: nan\n",
      "Total Accuracy: 0.5556259503294475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Erfan\\AnacondaProjects\\github\\NLP-Fall18-UOG\\utils\\models.py:89: RuntimeWarning: invalid value encountered in true_divide\n",
      "  percision = confusion_matrix.diagonal()/np.sum(confusion_matrix, axis=1)\n",
      "C:\\Users\\Erfan\\AnacondaProjects\\github\\NLP-Fall18-UOG\\utils\\models.py:90: RuntimeWarning: invalid value encountered in true_divide\n",
      "  recall = confusion_matrix.diagonal()/np.sum(confusion_matrix, axis=0)\n",
      "C:\\Users\\Erfan\\AnacondaProjects\\github\\NLP-Fall18-UOG\\utils\\models.py:91: RuntimeWarning: invalid value encountered in true_divide\n",
      "  f1_measure = 2*percision*recall/(percision+recall)\n"
     ]
    }
   ],
   "source": [
    "nb.evaluate(x_test, y_test, label_set=label_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Model for the Second Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = []\n",
    "for i, raw_label in enumerate(raw_labels):\n",
    "    l = []\n",
    "    for j, label in enumerate(raw_label):\n",
    "        l.append(np.argmax(label == label_set))\n",
    "    t.append(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%0 continue...\n",
      "%1000 continue...\n",
      "%2000 continue...\n",
      "%3000 continue...\n",
      "%4000 continue...\n",
      "%5000 continue...\n",
      "%6000 continue...\n",
      "%7000 continue...\n",
      "%8000 continue...\n",
      "%9000 continue...\n",
      "%10000 continue...\n",
      "%11000 continue...\n",
      "%12000 continue...\n",
      "%13000 continue...\n",
      "%14000 continue...\n",
      "%15000 continue...\n",
      "%16000 continue...\n",
      "%17000 continue...\n",
      "%18000 continue...\n",
      "%19000 continue...\n",
      "%20000 continue...\n",
      "%21000 continue...\n",
      "%22000 continue...\n",
      "%23000 continue...\n",
      "%24000 continue...\n",
      "%25000 continue...\n",
      "%26000 continue...\n",
      "%27000 continue...\n",
      "%28000 continue...\n",
      "%29000 continue...\n",
      "Total Score: -88182\n"
     ]
    }
   ],
   "source": [
    "nb.evaluate(documents, t, label_set, eval_type='multiple')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Model for the Third Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab created\n",
      "P(c) calculated\n",
      "2\n",
      "%0.0 continue...\n",
      "%0.5 continue...\n",
      "P(w|c) calculated\n"
     ]
    }
   ],
   "source": [
    "nb_th = NaiveBayes()\n",
    "nb_th.fit(x_train_th, y_train_th)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%0 continue...\n",
      "%1000 continue...\n",
      "%2000 continue...\n",
      "%3000 continue...\n",
      "%4000 continue...\n",
      "%5000 continue...\n",
      "%6000 continue...\n",
      "%7000 continue...\n",
      "%8000 continue...\n",
      "%9000 continue...\n",
      "%10000 continue...\n",
      "%11000 continue...\n",
      "%12000 continue...\n",
      "%13000 continue...\n",
      "%14000 continue...\n",
      "%15000 continue...\n",
      "%16000 continue...\n",
      "%17000 continue...\n",
      "%18000 continue...\n",
      "%19000 continue...\n",
      "%20000 continue...\n",
      "%21000 continue...\n",
      "%22000 continue...\n",
      "%23000 continue...\n",
      "Label AsrIran: \n",
      "     Precision: 0.965974765974766\n",
      "     Recall: 0.9862865691489362\n",
      "     F1-Measure: 0.9760250030842621\n",
      "Label Fars: \n",
      "     Precision: 0.9853072128227961\n",
      "     Recall: 0.9635983627971785\n",
      "     F1-Measure: 0.9743318804209042\n",
      "Total Accuracy: 0.9752073144801191\n"
     ]
    }
   ],
   "source": [
    "nb_th.evaluate(x_train_th, y_train_th, label_set_th)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%0 continue...\n",
      "%1000 continue...\n",
      "%2000 continue...\n",
      "%3000 continue...\n",
      "%4000 continue...\n",
      "%5000 continue...\n",
      "Label AsrIran: \n",
      "     Precision: 0.9382040553588671\n",
      "     Recall: 0.9821428571428571\n",
      "     F1-Measure: 0.959670781893004\n",
      "Label Fars: \n",
      "     Precision: 0.9808802308802309\n",
      "     Recall: 0.9340432840948127\n",
      "     F1-Measure: 0.9568889670948443\n",
      "Total Accuracy: 0.9583262459601973\n"
     ]
    }
   ],
   "source": [
    "nb_th.evaluate(x_test_th, y_test_th, label_set_th)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}