KDI-ivector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import  confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import cross_validate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# i-Vector"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Train & Validation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def report_classifer(y_test,predicts,target_names):\n",
    "    print(\"-----------------------------\")\n",
    "    print(\"Report Classifier\")\n",
    "    cm=confusion_matrix(y_test,predicts,labels=target_names)\n",
    "    print(cm)\n",
    "    print(classification_report(y_test, predicts, target_names=target_names))\n",
    "    print(\"Report Classifier\")\n",
    "    print(\"-----------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data_set(csv_vectore,number_feature=401):\n",
    "    vdf=pd.read_csv(csv_vectore,header=None)\n",
    "    columns=[ \"f{}\".format(i) for i in range(1,number_feature)]\n",
    "    columns.append('class')\n",
    "    vdf.columns=columns\n",
    "    features=vdf.iloc[:,0:-1]\n",
    "    dialects=vdf.iloc[:,-1]\n",
    "    return features,dialects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "features,dialects=read_data_set(\"kurdish_dialect_vectors/i-vectors.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics=['precision_macro', 'recall_macro','f1_macro','accuracy']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation on SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  9.2min finished\n"
     ]
    }
   ],
   "source": [
    "svc = SVC(gamma='auto',random_state=42)\n",
    "scores_svc = cross_validate(svc, features, dialects, cv=5,verbose=1,scoring=metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([ 80.34038544,  93.49890471,  89.8349278 ,  95.4978776 ,\n",
       "        100.26090145]),\n",
       " 'score_time': array([17.8331244 , 17.90214086, 18.12675238, 18.59246302, 18.78577709]),\n",
       " 'test_precision_macro': array([0.75915324, 0.88422508, 0.8518159 , 0.93785604, 0.93075817]),\n",
       " 'test_recall_macro': array([0.68435873, 0.81753112, 0.78314302, 0.93001267, 0.92212733]),\n",
       " 'test_f1_macro': array([0.68672608, 0.82591334, 0.79534975, 0.93355808, 0.92595182]),\n",
       " 'test_accuracy': array([0.68355203, 0.86176381, 0.81660055, 0.9337809 , 0.92612943])}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_svc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision on CSV:  0.87\n",
      "Recall on CSV :  0.83\n",
      "F1_score  on CSV:  0.83\n",
      "Acuracy  on CSV:  0.84\n"
     ]
    }
   ],
   "source": [
    "print(\"Precision on CSV: \", round(scores_svc[\"test_precision_macro\"].mean(),2))\n",
    "print(\"Recall on CSV : \", round(scores_svc[\"test_recall_macro\"].mean(),2))\n",
    "print(\"F1_score  on CSV: \", round(scores_svc[\"test_f1_macro\"].mean(),2))\n",
    "print(\"Acuracy  on CSV: \", round(scores_svc[\"test_accuracy\"].mean(),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation On Decision Tree\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  1.3min finished\n"
     ]
    }
   ],
   "source": [
    "dt = DecisionTreeClassifier(criterion='entropy',random_state=42)\n",
    "scores_dt = cross_validate(dt, features, dialects, cv=5,verbose=1,scoring=metrics)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([14.79413509, 15.07838273, 16.23881531, 15.71446395, 15.92835283]),\n",
       " 'score_time': array([0.04606724, 0.04750133, 0.04614282, 0.04696321, 0.0469718 ]),\n",
       " 'test_precision_macro': array([0.55695788, 0.66442563, 0.60559676, 0.76176744, 0.76699534]),\n",
       " 'test_recall_macro': array([0.55768373, 0.63350734, 0.59043466, 0.75761609, 0.75522425]),\n",
       " 'test_f1_macro': array([0.55412858, 0.63716984, 0.59201302, 0.75800884, 0.75872793]),\n",
       " 'test_accuracy': array([0.55660665, 0.66859933, 0.61641745, 0.75434849, 0.75030525])}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision on DT:  0.67\n",
      "Recall on DT:  0.66\n",
      "F1_score on DT:  0.66\n",
      "Acuracy on DT:  0.67\n"
     ]
    }
   ],
   "source": [
    "print(\"Precision on DT: \", round(scores_dt[\"test_precision_macro\"].mean(),2))\n",
    "print(\"Recall on DT: \", round(scores_dt[\"test_recall_macro\"].mean(),2))\n",
    "print(\"F1_score on DT: \", round(scores_dt[\"test_f1_macro\"].mean(),2))\n",
    "print(\"Acuracy on DT: \", round(scores_dt[\"test_accuracy\"].mean(),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion Matrix on SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=[\"hawrami\", \"kalhori\", \"kurmanji\",\"sorani\",\"zazaki\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  8.8min finished\n"
     ]
    }
   ],
   "source": [
    "svc = SVC(gamma='auto',random_state=42)\n",
    "y_pred = cross_val_predict(svc, features, dialects, cv=5,verbose=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------\n",
      "Report Classifier\n",
      "[[1292   85  144  118  237]\n",
      " [  29 3254   67   66  264]\n",
      " [  22   25 2814  320  422]\n",
      " [  27    9  322 2783  245]\n",
      " [  10   11  114   13 3691]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     hawrami       0.94      0.69      0.79      1876\n",
      "     kalhori       0.96      0.88      0.92      3680\n",
      "    kurmanji       0.81      0.78      0.80      3603\n",
      "      sorani       0.84      0.82      0.83      3386\n",
      "      zazaki       0.76      0.96      0.85      3839\n",
      "\n",
      "    accuracy                           0.84     16384\n",
      "   macro avg       0.86      0.83      0.84     16384\n",
      "weighted avg       0.85      0.84      0.84     16384\n",
      "\n",
      "Report Classifier\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "report_classifer(dialects, y_pred,labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion Matrix on DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  1.2min finished\n"
     ]
    }
   ],
   "source": [
    "dt = DecisionTreeClassifier(criterion='entropy',random_state=42)\n",
    "y_pred = cross_val_predict(dt, features, dialects, cv=5,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------\n",
      "Report Classifier\n",
      "[[1094  129  235  187  231]\n",
      " [ 169 2594  274  265  378]\n",
      " [ 120  156 2175  801  351]\n",
      " [ 114  173  752 2119  228]\n",
      " [ 106  233  322  195 2983]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     hawrami       0.68      0.58      0.63      1876\n",
      "     kalhori       0.79      0.70      0.74      3680\n",
      "    kurmanji       0.58      0.60      0.59      3603\n",
      "      sorani       0.59      0.63      0.61      3386\n",
      "      zazaki       0.72      0.78      0.74      3839\n",
      "\n",
      "    accuracy                           0.67     16384\n",
      "   macro avg       0.67      0.66      0.66     16384\n",
      "weighted avg       0.67      0.67      0.67     16384\n",
      "\n",
      "Report Classifier\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "report_classifer(dialects, y_pred,labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}