kiva_classification_simple.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df = pd.read_csv(\"data/kiva_cleaned.csv\")\n",
    "kiva_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df = kiva_df.dropna()\n",
    "kiva_df = kiva_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import re\n",
    "import unidecode\n",
    "\n",
    "stop_words = set(stopwords.words('english') + stopwords.words('spanish'))\n",
    "\n",
    "lemmer = WordNetLemmatizer()\n",
    "\n",
    "def preprocess(x):\n",
    "    x = x.lower()\n",
    "    \n",
    "    x = re.sub(r'[^\\w\\s]', '', x)\n",
    "    \n",
    "    x = unidecode.unidecode(x)\n",
    "    \n",
    "    x = re.sub(r'\\d+', '', x)\n",
    "    \n",
    "    x = [lemmer.lemmatize(w) for w in x.split() if w not in stop_words]\n",
    "\n",
    "    return ' '.join(x)\n",
    "\n",
    "kiva_df['en_clean_pre'] = kiva_df['en_clean'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df.iloc[0, :].en_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df.iloc[0, :].en_clean_pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(max_df = 0.5, min_df=0.05, max_features = 1000, ngram_range=[1,3])\n",
    "dtm = vectorizer.fit_transform(kiva_df['en_clean_pre'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow_df = pd.DataFrame(dtm.toarray(), \n",
    "                      columns=vectorizer.get_feature_names(), \n",
    "                      index=kiva_df.index)\n",
    "\n",
    "kiva_df_bow = pd.concat([kiva_df, bow_df], axis=1)\n",
    "kiva_df_bow.shape\n",
    "kiva_df_bow.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textstat\n",
    "  \n",
    "kiva_df['len'] = kiva_df['en_clean'].apply(lambda x: len(x))\n",
    "kiva_df['syllable_count'] = kiva_df['en_clean'].apply(\n",
    "    lambda x: textstat.syllable_count(x))\n",
    "kiva_df['flesch_reading_ease'] = kiva_df['en_clean'].apply(\n",
    "    lambda x: textstat.flesch_reading_ease(x))\n",
    "\n",
    "kiva_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_df_bow = kiva_df_bow.drop(\n",
    "    columns=['loan_id', 'sector', 'country', 'gender', \n",
    "             'loan_amount', 'nonpayment', 'en', 'en_clean', 'en_clean_pre'],\n",
    "    axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "y = kiva_df_bow['status']\n",
    "X = kiva_df_bow.drop(['status'], axis=1)\n",
    "\n",
    "feature_names = X.columns\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = DecisionTreeClassifier(random_state=42, \n",
    "                             min_samples_split=10, \n",
    "                             min_samples_leaf=10, \n",
    "                             max_depth=6)\n",
    "\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred_dt = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp = clf.tree_.compute_feature_importances(normalize=False)\n",
    "ind = sorted(range(len(imp)), key=lambda i: imp[i])[-15:]\n",
    "\n",
    "imp[ind]\n",
    "feature_names[ind]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_pred_dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "class_names = [str(x) for x in clf.classes_]\n",
    "\n",
    "print(classification_report(y_test, y_pred_dt, target_names=class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Needs sklearn 0.21 or higher\n",
    "from sklearn.tree import plot_tree\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=[10,5]);\n",
    "plot_tree(clf, filled=True, feature_names = feature_names, label='root', fontsize=10)\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import export_text\n",
    "\n",
    "print(export_text(clf, feature_names = list(feature_names)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "small_sklearn_kernel",
   "language": "python",
   "name": "small_sklearn_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}