src/results_method.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.2 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "2d76a938c981de1713584c398436925a46ce152d26d3ea2af97ee7c96e32c5a8"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! python -V\n",
    "import os\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import pandas as pd\n",
    "import matplotlib.colors as mcolors\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from experiment import Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_words = ['sit', 'plant', 'wonder', 'relation']\n",
    "exp = Experiment(\n",
    "    mapping='mapping_english.txt',\n",
    "    dictionary='cmudict-0.7b-with-vitz-nonce',\n",
    "    encoding='latin1',\n",
    "    words=experiment_words)\n",
    "dataset = exp.get_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def similarity_scores(df, methods, words, rank=False):\n",
    "    columns = {}\n",
    "    for method in methods:\n",
    "        scores = []\n",
    "        for word in words:\n",
    "            obtained = df[df['actual'] == word]['obtained'].to_numpy()\n",
    "            vw_predicted = df[df['actual'] == word][method].to_numpy()\n",
    "            pearson = np.corrcoef(obtained, vw_predicted)[0, 1]\n",
    "            spearman = stats.spearmanr(obtained, vw_predicted)[0]\n",
    "            scores.append(spearman if rank else pearson)\n",
    "        columns[method] = scores\n",
    "    return pd.DataFrame(columns, index=words)\n",
    "\n",
    "def draw_plot(dataset, methods, rank=False):\n",
    "    scores = similarity_scores(dataset, methods, experiment_words, rank=rank)\n",
    "    # colors = list(mcolors.TABLEAU_COLORS)[::-1]\n",
    "    colors = ['orange', 'lightskyblue', 'darkseagreen', 'palevioletred', 'silver', 'gold']\n",
    "    fig, ax = plt.subplots(facecolor='w')\n",
    "    scores.plot.bar(ax=ax, width=0.8, legend=False, figsize=(12,5), color=colors, fontsize=18)\n",
    "    ax.patch.set_facecolor('w')\n",
    "    title = 'Spearman' if rank else 'Pearson'\n",
    "    ax.set_ylabel(f'{title}\\ncorrelation\\ncoefficient\\n', fontsize=18)\n",
    "    ax.set_xlim(-0.5, len(scores)-.5)\n",
    "    ax.set_ylim(np.around(scores.min(numeric_only=True).to_numpy().min()-0.05, decimals=1), 1)\n",
    "    ax.axes.get_xaxis().set_visible(False)\n",
    "    # ax.legend(loc='center right', bbox_to_anchor=(1.35, 0.5), shadow=True, ncol=1)\n",
    "    table = pd.plotting.table(ax, np.round(scores.T, 5), loc='bottom', cellLoc='center', rowColours=colors)\n",
    "    # table.update({'text.color' : \"blue\", 'axes.labelcolor' : \"blue\"})\n",
    "    # print(dir(table.rcParams))\n",
    "    table.set_fontsize(18)\n",
    "    table.scale(1, 2)\n",
    "\n",
    "draw_plot(dataset, ['unigram', 'bigram', 'bigram p=2.5', 'bigram p=2.5 VW'])\n",
    "# draw_plot(dataset, ['unigram', 'bigram', 'bigram p=2.5'], True)\n",
    "draw_plot(dataset, ['PSSVec', 'bigram p=2.5 VW'])\n",
    "# draw_plot(dataset, ['vw_predicted', 'PSSVec', 'bigram p=2.5', 'bigram p=2.5 VW'], True)\n",
    "draw_plot(dataset, ['PSSVec', 'Ours'])\n",
    "# draw_plot(dataset, ['PSSVec', 'Ours'], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "penalties = exp.penalty_analysis(experiment_words, 1, 5, 33, bigram=True, vowel=False)\n",
    "best_penalty = penalties['avg'].idxmax()\n",
    "print(best_penalty, penalties['avg'].max())\n",
    "penalties = penalties.drop(columns=['avg'])\n",
    "fig, ax = plt.subplots(facecolor='w')\n",
    "penalties.plot.line(ax=ax, figsize=(10,4), fontsize=16)\n",
    "ax.set_xlabel('Penalty', fontsize=16)\n",
    "ax.set_ylabel('Pearson\\ncorrelation\\ncoefficient\\n', fontsize=16)\n",
    "ax.legend(loc='center right', bbox_to_anchor=(1.25, 0.5), shadow=True, ncol=1, fontsize=16)\n",
    "ax.axvline(best_penalty, color='k', linestyle='--')\n",
    "plt.text(best_penalty, 0.5, ' max of average', rotation=0, fontsize=12)"
   ]
  }
 ]
}