02-Graph2Speak.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1 style=\"font-size:3em\"> Graph2Speak </h1></center>\n",
    "<center><h3> Improving Speaker Identification using Network Knowledge in Criminal Conversational Data </h3><center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Paper: https://arxiv.org/abs/2006.02093\n",
    "\n",
    "*Maël Fabien, Seyyed Saeed Sarfjoo, Petr Motlicek, Srikanth Madikeri*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Set of functions\n",
    "from src.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "episode = \"s01e07\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'eddiewillows': '1001_csi',\n",
       "  'jesseoverton': '1002_csi',\n",
       "  'conradecklie': '1003_csi',\n",
       "  'sheriff_brianmobley': '1004_csi',\n",
       "  'tedgoggle': '1005_csi',\n",
       "  'lie_detector_operator': '1006_csi',\n",
       "  'nick': '1007_csi',\n",
       "  'warrick': '1008_csi',\n",
       "  'det_oriley': '1009_csi',\n",
       "  'brass': '1010_csi',\n",
       "  'tinacollins': '1011_csi',\n",
       "  'sara': '1012_csi',\n",
       "  'catherine': '1013_csi',\n",
       "  'grissom': '1014_csi'},\n",
       " {'1001_csi': 'eddiewillows',\n",
       "  '1002_csi': 'jesseoverton',\n",
       "  '1003_csi': 'conradecklie',\n",
       "  '1004_csi': 'sheriff_brianmobley',\n",
       "  '1005_csi': 'tedgoggle',\n",
       "  '1006_csi': 'lie_detector_operator',\n",
       "  '1007_csi': 'nick',\n",
       "  '1008_csi': 'warrick',\n",
       "  '1009_csi': 'det_oriley',\n",
       "  '1010_csi': 'brass',\n",
       "  '1011_csi': 'tinacollins',\n",
       "  '1012_csi': 'sara',\n",
       "  '1013_csi': 'catherine',\n",
       "  '1014_csi': 'grissom'},\n",
       " {'eddiewillows': [50, 50],\n",
       "  'jesseoverton': [50, 100],\n",
       "  'conradecklie': [50, 150],\n",
       "  'sheriffbrianmobley': [100, 50],\n",
       "  'tedgoggle': [100, 100],\n",
       "  'liedetectoroperator': [100, 150],\n",
       "  'nick': [150, 50],\n",
       "  'warrick': [150, 100],\n",
       "  'detoriley': [150, 150],\n",
       "  'brass': [200, 50],\n",
       "  'tinacollins': [200, 100],\n",
       "  'sara': [200, 150],\n",
       "  'catherine': [300, 50],\n",
       "  'grissom': [300, 100]})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict_spk, spk_dict, spk_coord = ep_dicts(episode)\n",
    "dict_spk, spk_dict, spk_coord"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>conv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       speaker  conv\n",
       "0  tinacollins   0.0\n",
       "1  tinacollins   0.0\n",
       "2  tinacollins   0.0\n",
       "3  tinacollins   0.0\n",
       "4  tinacollins   0.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "truth_events = pd.read_csv(\"src/graph_input/all_events_%s.csv\"%episode).drop_duplicates()\n",
    "\n",
    "dict_len = {}\n",
    "for c in np.unique(truth_events['conv']):\n",
    "    try:\n",
    "        dict_len[int(c)] = len(truth_events[truth_events['conv']==c])\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "truth_events = truth_events[['speaker', 'conv']].dropna()\n",
    "truth_events['speaker'] = truth_events['speaker'].apply(lambda x: x.replace(\"/\", \"\").replace(\".\", \"\").replace(\"'\", \"\").replace(\"_\", \"\"))\n",
    "truth_events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"src/speaker_id_input/%s.txt\"%episode, \"r\")\n",
    "list_spk_keep = []\n",
    "\n",
    "for line in f:\n",
    "    list_spk_keep.append(line.replace(\"\\n\", \"\").replace(\".\", \"\").replace(\"'\", \"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['brass', 'brendacollins', 'catherine', 'conniebellvue',\n",
       "       'conradecklie', 'davidphillips', 'detoriley', 'dralbertrobbins',\n",
       "       'drleever', 'eddiewillows', 'ems#1', 'grissom', 'jesseoverton',\n",
       "       'liedetectoroperator', 'lindsey', 'lindseywillows', 'nick',\n",
       "       'officer', 'oliver', 'reporter1(woman)', 'reporter2(woman)',\n",
       "       'reporter3(man)', 'sara', 'sheriffbrianmobley', 'shibley',\n",
       "       'suburbanteenguy#1', 'suburbanteenguy#2', 'tedgoggle',\n",
       "       'tinacollins', 'warrick'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(truth_events['speaker'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['brass', 'catherine', 'conradecklie', 'detoriley', 'eddiewillows',\n",
       "       'grissom', 'jesseoverton', 'liedetectoroperator', 'nick', 'sara',\n",
       "       'sheriffbrianmobley', 'tedgoggle', 'tinacollins', 'warrick'],\n",
       "      dtype='<U19')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(list_spk_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "truth_events = truth_events[truth_events['speaker'].isin(list_spk_keep)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"500px\"\n",
       "            src=\"src/generated_graph/s01e07/truth.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa789766a90>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G, plot = build_graph(truth_events, \"conv\", \"speaker\", \"truth\", episode, spk_coord)\n",
    "plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# II. Speaker ID Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Benchmark performance from Kaldi:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "perf_s01e07 = 0.916\n",
    "perf_s01e08 = 0.919\n",
    "perf_s01e19 = 0.579\n",
    "perf_s01e20 = 0.746\n",
    "perf_s01e23 = 0.686\n",
    "perf_s02e01 = 0.880\n",
    "perf_s02e04 = 0.894\n",
    "perf_s02e06 = 0.855"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need 2 dataframes here, a summary of all of the scores of all speaker against each file, and another file of who has the maximum score, corresponding to the prediction of Speaker Id:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>File</th>\n",
       "      <th>Truth</th>\n",
       "      <th>Conv</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>eddiewillows</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-17.55911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>jesseoverton</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.16105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>conradecklie</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-17.07024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>sheriff_brianmobley</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-32.40960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>tedgoggle</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-28.67418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Model               File        Truth Conv     Score\n",
       "27          eddiewillows  tinacollins_Conv0  tinacollins    0 -17.55911\n",
       "123         jesseoverton  tinacollins_Conv0  tinacollins    0  -4.16105\n",
       "219         conradecklie  tinacollins_Conv0  tinacollins    0 -17.07024\n",
       "315  sheriff_brianmobley  tinacollins_Conv0  tinacollins    0 -32.40960\n",
       "411            tedgoggle  tinacollins_Conv0  tinacollins    0 -28.67418"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = get_all_pred_scores(\"src/speaker_id_output/scores_%s/csi_test_unique_scores\"%episode, spk_dict)\n",
    "pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pred</th>\n",
       "      <th>Truth</th>\n",
       "      <th>Conv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>det_oriley</td>\n",
       "      <td>det_oriley</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>grissom</td>\n",
       "      <td>grissom</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>grissom</td>\n",
       "      <td>grissom</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>det_oriley</td>\n",
       "      <td>det_oriley</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Pred        Truth Conv\n",
       "0  tinacollins  tinacollins    0\n",
       "1   det_oriley   det_oriley    1\n",
       "2      grissom      grissom    1\n",
       "3      grissom      grissom   10\n",
       "4   det_oriley   det_oriley   11"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "winners = get_pred_speakers(pred)\n",
    "winners.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Re-compute the speaker accuracy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9166666666666666"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "speaker_accuracy(winners)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And plot the predicted network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"500px\"\n",
       "            src=\"src/generated_graph/s01e07/pred.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa7897b3240>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G_pred, plot_pred = build_graph(winners, \"Conv\", \"Pred\", \"pred\", episode, spk_coord)\n",
    "plot_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# III. Improving Speaker Identification using Network Knowledge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need 2 datasets again, one to build the list of all candidates, and another one to keep all the candidates from pred above a given threshold:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Conv</th>\n",
       "      <th>NumChar</th>\n",
       "      <th>Conversation</th>\n",
       "      <th>Truth</th>\n",
       "      <th>Candidate</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0_tinacollins</td>\n",
       "      <td>[tinacollins]</td>\n",
       "      <td>[eddiewillows, jesseoverton, conradecklie, she...</td>\n",
       "      <td>[-17.55911, -4.16105, -17.07024, -32.4096, -28...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1_det_oriley</td>\n",
       "      <td>[det_oriley, grissom]</td>\n",
       "      <td>[eddiewillows, jesseoverton, conradecklie, she...</td>\n",
       "      <td>[-13.57533, -16.27033, -10.17966, 7.101351, -3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2_grissom</td>\n",
       "      <td>[grissom, sara]</td>\n",
       "      <td>[eddiewillows, jesseoverton, conradecklie, she...</td>\n",
       "      <td>[6.577901, 4.356473, -4.583953, 20.2203, 0.100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3_det_oriley</td>\n",
       "      <td>[det_oriley, grissom]</td>\n",
       "      <td>[eddiewillows, jesseoverton, conradecklie, she...</td>\n",
       "      <td>[-25.47741, -7.629332, -15.41951, -16.17051, -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5_det_oriley</td>\n",
       "      <td>[det_oriley, grissom, tinacollins]</td>\n",
       "      <td>[eddiewillows, jesseoverton, conradecklie, she...</td>\n",
       "      <td>[-3.301001, -8.831428, -19.57747, 5.522082, -4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Conv  NumChar   Conversation                               Truth  \\\n",
       "0      0        1  0_tinacollins                       [tinacollins]   \n",
       "1      1        2   1_det_oriley               [det_oriley, grissom]   \n",
       "12     2        2      2_grissom                     [grissom, sara]   \n",
       "23     3        2   3_det_oriley               [det_oriley, grissom]   \n",
       "41     5        3   5_det_oriley  [det_oriley, grissom, tinacollins]   \n",
       "\n",
       "                                            Candidate  \\\n",
       "0   [eddiewillows, jesseoverton, conradecklie, she...   \n",
       "1   [eddiewillows, jesseoverton, conradecklie, she...   \n",
       "12  [eddiewillows, jesseoverton, conradecklie, she...   \n",
       "23  [eddiewillows, jesseoverton, conradecklie, she...   \n",
       "41  [eddiewillows, jesseoverton, conradecklie, she...   \n",
       "\n",
       "                                                Score  \n",
       "0   [-17.55911, -4.16105, -17.07024, -32.4096, -28...  \n",
       "1   [-13.57533, -16.27033, -10.17966, 7.101351, -3...  \n",
       "12  [6.577901, 4.356473, -4.583953, 20.2203, 0.100...  \n",
       "23  [-25.47741, -7.629332, -15.41951, -16.17051, -...  \n",
       "41  [-3.301001, -8.831428, -19.57747, 5.522082, -4...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cand = build_candidates(pred)\n",
    "cand.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/maelfabien/Graph2Speak/src/utils.py:683: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  score_sup[\"Conv\"] = score_sup[\"Conv\"].astype(int)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>File</th>\n",
       "      <th>Truth</th>\n",
       "      <th>Conv</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jesseoverton</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.16105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lie_detector_operator</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-14.53733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nick</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>-12.12411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tinacollins</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>18.20214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sara</td>\n",
       "      <td>tinacollins_Conv0</td>\n",
       "      <td>tinacollins</td>\n",
       "      <td>0</td>\n",
       "      <td>1.25702</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Model               File        Truth  Conv     Score\n",
       "0           jesseoverton  tinacollins_Conv0  tinacollins     0  -4.16105\n",
       "1  lie_detector_operator  tinacollins_Conv0  tinacollins     0 -14.53733\n",
       "2                   nick  tinacollins_Conv0  tinacollins     0 -12.12411\n",
       "3            tinacollins  tinacollins_Conv0  tinacollins     0  18.20214\n",
       "4                   sara  tinacollins_Conv0  tinacollins     0   1.25702"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_sup = keep_higher_scores(pred, threshold=-15)\n",
    "score_sup.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conversation 0 out of 48\n",
      "Conversation 1 out of 48\n",
      "Conversation 2 out of 48\n",
      "Conversation 3 out of 48\n",
      "Conversation 5 out of 48\n",
      "Conversation 6 out of 48\n",
      "Conversation 7 out of 48\n",
      "Conversation 9 out of 48\n",
      "Conversation 10 out of 48\n",
      "Conversation 11 out of 48\n",
      "Conversation 12 out of 48\n",
      "Conversation 13 out of 48\n",
      "Conversation 14 out of 48\n",
      "Conversation 15 out of 48\n",
      "Conversation 16 out of 48\n",
      "Conversation 17 out of 48\n",
      "Conversation 18 out of 48\n",
      "Conversation 19 out of 48\n",
      "Conversation 20 out of 48\n",
      "Conversation 21 out of 48\n",
      "Conversation 22 out of 48\n",
      "Conversation 23 out of 48\n",
      "Conversation 24 out of 48\n",
      "Conversation 25 out of 48\n",
      "Conversation 26 out of 48\n",
      "Conversation 27 out of 48\n",
      "Conversation 28 out of 48\n",
      "Conversation 29 out of 48\n",
      "Conversation 30 out of 48\n",
      "Conversation 33 out of 48\n",
      "Conversation 34 out of 48\n",
      "Conversation 35 out of 48\n",
      "Conversation 36 out of 48\n",
      "Conversation 37 out of 48\n",
      "Conversation 38 out of 48\n",
      "Conversation 39 out of 48\n",
      "Conversation 40 out of 48\n",
      "Conversation 41 out of 48\n",
      "Conversation 42 out of 48\n",
      "Conversation 43 out of 48\n",
      "Conversation 44 out of 48\n",
      "Conversation 45 out of 48\n",
      "Conversation 46 out of 48\n",
      "Conversation 47 out of 48\n",
      "Conversation 48 out of 48\n"
     ]
    }
   ],
   "source": [
    "df_res, G_rank, trace_conv = rerank_graph(score_sup, winners, cand, dict_len, threshold=-15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Where are predictions different?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Conv</th>\n",
       "      <th>GaphEnhance</th>\n",
       "      <th>Truth</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>[eddiewillows, grissom, grissom]</td>\n",
       "      <td>[catherine, grissom, nick, warrick]</td>\n",
       "      <td>[eddiewillows, grissom, grissom, warrick]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>15</td>\n",
       "      <td>[grissom, sara]</td>\n",
       "      <td>[grissom, sara]</td>\n",
       "      <td>[brass, sara]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>39</td>\n",
       "      <td>[brass, catherine, grissom, nick, warrick]</td>\n",
       "      <td>[brass, catherine, grissom, nick, warrick]</td>\n",
       "      <td>[brass, catherine, jesseoverton, nick, warrick]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>48</td>\n",
       "      <td>[grissom, nick]</td>\n",
       "      <td>[catherine, grissom, nick]</td>\n",
       "      <td>[grissom, nick, tinacollins]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Conv                                 GaphEnhance  \\\n",
       "6     7            [eddiewillows, grissom, grissom]   \n",
       "13   15                             [grissom, sara]   \n",
       "35   39  [brass, catherine, grissom, nick, warrick]   \n",
       "44   48                             [grissom, nick]   \n",
       "\n",
       "                                         Truth  \\\n",
       "6          [catherine, grissom, nick, warrick]   \n",
       "13                             [grissom, sara]   \n",
       "35  [brass, catherine, grissom, nick, warrick]   \n",
       "44                  [catherine, grissom, nick]   \n",
       "\n",
       "                                         Prediction  \n",
       "6         [eddiewillows, grissom, grissom, warrick]  \n",
       "13                                    [brass, sara]  \n",
       "35  [brass, catherine, jesseoverton, nick, warrick]  \n",
       "44                     [grissom, nick, tinacollins]  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_res[df_res['GaphEnhance'] != df_res['Prediction']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conversation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8444444444444444"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversation_accuracy(df_res, \"Prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8888888888888888"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conversation_accuracy(df_res, \"GaphEnhance\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Speaker accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9166666666666666"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_speaker_accuracy(df_res, \"Prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9270833333333334"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_speaker_accuracy(df_res, \"GaphEnhance\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"500px\"\n",
       "            src=\"src/generated_graph/s01e07/rerank.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fa785913a58>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_rank = final_graph(G_rank, trace_conv, episode, spk_coord)\n",
    "plot_rank"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}