results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "3dc951f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os\n",
    "import sys\n",
    "import inspect\n",
    "import pandas as pd\n",
    "\n",
    "from evaluate import evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "777dea9a",
   "metadata": {},
   "source": [
    "## Models trained on Thumbset "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fbf727e",
   "metadata": {},
   "source": [
    "#### Best models ablated by encoder type and soft/no soft labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ec58decb",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "gnn = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#gnn:ar.pth\",\n",
    "    beam_k=6\n",
    ").compute()\n",
    "gnn_soft = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#soft(gnn:ar).pth\",\n",
    "    beam_k=6\n",
    ").compute()\n",
    "lstm = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#lstm:ar.pth\",\n",
    "    beam_k=6\n",
    ").compute()\n",
    "lstm_soft = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#soft(lstm:ar).pth\",\n",
    "    beam_k=6\n",
    ").compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e161a95b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM-soft</th>\n",
       "      <td>64.082979</td>\n",
       "      <td>70.032417</td>\n",
       "      <td>83.606837</td>\n",
       "      <td>76.197673</td>\n",
       "      <td>1.044268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>64.376029</td>\n",
       "      <td>70.593300</td>\n",
       "      <td>84.455241</td>\n",
       "      <td>76.580648</td>\n",
       "      <td>1.009667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>65.236691</td>\n",
       "      <td>71.035931</td>\n",
       "      <td>84.815295</td>\n",
       "      <td>76.625344</td>\n",
       "      <td>0.966779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN</th>\n",
       "      <td>64.698163</td>\n",
       "      <td>70.155646</td>\n",
       "      <td>84.749871</td>\n",
       "      <td>76.601972</td>\n",
       "      <td>0.980424</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        gmr        hmr        smr        rmr       cpr\n",
       "ArThumbLSTM-soft  64.082979  70.032417  83.606837  76.197673  1.044268\n",
       "ArThumbLSTM       64.376029  70.593300  84.455241  76.580648  1.009667\n",
       "ArThumbGNN-soft   65.236691  71.035931  84.815295  76.625344  0.966779\n",
       "ArThumbGNN        64.698163  70.155646  84.749871  76.601972  0.980424"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = {\n",
    "    \"ArThumbLSTM-soft\": lstm_soft['stats']['full'],\n",
    "    \"ArThumbLSTM\": lstm['stats']['full'], \n",
    "    \"ArThumbGNN-soft\": gnn_soft['stats']['full'],\n",
    "    \"ArThumbGNN\": gnn['stats']['full']\n",
    "}\n",
    "only_thumbset = pd.DataFrame(table).transpose()\n",
    "only_thumbset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8e2ec59",
   "metadata": {},
   "source": [
    "#### ablation by hand of gnn model and lstm model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "408fce27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ArThumbLSTM-soft</th>\n",
       "      <th>full</th>\n",
       "      <td>64.082979</td>\n",
       "      <td>70.032417</td>\n",
       "      <td>83.606837</td>\n",
       "      <td>76.197673</td>\n",
       "      <td>1.044268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>59.756540</td>\n",
       "      <td>65.048497</td>\n",
       "      <td>82.433585</td>\n",
       "      <td>73.368057</td>\n",
       "      <td>1.068056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>69.124774</td>\n",
       "      <td>76.213902</td>\n",
       "      <td>85.637848</td>\n",
       "      <td>79.752799</td>\n",
       "      <td>1.026578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ArThumbLSTM</th>\n",
       "      <th>full</th>\n",
       "      <td>64.376029</td>\n",
       "      <td>70.593300</td>\n",
       "      <td>84.455241</td>\n",
       "      <td>76.580648</td>\n",
       "      <td>1.009667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>59.563759</td>\n",
       "      <td>65.529533</td>\n",
       "      <td>82.712194</td>\n",
       "      <td>73.370709</td>\n",
       "      <td>1.054790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>69.360201</td>\n",
       "      <td>76.283783</td>\n",
       "      <td>86.409522</td>\n",
       "      <td>80.022446</td>\n",
       "      <td>0.966937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ArThumbGNN-soft</th>\n",
       "      <th>full</th>\n",
       "      <td>65.236691</td>\n",
       "      <td>71.035931</td>\n",
       "      <td>84.815295</td>\n",
       "      <td>76.625344</td>\n",
       "      <td>0.966779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>63.519726</td>\n",
       "      <td>70.118263</td>\n",
       "      <td>86.749809</td>\n",
       "      <td>77.145793</td>\n",
       "      <td>0.984018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>66.701942</td>\n",
       "      <td>71.789332</td>\n",
       "      <td>82.549761</td>\n",
       "      <td>75.828254</td>\n",
       "      <td>0.967900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ArThumbGNN</th>\n",
       "      <th>full</th>\n",
       "      <td>64.698163</td>\n",
       "      <td>70.155646</td>\n",
       "      <td>84.749871</td>\n",
       "      <td>76.601972</td>\n",
       "      <td>0.980424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>63.445776</td>\n",
       "      <td>69.178295</td>\n",
       "      <td>86.822357</td>\n",
       "      <td>77.467503</td>\n",
       "      <td>0.986825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>65.703146</td>\n",
       "      <td>71.064916</td>\n",
       "      <td>82.625094</td>\n",
       "      <td>75.429066</td>\n",
       "      <td>0.968995</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              gmr        hmr        smr        rmr       cpr\n",
       "ArThumbLSTM-soft full   64.082979  70.032417  83.606837  76.197673  1.044268\n",
       "                 right  59.756540  65.048497  82.433585  73.368057  1.068056\n",
       "                 left   69.124774  76.213902  85.637848  79.752799  1.026578\n",
       "ArThumbLSTM      full   64.376029  70.593300  84.455241  76.580648  1.009667\n",
       "                 right  59.563759  65.529533  82.712194  73.370709  1.054790\n",
       "                 left   69.360201  76.283783  86.409522  80.022446  0.966937\n",
       "ArThumbGNN-soft  full   65.236691  71.035931  84.815295  76.625344  0.966779\n",
       "                 right  63.519726  70.118263  86.749809  77.145793  0.984018\n",
       "                 left   66.701942  71.789332  82.549761  75.828254  0.967900\n",
       "ArThumbGNN       full   64.698163  70.155646  84.749871  76.601972  0.980424\n",
       "                 right  63.445776  69.178295  86.822357  77.467503  0.986825\n",
       "                 left   65.703146  71.064916  82.625094  75.429066  0.968995"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = {\n",
    "    \"ArThumbLSTM-soft\": lstm_soft['stats'],\n",
    "    \"ArThumbLSTM\": lstm['stats'], \n",
    "    \"ArThumbGNN-soft\": gnn_soft['stats'],\n",
    "    \"ArThumbGNN\": gnn['stats']\n",
    "}\n",
    "pd.concat({k: pd.DataFrame(v).T for k, v in table.items()}, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c201f24c",
   "metadata": {},
   "source": [
    "#### ablation of beam decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7f8ea38e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "gnn = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#gnn:ar.pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']\n",
    "gnn_soft = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#soft(gnn:ar).pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']\n",
    "lstm = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#lstm:ar.pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']\n",
    "lstm_soft = evaluate(\n",
    "    \"best_official_base#augmented_noisy_random_seq2seq#seq2seq_noisy#soft(lstm:ar).pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f5fd4fd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM-soft</th>\n",
       "      <td>62.678863</td>\n",
       "      <td>69.066792</td>\n",
       "      <td>82.969975</td>\n",
       "      <td>75.191119</td>\n",
       "      <td>1.010059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>62.830024</td>\n",
       "      <td>68.909288</td>\n",
       "      <td>82.821988</td>\n",
       "      <td>74.490360</td>\n",
       "      <td>1.012468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>62.447840</td>\n",
       "      <td>68.125594</td>\n",
       "      <td>82.244365</td>\n",
       "      <td>73.581235</td>\n",
       "      <td>0.958181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN</th>\n",
       "      <td>61.214570</td>\n",
       "      <td>66.113227</td>\n",
       "      <td>81.254813</td>\n",
       "      <td>72.448937</td>\n",
       "      <td>0.950440</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        gmr        hmr        smr        rmr       cpr\n",
       "ArThumbLSTM-soft  62.678863  69.066792  82.969975  75.191119  1.010059\n",
       "ArThumbLSTM       62.830024  68.909288  82.821988  74.490360  1.012468\n",
       "ArThumbGNN-soft   62.447840  68.125594  82.244365  73.581235  0.958181\n",
       "ArThumbGNN        61.214570  66.113227  81.254813  72.448937  0.950440"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = {\n",
    "    \"ArThumbLSTM-soft\": lstm_soft,\n",
    "    \"ArThumbLSTM\": lstm, \n",
    "    \"ArThumbGNN-soft\": gnn_soft,\n",
    "    \"ArThumbGNN\": gnn\n",
    "}\n",
    "pd.DataFrame(table).transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a124acf8",
   "metadata": {},
   "source": [
    "#### ablation data augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6bd078bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "gnn = evaluate(\n",
    "    \"best_ablation_base_augmentation#no_augmented_noisy_random_seq2seq#seq2seq_noisy#gnn:ar.pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']\n",
    "gnn_soft = evaluate(\n",
    "    \"best_ablation_base_augmentation#no_augmented_noisy_random_seq2seq#seq2seq_noisy#soft(gnn:ar).pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']\n",
    "lstm = evaluate(\n",
    "    \"best_ablation_base_augmentation#no_augmented_noisy_random_seq2seq#seq2seq_noisy#lstm:ar.pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']\n",
    "lstm_soft = evaluate(\n",
    "    \"best_ablation_base_augmentation#no_augmented_noisy_random_seq2seq#seq2seq_noisy#soft(lstm:ar).pth\",\n",
    "    beam_k=1\n",
    ").compute()['stats']['full']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "de92a06f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM-soft</th>\n",
       "      <td>62.143064</td>\n",
       "      <td>67.817249</td>\n",
       "      <td>82.106669</td>\n",
       "      <td>73.430414</td>\n",
       "      <td>1.003380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>58.677356</td>\n",
       "      <td>64.380924</td>\n",
       "      <td>78.419827</td>\n",
       "      <td>69.656638</td>\n",
       "      <td>0.984590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>53.044024</td>\n",
       "      <td>57.148342</td>\n",
       "      <td>72.539399</td>\n",
       "      <td>62.367956</td>\n",
       "      <td>0.881640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN</th>\n",
       "      <td>53.783114</td>\n",
       "      <td>59.063806</td>\n",
       "      <td>73.528206</td>\n",
       "      <td>63.665884</td>\n",
       "      <td>0.905592</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        gmr        hmr        smr        rmr       cpr\n",
       "ArThumbLSTM-soft  62.143064  67.817249  82.106669  73.430414  1.003380\n",
       "ArThumbLSTM       58.677356  64.380924  78.419827  69.656638  0.984590\n",
       "ArThumbGNN-soft   53.044024  57.148342  72.539399  62.367956  0.881640\n",
       "ArThumbGNN        53.783114  59.063806  73.528206  63.665884  0.905592"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = {\n",
    "    \"ArThumbLSTM-soft\": lstm_soft,\n",
    "    \"ArThumbLSTM\": lstm, \n",
    "    \"ArThumbGNN-soft\": gnn_soft,\n",
    "    \"ArThumbGNN\": gnn\n",
    "}\n",
    "pd.DataFrame(table).transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3b7cc82",
   "metadata": {},
   "source": [
    "#### ablation autoregressive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0e7e9568",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "gnn_soft = evaluate(\n",
    "    \"best_ablation_base_autoregressive#augmented_noisy_random_seq2seq#seq2seq_noisy#soft(gnn:fc).pth\",\n",
    "    alias_architecture='fc'\n",
    ").compute()['stats']['full']\n",
    "\n",
    "lstm = evaluate(\n",
    "    \"best_ablation_base_autoregressive#augmented_noisy_random_seq2seq#seq2seq_noisy#lstm:fc.pth\",\n",
    "    alias_architecture='fc'\n",
    ").compute()['stats']['full']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "222b25f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>27.385242</td>\n",
       "      <td>29.790501</td>\n",
       "      <td>36.357534</td>\n",
       "      <td>30.588727</td>\n",
       "      <td>0.314443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>61.560856</td>\n",
       "      <td>65.956026</td>\n",
       "      <td>81.263443</td>\n",
       "      <td>69.248868</td>\n",
       "      <td>0.783834</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       gmr        hmr        smr        rmr       cpr\n",
       "ArThumbLSTM      27.385242  29.790501  36.357534  30.588727  0.314443\n",
       "ArThumbGNN-soft  61.560856  65.956026  81.263443  69.248868  0.783834"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = {\n",
    "#     \"ArThumbLSTM-soft\": lstm_soft,\n",
    "    \"ArThumbLSTM\": lstm, \n",
    "    \"ArThumbGNN-soft\": gnn_soft,\n",
    "#     \"ArThumbGNN\": gnn\n",
    "}\n",
    "pd.DataFrame(table).transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c09958b",
   "metadata": {},
   "source": [
    "## FINETUNING"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ee11fbf",
   "metadata": {},
   "source": [
    "#### Best models ablated by encoder type and soft/no soft labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d1845126",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    \"frozen\": {\n",
    "        \"ArThumbLSTM-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(lstm:ar)_1.pth',\n",
    "        \"ArThumbLSTM\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:ar_1.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(gnn:ar)_1.pth',\n",
    "        \"ArThumbGNN\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#gnn:ar_1.pth'\n",
    "    },\n",
    "    \"no-frozen\":{\n",
    "        \"ArThumbLSTM-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(lstm:ar)_0.pth',\n",
    "        \"ArThumbLSTM\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:ar_0.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(gnn:ar)_0.pth',\n",
    "        \"ArThumbGNN\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#gnn:ar_0.pth' \n",
    "    }\n",
    "}\n",
    "\n",
    "table = models.copy()\n",
    "for encoder, v in models.items():\n",
    "    for model, lh in v.items():\n",
    "        rh = lh.replace('lh', 'rh')\n",
    "        table[encoder][model] = evaluate(\n",
    "            rh, lh,\n",
    "            beam_k=6\n",
    "        ).compute()['stats']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d2e21230",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbLSTM-soft</th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbLSTM</th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbGNN-soft</th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbGNN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>65.32</td>\n",
       "      <td>71.62</td>\n",
       "      <td>85.21</td>\n",
       "      <td>77.69</td>\n",
       "      <td>1.02</td>\n",
       "      <td>65.34</td>\n",
       "      <td>71.73</td>\n",
       "      <td>85.49</td>\n",
       "      <td>77.80</td>\n",
       "      <td>1.01</td>\n",
       "      <td>65.99</td>\n",
       "      <td>72.43</td>\n",
       "      <td>85.71</td>\n",
       "      <td>77.59</td>\n",
       "      <td>0.94</td>\n",
       "      <td>66.34</td>\n",
       "      <td>72.19</td>\n",
       "      <td>86.31</td>\n",
       "      <td>78.02</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>61.84</td>\n",
       "      <td>68.25</td>\n",
       "      <td>85.05</td>\n",
       "      <td>76.04</td>\n",
       "      <td>1.04</td>\n",
       "      <td>60.41</td>\n",
       "      <td>66.91</td>\n",
       "      <td>84.07</td>\n",
       "      <td>74.75</td>\n",
       "      <td>1.04</td>\n",
       "      <td>62.88</td>\n",
       "      <td>69.66</td>\n",
       "      <td>86.01</td>\n",
       "      <td>76.65</td>\n",
       "      <td>0.96</td>\n",
       "      <td>63.40</td>\n",
       "      <td>69.21</td>\n",
       "      <td>86.55</td>\n",
       "      <td>77.20</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>69.20</td>\n",
       "      <td>75.90</td>\n",
       "      <td>86.12</td>\n",
       "      <td>79.95</td>\n",
       "      <td>1.01</td>\n",
       "      <td>70.69</td>\n",
       "      <td>77.30</td>\n",
       "      <td>87.57</td>\n",
       "      <td>81.36</td>\n",
       "      <td>1.01</td>\n",
       "      <td>69.18</td>\n",
       "      <td>75.34</td>\n",
       "      <td>85.56</td>\n",
       "      <td>78.58</td>\n",
       "      <td>0.94</td>\n",
       "      <td>69.09</td>\n",
       "      <td>75.27</td>\n",
       "      <td>86.19</td>\n",
       "      <td>78.81</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ArThumbLSTM-soft                            ArThumbLSTM                \\\n",
       "                   gmr    hmr    smr    rmr   cpr         gmr    hmr    smr   \n",
       "full             65.32  71.62  85.21  77.69  1.02       65.34  71.73  85.49   \n",
       "right            61.84  68.25  85.05  76.04  1.04       60.41  66.91  84.07   \n",
       "left             69.20  75.90  86.12  79.95  1.01       70.69  77.30  87.57   \n",
       "\n",
       "                   ArThumbGNN-soft                            ArThumbGNN  \\\n",
       "         rmr   cpr             gmr    hmr    smr    rmr   cpr        gmr   \n",
       "full   77.80  1.01           65.99  72.43  85.71  77.59  0.94      66.34   \n",
       "right  74.75  1.04           62.88  69.66  86.01  76.65  0.96      63.40   \n",
       "left   81.36  1.01           69.18  75.34  85.56  78.58  0.94      69.09   \n",
       "\n",
       "                                  \n",
       "         hmr    smr    rmr   cpr  \n",
       "full   72.19  86.31  78.02  0.96  \n",
       "right  69.21  86.55  77.20  0.98  \n",
       "left   75.27  86.19  78.81  0.95  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finetuning_results_frozen = pd.concat({k: pd.DataFrame(v).T for k, v in table['frozen'].items()}, axis=1)\n",
    "finetuning_results_frozen.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "28f24e78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbLSTM-soft</th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbLSTM</th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbGNN-soft</th>\n",
       "      <th colspan=\"5\" halign=\"left\">ArThumbGNN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>65.07</td>\n",
       "      <td>71.58</td>\n",
       "      <td>85.27</td>\n",
       "      <td>77.92</td>\n",
       "      <td>1.02</td>\n",
       "      <td>64.01</td>\n",
       "      <td>69.95</td>\n",
       "      <td>84.32</td>\n",
       "      <td>76.40</td>\n",
       "      <td>1.05</td>\n",
       "      <td>66.84</td>\n",
       "      <td>72.62</td>\n",
       "      <td>86.83</td>\n",
       "      <td>78.55</td>\n",
       "      <td>0.99</td>\n",
       "      <td>66.48</td>\n",
       "      <td>72.75</td>\n",
       "      <td>86.47</td>\n",
       "      <td>78.50</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>60.99</td>\n",
       "      <td>67.38</td>\n",
       "      <td>84.69</td>\n",
       "      <td>75.70</td>\n",
       "      <td>1.03</td>\n",
       "      <td>59.87</td>\n",
       "      <td>65.55</td>\n",
       "      <td>83.84</td>\n",
       "      <td>74.21</td>\n",
       "      <td>1.07</td>\n",
       "      <td>62.77</td>\n",
       "      <td>68.87</td>\n",
       "      <td>86.15</td>\n",
       "      <td>76.60</td>\n",
       "      <td>0.99</td>\n",
       "      <td>63.09</td>\n",
       "      <td>69.62</td>\n",
       "      <td>85.99</td>\n",
       "      <td>77.12</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>69.53</td>\n",
       "      <td>76.67</td>\n",
       "      <td>86.43</td>\n",
       "      <td>80.56</td>\n",
       "      <td>1.00</td>\n",
       "      <td>68.78</td>\n",
       "      <td>75.34</td>\n",
       "      <td>85.70</td>\n",
       "      <td>79.31</td>\n",
       "      <td>1.03</td>\n",
       "      <td>70.92</td>\n",
       "      <td>76.32</td>\n",
       "      <td>87.58</td>\n",
       "      <td>80.37</td>\n",
       "      <td>0.99</td>\n",
       "      <td>70.17</td>\n",
       "      <td>76.34</td>\n",
       "      <td>87.37</td>\n",
       "      <td>80.06</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ArThumbLSTM-soft                            ArThumbLSTM                \\\n",
       "                   gmr    hmr    smr    rmr   cpr         gmr    hmr    smr   \n",
       "full             65.07  71.58  85.27  77.92  1.02       64.01  69.95  84.32   \n",
       "right            60.99  67.38  84.69  75.70  1.03       59.87  65.55  83.84   \n",
       "left             69.53  76.67  86.43  80.56  1.00       68.78  75.34  85.70   \n",
       "\n",
       "                   ArThumbGNN-soft                            ArThumbGNN  \\\n",
       "         rmr   cpr             gmr    hmr    smr    rmr   cpr        gmr   \n",
       "full   76.40  1.05           66.84  72.62  86.83  78.55  0.99      66.48   \n",
       "right  74.21  1.07           62.77  68.87  86.15  76.60  0.99      63.09   \n",
       "left   79.31  1.03           70.92  76.32  87.58  80.37  0.99      70.17   \n",
       "\n",
       "                                  \n",
       "         hmr    smr    rmr   cpr  \n",
       "full   72.75  86.47  78.50  0.97  \n",
       "right  69.62  85.99  77.12  0.99  \n",
       "left   76.34  87.37  80.06  0.96  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finetuning_results = pd.concat({k: pd.DataFrame(v).T for k, v in table['no-frozen'].items()}, axis=1)\n",
    "finetuning_results.round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "098990cd",
   "metadata": {},
   "source": [
    "## The same table than above but without beam decoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3fafe21d",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    \"frozen\": {\n",
    "        \"ArThumbLSTM-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(lstm:ar)_1.pth',\n",
    "        \"ArThumbLSTM\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:ar_1.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(gnn:ar)_1.pth',\n",
    "        \"ArThumbGNN\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#gnn:ar_1.pth'\n",
    "    },\n",
    "    \"no-frozen\":{\n",
    "        \"ArThumbLSTM-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(lstm:ar)_0.pth',\n",
    "        \"ArThumbLSTM\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:ar_0.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(gnn:ar)_0.pth',\n",
    "        \"ArThumbGNN\": 'best_lh_official#nakamura_augmented_seq2seq_separated#finetuning_separated#gnn:ar_0.pth' \n",
    "    }\n",
    "}\n",
    "\n",
    "table = models.copy()\n",
    "for encoder, v in models.items():\n",
    "    for model, lh in v.items():\n",
    "        rh = lh.replace('lh', 'rh')\n",
    "        table[encoder][model] = evaluate(\n",
    "            rh, lh,\n",
    "            beam_k=1\n",
    "        ).compute()['stats']['full']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aef10dac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">frozen</th>\n",
       "      <th>ArThumbLSTM-soft</th>\n",
       "      <td>63.420746</td>\n",
       "      <td>69.399688</td>\n",
       "      <td>83.605792</td>\n",
       "      <td>75.699630</td>\n",
       "      <td>1.015895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>64.251577</td>\n",
       "      <td>70.726135</td>\n",
       "      <td>84.881778</td>\n",
       "      <td>76.749882</td>\n",
       "      <td>1.021898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>63.704248</td>\n",
       "      <td>69.705066</td>\n",
       "      <td>83.566685</td>\n",
       "      <td>74.917050</td>\n",
       "      <td>0.946713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN</th>\n",
       "      <td>63.578682</td>\n",
       "      <td>68.808781</td>\n",
       "      <td>83.699848</td>\n",
       "      <td>74.734472</td>\n",
       "      <td>0.950548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">no-frozen</th>\n",
       "      <th>ArThumbLSTM-soft</th>\n",
       "      <td>62.918320</td>\n",
       "      <td>69.037796</td>\n",
       "      <td>83.128953</td>\n",
       "      <td>75.397212</td>\n",
       "      <td>1.000400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>62.908694</td>\n",
       "      <td>68.533019</td>\n",
       "      <td>83.249880</td>\n",
       "      <td>75.191578</td>\n",
       "      <td>1.019288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>64.698887</td>\n",
       "      <td>70.116245</td>\n",
       "      <td>84.942785</td>\n",
       "      <td>75.925227</td>\n",
       "      <td>0.954638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN</th>\n",
       "      <td>64.498522</td>\n",
       "      <td>70.189606</td>\n",
       "      <td>84.939970</td>\n",
       "      <td>75.851965</td>\n",
       "      <td>0.954637</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  gmr        hmr        smr        rmr  \\\n",
       "frozen    ArThumbLSTM-soft  63.420746  69.399688  83.605792  75.699630   \n",
       "          ArThumbLSTM       64.251577  70.726135  84.881778  76.749882   \n",
       "          ArThumbGNN-soft   63.704248  69.705066  83.566685  74.917050   \n",
       "          ArThumbGNN        63.578682  68.808781  83.699848  74.734472   \n",
       "no-frozen ArThumbLSTM-soft  62.918320  69.037796  83.128953  75.397212   \n",
       "          ArThumbLSTM       62.908694  68.533019  83.249880  75.191578   \n",
       "          ArThumbGNN-soft   64.698887  70.116245  84.942785  75.925227   \n",
       "          ArThumbGNN        64.498522  70.189606  84.939970  75.851965   \n",
       "\n",
       "                                 cpr  \n",
       "frozen    ArThumbLSTM-soft  1.015895  \n",
       "          ArThumbLSTM       1.021898  \n",
       "          ArThumbGNN-soft   0.946713  \n",
       "          ArThumbGNN        0.950548  \n",
       "no-frozen ArThumbLSTM-soft  1.000400  \n",
       "          ArThumbLSTM       1.019288  \n",
       "          ArThumbGNN-soft   0.954638  \n",
       "          ArThumbGNN        0.954637  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ablation_beam_decoder = pd.concat({k: pd.DataFrame(v).T for k, v in table.items()}, axis=0)\n",
    "ablation_beam_decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc6fde3d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f21c9b25",
   "metadata": {},
   "source": [
    "### GNN model and LSTM model trained without domain adaptation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "465540a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    \"ArThumbLSTM\": 'best_lh_official2#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:ar_-1.pth',\n",
    "    \"ArThumbGNN\": 'best_lh_official2#nakamura_augmented_seq2seq_separated#finetuning_separated#gnn:ar_-1.pth'\n",
    "}\n",
    "\n",
    "table = models.copy()\n",
    "for model, lh in models.items():\n",
    "    rh = lh.replace('lh', 'rh')\n",
    "    table[model] = evaluate(\n",
    "        rh, lh,\n",
    "        beam_k=1\n",
    "    ).compute()['stats']['full']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bfc69834",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>26.818374</td>\n",
       "      <td>29.568957</td>\n",
       "      <td>37.379360</td>\n",
       "      <td>31.478006</td>\n",
       "      <td>0.436680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN</th>\n",
       "      <td>60.527140</td>\n",
       "      <td>65.311447</td>\n",
       "      <td>80.379531</td>\n",
       "      <td>69.929908</td>\n",
       "      <td>0.872351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   gmr        hmr        smr        rmr       cpr\n",
       "ArThumbLSTM  26.818374  29.568957  37.379360  31.478006  0.436680\n",
       "ArThumbGNN   60.527140  65.311447  80.379531  69.929908  0.872351"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_PIG = pd.DataFrame(table).transpose()\n",
    "only_PIG"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "909a6ceb",
   "metadata": {},
   "source": [
    "### ABLATION DECODER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "97a33432",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    \"frozen\": {\n",
    "        \"ArThumbLSTM\": 'best_lh_ablation_base_autoregressive_finetuning#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:fc_1.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_ablation_base_autoregressive_finetuning#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(gnn:fc)_1.pth',\n",
    "    },\n",
    "    \"no-frozen\":{\n",
    "        \"ArThumbLSTM\": 'best_lh_ablation_base_autoregressive_finetuning#nakamura_augmented_seq2seq_separated#finetuning_separated#lstm:fc_0.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_ablation_base_autoregressive_finetuning#nakamura_augmented_seq2seq_separated#finetuning_separated#soft(gnn:fc)_0.pth'\n",
    "    }\n",
    "}\n",
    "\n",
    "table = models.copy()\n",
    "for encoder, v in models.items():\n",
    "    for model, lh in v.items():\n",
    "        rh = lh.replace('lh', 'rh')\n",
    "        table[encoder][model] = evaluate(\n",
    "            rh, lh,\n",
    "            alias_architecture = \"fc\"\n",
    "        ).compute()['stats']['full']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9ef9ecc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">frozen</th>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>27.385242</td>\n",
       "      <td>29.790501</td>\n",
       "      <td>36.357534</td>\n",
       "      <td>30.588727</td>\n",
       "      <td>0.314443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>61.590995</td>\n",
       "      <td>65.968255</td>\n",
       "      <td>81.286494</td>\n",
       "      <td>69.277970</td>\n",
       "      <td>0.785787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">no-frozen</th>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>27.385242</td>\n",
       "      <td>29.790501</td>\n",
       "      <td>36.357534</td>\n",
       "      <td>30.588727</td>\n",
       "      <td>0.314443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>63.554797</td>\n",
       "      <td>68.089687</td>\n",
       "      <td>83.563469</td>\n",
       "      <td>71.386255</td>\n",
       "      <td>0.830266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 gmr        hmr        smr        rmr  \\\n",
       "frozen    ArThumbLSTM      27.385242  29.790501  36.357534  30.588727   \n",
       "          ArThumbGNN-soft  61.590995  65.968255  81.286494  69.277970   \n",
       "no-frozen ArThumbLSTM      27.385242  29.790501  36.357534  30.588727   \n",
       "          ArThumbGNN-soft  63.554797  68.089687  83.563469  71.386255   \n",
       "\n",
       "                                cpr  \n",
       "frozen    ArThumbLSTM      0.314443  \n",
       "          ArThumbGNN-soft  0.785787  \n",
       "no-frozen ArThumbLSTM      0.314443  \n",
       "          ArThumbGNN-soft  0.830266  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ablation_decoder = pd.concat({k: pd.DataFrame(v).T for k, v in table.items()}, axis=0)\n",
    "ablation_decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74c60bc9",
   "metadata": {},
   "source": [
    "#### ABLATION data-augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "035a8f92",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    \"frozen\": {\n",
    "        \"ArThumbLSTM\": 'best_lh_ablation_base_augmentation_finetuning#nakamura_no_augmented_seq2seq_separated#finetuning_separated#noauglstm:ar_1.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_ablation_base_augmentation_finetuning#nakamura_no_augmented_seq2seq_separated#finetuning_separated#noaugsoft(gnn:ar)_1.pth',\n",
    "    },\n",
    "    \"no-frozen\":{\n",
    "        \"ArThumbLSTM\": 'best_lh_ablation_base_augmentation_finetuning#nakamura_no_augmented_seq2seq_separated#finetuning_separated#noauglstm:ar_0.pth', \n",
    "        \"ArThumbGNN-soft\": 'best_lh_ablation_base_augmentation_finetuning#nakamura_no_augmented_seq2seq_separated#finetuning_separated#noaugsoft(gnn:ar)_0.pth'\n",
    "    }\n",
    "}\n",
    "\n",
    "table = models.copy()\n",
    "for encoder, v in models.items():\n",
    "    for model, lh in v.items():\n",
    "        rh = lh.replace('lh', 'rh')\n",
    "        table[encoder][model] = evaluate(\n",
    "            rh, lh,\n",
    "            alias_architecture = \"ar\",\n",
    "            beam_k=6,\n",
    "        ).compute()['stats']['full']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8ab00046",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">frozen</th>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>64.692389</td>\n",
       "      <td>70.724903</td>\n",
       "      <td>84.006834</td>\n",
       "      <td>75.967938</td>\n",
       "      <td>1.012891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>57.403008</td>\n",
       "      <td>61.991601</td>\n",
       "      <td>76.427949</td>\n",
       "      <td>66.663674</td>\n",
       "      <td>0.893929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">no-frozen</th>\n",
       "      <th>ArThumbLSTM</th>\n",
       "      <td>65.177536</td>\n",
       "      <td>71.402118</td>\n",
       "      <td>85.149655</td>\n",
       "      <td>76.914269</td>\n",
       "      <td>1.042029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArThumbGNN-soft</th>\n",
       "      <td>60.356269</td>\n",
       "      <td>65.821740</td>\n",
       "      <td>80.443958</td>\n",
       "      <td>70.973047</td>\n",
       "      <td>0.944246</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 gmr        hmr        smr        rmr  \\\n",
       "frozen    ArThumbLSTM      64.692389  70.724903  84.006834  75.967938   \n",
       "          ArThumbGNN-soft  57.403008  61.991601  76.427949  66.663674   \n",
       "no-frozen ArThumbLSTM      65.177536  71.402118  85.149655  76.914269   \n",
       "          ArThumbGNN-soft  60.356269  65.821740  80.443958  70.973047   \n",
       "\n",
       "                                cpr  \n",
       "frozen    ArThumbLSTM      1.012891  \n",
       "          ArThumbGNN-soft  0.893929  \n",
       "no-frozen ArThumbLSTM      1.042029  \n",
       "          ArThumbGNN-soft  0.944246  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ablation_augmentation = pd.concat({k: pd.DataFrame(v).T for k, v in table.items()}, axis=0)\n",
    "ablation_augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b31a8366",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8bcbc36c",
   "metadata": {},
   "source": [
    "# NAKAMURA BASELINES"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28b3230",
   "metadata": {},
   "source": [
    "## HMM-based baselines"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6effe291",
   "metadata": {},
   "source": [
    "### trained on PIG dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "95098841",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    'HMM1': 'official_results_HMM/PIG/HMM1/test_output',\n",
    "    'HMM2': 'official_results_HMM/PIG/HMM2/test_output',\n",
    "    'HMM3': 'official_results_HMM/PIG/HMM3/test_output'\n",
    "}\n",
    "results = models.copy()\n",
    "for hmm_type in ['HMM1', 'HMM2', 'HMM3']:\n",
    "    results[hmm_type] = evaluate(f\"official#PIG#{hmm_type}#0\").compute_from_files(models[hmm_type])['stats']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "86045d4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM1</th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM2</th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>61.77</td>\n",
       "      <td>67.66</td>\n",
       "      <td>81.09</td>\n",
       "      <td>73.15</td>\n",
       "      <td>0.89</td>\n",
       "      <td>63.78</td>\n",
       "      <td>69.49</td>\n",
       "      <td>83.59</td>\n",
       "      <td>76.12</td>\n",
       "      <td>0.98</td>\n",
       "      <td>63.63</td>\n",
       "      <td>69.40</td>\n",
       "      <td>83.05</td>\n",
       "      <td>75.42</td>\n",
       "      <td>0.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>58.34</td>\n",
       "      <td>65.08</td>\n",
       "      <td>81.10</td>\n",
       "      <td>72.04</td>\n",
       "      <td>0.91</td>\n",
       "      <td>60.65</td>\n",
       "      <td>66.72</td>\n",
       "      <td>83.92</td>\n",
       "      <td>75.17</td>\n",
       "      <td>1.02</td>\n",
       "      <td>60.54</td>\n",
       "      <td>66.86</td>\n",
       "      <td>83.38</td>\n",
       "      <td>74.42</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>65.08</td>\n",
       "      <td>70.17</td>\n",
       "      <td>81.10</td>\n",
       "      <td>74.12</td>\n",
       "      <td>0.89</td>\n",
       "      <td>66.30</td>\n",
       "      <td>71.88</td>\n",
       "      <td>82.98</td>\n",
       "      <td>76.61</td>\n",
       "      <td>0.95</td>\n",
       "      <td>66.24</td>\n",
       "      <td>71.70</td>\n",
       "      <td>82.41</td>\n",
       "      <td>75.91</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        HMM1                              HMM2                             \\\n",
       "         gmr    hmr    smr    rmr   cpr    gmr    hmr    smr    rmr   cpr   \n",
       "full   61.77  67.66  81.09  73.15  0.89  63.78  69.49  83.59  76.12  0.98   \n",
       "right  58.34  65.08  81.10  72.04  0.91  60.65  66.72  83.92  75.17  1.02   \n",
       "left   65.08  70.17  81.10  74.12  0.89  66.30  71.88  82.98  76.61  0.95   \n",
       "\n",
       "        HMM3                             \n",
       "         gmr    hmr    smr    rmr   cpr  \n",
       "full   63.63  69.40  83.05  75.42  0.94  \n",
       "right  60.54  66.86  83.38  74.42  0.95  \n",
       "left   66.24  71.70  82.41  75.91  0.95  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmm_results = pd.concat({k: pd.DataFrame(v).T for k, v in results.items()}, axis=1)\n",
    "hmm_results.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "ff9d29d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "models = {\n",
    "    'HMM1': 'official_results_HMM/naka/Res_HMM1',\n",
    "    'HMM2': 'official_results_HMM/naka/Res_HMM2',\n",
    "    'HMM3': 'official_results_HMM/naka/Res_HMM3'\n",
    "}\n",
    "results = models.copy()\n",
    "for hmm_type in ['HMM1', 'HMM2', 'HMM3']:\n",
    "    results[hmm_type] = evaluate(f\"official#PIG#{hmm_type}#0\").compute_from_files(models[hmm_type])['stats']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "da7d32bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM1</th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM2</th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>61.77</td>\n",
       "      <td>67.66</td>\n",
       "      <td>81.09</td>\n",
       "      <td>73.15</td>\n",
       "      <td>0.89</td>\n",
       "      <td>63.78</td>\n",
       "      <td>69.49</td>\n",
       "      <td>83.59</td>\n",
       "      <td>76.12</td>\n",
       "      <td>0.98</td>\n",
       "      <td>63.63</td>\n",
       "      <td>69.40</td>\n",
       "      <td>83.05</td>\n",
       "      <td>75.42</td>\n",
       "      <td>0.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>58.34</td>\n",
       "      <td>65.08</td>\n",
       "      <td>81.10</td>\n",
       "      <td>72.04</td>\n",
       "      <td>0.91</td>\n",
       "      <td>60.65</td>\n",
       "      <td>66.72</td>\n",
       "      <td>83.92</td>\n",
       "      <td>75.17</td>\n",
       "      <td>1.02</td>\n",
       "      <td>60.54</td>\n",
       "      <td>66.86</td>\n",
       "      <td>83.38</td>\n",
       "      <td>74.42</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>65.08</td>\n",
       "      <td>70.17</td>\n",
       "      <td>81.10</td>\n",
       "      <td>74.12</td>\n",
       "      <td>0.89</td>\n",
       "      <td>66.30</td>\n",
       "      <td>71.88</td>\n",
       "      <td>82.98</td>\n",
       "      <td>76.61</td>\n",
       "      <td>0.95</td>\n",
       "      <td>66.24</td>\n",
       "      <td>71.70</td>\n",
       "      <td>82.41</td>\n",
       "      <td>75.91</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        HMM1                              HMM2                             \\\n",
       "         gmr    hmr    smr    rmr   cpr    gmr    hmr    smr    rmr   cpr   \n",
       "full   61.77  67.66  81.09  73.15  0.89  63.78  69.49  83.59  76.12  0.98   \n",
       "right  58.34  65.08  81.10  72.04  0.91  60.65  66.72  83.92  75.17  1.02   \n",
       "left   65.08  70.17  81.10  74.12  0.89  66.30  71.88  82.98  76.61  0.95   \n",
       "\n",
       "        HMM3                             \n",
       "         gmr    hmr    smr    rmr   cpr  \n",
       "full   63.63  69.40  83.05  75.42  0.94  \n",
       "right  60.54  66.86  83.38  74.42  0.95  \n",
       "left   66.24  71.70  82.41  75.91  0.95  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmm_results = pd.concat({k: pd.DataFrame(v).T for k, v in results.items()}, axis=1)\n",
    "hmm_results.round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9979ab3",
   "metadata": {},
   "source": [
    "### Trained on Thumbset dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "2cc551b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "output = {\n",
    "    'HMM1': 'official_results_HMM/ThumbSet/HMM1',\n",
    "    'HMM2': 'official_results_HMM/ThumbSet/HMM2',\n",
    "    'HMM3': 'official_results_HMM/ThumbSet/HMM3'\n",
    "}\n",
    "results = output.copy()\n",
    "for hmm_type in ['HMM1', 'HMM2', 'HMM3']:\n",
    "    results[hmm_type] = evaluate(f\"official#ThumbSet#{hmm_type}#0\").compute_from_files(output[hmm_type])['stats']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "15bf4799",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM1</th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM2</th>\n",
       "      <th colspan=\"5\" halign=\"left\">HMM3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>61.772802</td>\n",
       "      <td>67.661373</td>\n",
       "      <td>81.085877</td>\n",
       "      <td>73.147281</td>\n",
       "      <td>0.894376</td>\n",
       "      <td>63.782128</td>\n",
       "      <td>69.490986</td>\n",
       "      <td>83.593670</td>\n",
       "      <td>76.121843</td>\n",
       "      <td>0.980442</td>\n",
       "      <td>63.626609</td>\n",
       "      <td>69.400962</td>\n",
       "      <td>83.051283</td>\n",
       "      <td>75.424242</td>\n",
       "      <td>0.944073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>58.344865</td>\n",
       "      <td>65.078334</td>\n",
       "      <td>81.096640</td>\n",
       "      <td>72.042244</td>\n",
       "      <td>0.905541</td>\n",
       "      <td>60.646635</td>\n",
       "      <td>66.720136</td>\n",
       "      <td>83.919112</td>\n",
       "      <td>75.168445</td>\n",
       "      <td>1.017210</td>\n",
       "      <td>60.541365</td>\n",
       "      <td>66.860791</td>\n",
       "      <td>83.382152</td>\n",
       "      <td>74.420723</td>\n",
       "      <td>0.951814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>65.082201</td>\n",
       "      <td>70.174935</td>\n",
       "      <td>81.102830</td>\n",
       "      <td>74.119523</td>\n",
       "      <td>0.893736</td>\n",
       "      <td>66.299342</td>\n",
       "      <td>71.881363</td>\n",
       "      <td>82.977060</td>\n",
       "      <td>76.605360</td>\n",
       "      <td>0.950188</td>\n",
       "      <td>66.238776</td>\n",
       "      <td>71.700458</td>\n",
       "      <td>82.413279</td>\n",
       "      <td>75.909720</td>\n",
       "      <td>0.949349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            HMM1                                                  HMM2  \\\n",
       "             gmr        hmr        smr        rmr       cpr        gmr   \n",
       "full   61.772802  67.661373  81.085877  73.147281  0.894376  63.782128   \n",
       "right  58.344865  65.078334  81.096640  72.042244  0.905541  60.646635   \n",
       "left   65.082201  70.174935  81.102830  74.119523  0.893736  66.299342   \n",
       "\n",
       "                                                       HMM3             \\\n",
       "             hmr        smr        rmr       cpr        gmr        hmr   \n",
       "full   69.490986  83.593670  76.121843  0.980442  63.626609  69.400962   \n",
       "right  66.720136  83.919112  75.168445  1.017210  60.541365  66.860791   \n",
       "left   71.881363  82.977060  76.605360  0.950188  66.238776  71.700458   \n",
       "\n",
       "                                       \n",
       "             smr        rmr       cpr  \n",
       "full   83.051283  75.424242  0.944073  \n",
       "right  83.382152  74.420723  0.951814  \n",
       "left   82.413279  75.909720  0.949349  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat({k: pd.DataFrame(v).T for k, v in results.items()}, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e9d1234",
   "metadata": {},
   "source": [
    "### DEEP-BASED baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d99b8e70",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "fc = evaluate(\n",
    "        \"baseline_official_models/right#PIG#classification#nakamura_baseline_ff.pth\",\n",
    "        \"baseline_official_models/left#PIG#classification#nakamura_baseline_ff.pth\",\n",
    "        alias_architecture='windowed_nakamura'\n",
    "    ).compute()['stats']\n",
    "\n",
    "lstm = evaluate(\n",
    "    \"baseline_official_models/right#PIG#classification#nakamura_baseline_lstm.pth\",\n",
    "    \"baseline_official_models/left#PIG#classification#nakamura_baseline_lstm.pth\",\n",
    "    alias_architecture='windowed_nakamura_original'\n",
    ").compute()['stats']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9dfb38c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"5\" halign=\"left\">LSTM-baseline</th>\n",
       "      <th colspan=\"5\" halign=\"left\">FF-baseline</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>62.706237</td>\n",
       "      <td>67.342165</td>\n",
       "      <td>82.687373</td>\n",
       "      <td>71.089658</td>\n",
       "      <td>0.925258</td>\n",
       "      <td>61.440486</td>\n",
       "      <td>66.312495</td>\n",
       "      <td>81.341563</td>\n",
       "      <td>69.820678</td>\n",
       "      <td>0.843684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>58.446861</td>\n",
       "      <td>63.074706</td>\n",
       "      <td>81.580026</td>\n",
       "      <td>67.491235</td>\n",
       "      <td>0.928684</td>\n",
       "      <td>56.890664</td>\n",
       "      <td>61.768745</td>\n",
       "      <td>79.982022</td>\n",
       "      <td>66.567403</td>\n",
       "      <td>0.852543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>67.510308</td>\n",
       "      <td>72.171179</td>\n",
       "      <td>84.560512</td>\n",
       "      <td>75.261245</td>\n",
       "      <td>0.902508</td>\n",
       "      <td>66.379208</td>\n",
       "      <td>71.407713</td>\n",
       "      <td>83.332069</td>\n",
       "      <td>73.585441</td>\n",
       "      <td>0.818459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      LSTM-baseline                                            FF-baseline  \\\n",
       "                gmr        hmr        smr        rmr       cpr         gmr   \n",
       "full      62.706237  67.342165  82.687373  71.089658  0.925258   61.440486   \n",
       "right     58.446861  63.074706  81.580026  67.491235  0.928684   56.890664   \n",
       "left      67.510308  72.171179  84.560512  75.261245  0.902508   66.379208   \n",
       "\n",
       "                                                  \n",
       "             hmr        smr        rmr       cpr  \n",
       "full   66.312495  81.341563  69.820678  0.843684  \n",
       "right  61.768745  79.982022  66.567403  0.852543  \n",
       "left   71.407713  83.332069  73.585441  0.818459  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = {\"LSTM-baseline\": lstm, \"FF-baseline\": fc}\n",
    "dnn_nakamura_results = pd.concat({k: pd.DataFrame(v).T for k, v in results.items()}, axis=1)\n",
    "dnn_nakamura_results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b524b123",
   "metadata": {},
   "source": [
    "# First Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3ffba64b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>61.77</td>\n",
       "      <td>67.66</td>\n",
       "      <td>81.09</td>\n",
       "      <td>73.15</td>\n",
       "      <td>0.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>58.34</td>\n",
       "      <td>65.08</td>\n",
       "      <td>81.10</td>\n",
       "      <td>72.04</td>\n",
       "      <td>0.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>65.08</td>\n",
       "      <td>70.17</td>\n",
       "      <td>81.10</td>\n",
       "      <td>74.12</td>\n",
       "      <td>0.89</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   61.77  67.66  81.09  73.15  0.89\n",
       "right  58.34  65.08  81.10  72.04  0.91\n",
       "left   65.08  70.17  81.10  74.12  0.89"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table = {'HMM1': hmm_results.round(2)['HMM1'].to_dict()}\n",
    "hmm_results.round(2)['HMM1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "badfb88e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>63.78</td>\n",
       "      <td>69.49</td>\n",
       "      <td>83.59</td>\n",
       "      <td>76.12</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>60.65</td>\n",
       "      <td>66.72</td>\n",
       "      <td>83.92</td>\n",
       "      <td>75.17</td>\n",
       "      <td>1.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>66.30</td>\n",
       "      <td>71.88</td>\n",
       "      <td>82.98</td>\n",
       "      <td>76.61</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   63.78  69.49  83.59  76.12  0.98\n",
       "right  60.65  66.72  83.92  75.17  1.02\n",
       "left   66.30  71.88  82.98  76.61  0.95"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table['HMM2'] = hmm_results.round(2)['HMM2'].to_dict()\n",
    "hmm_results.round(2)['HMM2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "15ac8987",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>63.63</td>\n",
       "      <td>69.40</td>\n",
       "      <td>83.05</td>\n",
       "      <td>75.42</td>\n",
       "      <td>0.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>60.54</td>\n",
       "      <td>66.86</td>\n",
       "      <td>83.38</td>\n",
       "      <td>74.42</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>66.24</td>\n",
       "      <td>71.70</td>\n",
       "      <td>82.41</td>\n",
       "      <td>75.91</td>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   63.63  69.40  83.05  75.42  0.94\n",
       "right  60.54  66.86  83.38  74.42  0.95\n",
       "left   66.24  71.70  82.41  75.91  0.95"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table['HMM3'] = hmm_results.round(2)['HMM3'].to_dict()\n",
    "hmm_results.round(2)['HMM3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0174d92f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>61.44</td>\n",
       "      <td>66.31</td>\n",
       "      <td>81.34</td>\n",
       "      <td>69.82</td>\n",
       "      <td>0.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>56.89</td>\n",
       "      <td>61.77</td>\n",
       "      <td>79.98</td>\n",
       "      <td>66.57</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>66.38</td>\n",
       "      <td>71.41</td>\n",
       "      <td>83.33</td>\n",
       "      <td>73.59</td>\n",
       "      <td>0.82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   61.44  66.31  81.34  69.82  0.84\n",
       "right  56.89  61.77  79.98  66.57  0.85\n",
       "left   66.38  71.41  83.33  73.59  0.82"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table['FF-baseline'] = dnn_nakamura_results.round(2)['FF-baseline'].to_dict()\n",
    "dnn_nakamura_results.round(2)['FF-baseline']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "17294670",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>62.71</td>\n",
       "      <td>67.34</td>\n",
       "      <td>82.69</td>\n",
       "      <td>71.09</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>58.45</td>\n",
       "      <td>63.07</td>\n",
       "      <td>81.58</td>\n",
       "      <td>67.49</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>67.51</td>\n",
       "      <td>72.17</td>\n",
       "      <td>84.56</td>\n",
       "      <td>75.26</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   62.71  67.34  82.69  71.09  0.93\n",
       "right  58.45  63.07  81.58  67.49  0.93\n",
       "left   67.51  72.17  84.56  75.26  0.90"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table['LSTM-baseline'] = dnn_nakamura_results.round(2)['LSTM-baseline'].to_dict()\n",
    "dnn_nakamura_results.round(2)['LSTM-baseline']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b69766c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>66.84</td>\n",
       "      <td>72.62</td>\n",
       "      <td>86.83</td>\n",
       "      <td>78.55</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>62.77</td>\n",
       "      <td>68.87</td>\n",
       "      <td>86.15</td>\n",
       "      <td>76.60</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>70.92</td>\n",
       "      <td>76.32</td>\n",
       "      <td>87.58</td>\n",
       "      <td>80.37</td>\n",
       "      <td>0.99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   66.84  72.62  86.83  78.55  0.99\n",
       "right  62.77  68.87  86.15  76.60  0.99\n",
       "left   70.92  76.32  87.58  80.37  0.99"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table['ArThumbGNN-soft'] = finetuning_results.round(2)['ArThumbGNN-soft'].to_dict()\n",
    "finetuning_results.round(2)['ArThumbGNN-soft']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "23403276",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gmr</th>\n",
       "      <th>hmr</th>\n",
       "      <th>smr</th>\n",
       "      <th>rmr</th>\n",
       "      <th>cpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full</th>\n",
       "      <td>65.34</td>\n",
       "      <td>71.73</td>\n",
       "      <td>85.49</td>\n",
       "      <td>77.80</td>\n",
       "      <td>1.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>right</th>\n",
       "      <td>60.41</td>\n",
       "      <td>66.91</td>\n",
       "      <td>84.07</td>\n",
       "      <td>74.75</td>\n",
       "      <td>1.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>left</th>\n",
       "      <td>70.69</td>\n",
       "      <td>77.30</td>\n",
       "      <td>87.57</td>\n",
       "      <td>81.36</td>\n",
       "      <td>1.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         gmr    hmr    smr    rmr   cpr\n",
       "full   65.34  71.73  85.49  77.80  1.01\n",
       "right  60.41  66.91  84.07  74.75  1.04\n",
       "left   70.69  77.30  87.57  81.36  1.01"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_table['ArThumbLSTM-frozen'] = finetuning_results_frozen.round(2)['ArThumbLSTM'].to_dict()\n",
    "finetuning_results_frozen.round(2)['ArThumbLSTM']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6b1596d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FF-baseline\t56.89\t61.77\t79.98\t66.57\t0.85\t66.38\t71.41\t83.33\t73.59\t0.82\t61.44\t66.31\t81.34\t69.82\t0.84\t\n",
      "HMM1\t58.34\t65.08\t81.1\t72.04\t0.91\t65.08\t70.17\t81.1\t74.12\t0.89\t61.77\t67.66\t81.09\t73.15\t0.89\t\n",
      "LSTM-baseline\t58.45\t63.07\t81.58\t67.49\t0.93\t67.51\t72.17\t84.56\t75.26\t0.9\t62.71\t67.34\t82.69\t71.09\t0.93\t\n",
      "HMM2\t60.65\t66.72\t83.92\t75.17\t1.02\t66.3\t71.88\t82.98\t76.61\t0.95\t63.78\t69.49\t83.59\t76.12\t0.98\t\n",
      "HMM3\t60.54\t66.86\t83.38\t74.42\t0.95\t66.24\t71.7\t82.41\t75.91\t0.95\t63.63\t69.4\t83.05\t75.42\t0.94\t\n",
      "ArThumbLSTM-frozen\t60.41\t66.91\t84.07\t74.75\t1.04\t70.69\t77.3\t87.57\t81.36\t1.01\t65.34\t71.73\t85.49\t77.8\t1.01\t\n",
      "ArThumbGNN-soft\t62.77\t68.87\t86.15\t76.6\t0.99\t70.92\t76.32\t87.58\t80.37\t0.99\t66.84\t72.62\t86.83\t78.55\t0.99\t\n"
     ]
    }
   ],
   "source": [
    "for name_model in reversed(['ArThumbGNN-soft', 'ArThumbLSTM-frozen', 'HMM3','HMM2', 'LSTM-baseline', 'HMM1', 'FF-baseline']):\n",
    "    print(f\"{name_model}\\t\", end='')\n",
    "    for hand in ['right', 'left', 'full']:\n",
    "        for metric in ['gmr', 'hmr', 'smr', 'rmr', 'cpr']:\n",
    "            print(f'{first_table[name_model][metric][hand]}\\t', end=\"\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d94190ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "583566ed",
   "metadata": {},
   "source": [
    "##  Table 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "3f3be5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_ablation = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "cbc7bcf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "gnn_gmr = finetuning_results.round(2).to_dict()[('ArThumbGNN-soft', 'gmr')]['full']\n",
    "gnn_cpr = finetuning_results.round(2).to_dict()[('ArThumbGNN-soft', 'cpr')]['full']\n",
    "\n",
    "lstm_gmr = finetuning_results_frozen.round(2).to_dict()[('ArThumbLSTM', 'gmr')]['full']\n",
    "lstm_cpr = finetuning_results_frozen.round(2).to_dict()[('ArThumbLSTM', 'cpr')]['full']\n",
    "\n",
    "table_ablation[\"Models proposed\"] = {\n",
    "    'ArThumbGNN-soft': {\n",
    "        'gmr': gnn_gmr,\n",
    "        'cpr': gnn_cpr\n",
    "    },\n",
    "    'ArThumbLSTM-frozen': {\n",
    "        'gmr': lstm_gmr,\n",
    "        'cpr': lstm_cpr\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5619affc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# soft labels\n",
    "gnn_gmr = finetuning_results.round(2).to_dict()[('ArThumbGNN', 'gmr')]['full']\n",
    "gnn_cpr = finetuning_results.round(2).to_dict()[('ArThumbGNN', 'cpr')]['full']\n",
    "\n",
    "lstm_gmr = finetuning_results_frozen.round(2).to_dict()[('ArThumbLSTM-soft', 'gmr')]['full']\n",
    "lstm_cpr = finetuning_results_frozen.round(2).to_dict()[('ArThumbLSTM-soft', 'cpr')]['full']\n",
    "\n",
    "table_ablation[\"Soft labels ablation\"] = {\n",
    "    'ArThumbGNN': {\n",
    "        'gmr': gnn_gmr,\n",
    "        'cpr': gnn_cpr\n",
    "    },\n",
    "    'ArThumbLSTM-soft-frozen': {\n",
    "        'gmr': lstm_gmr,\n",
    "        'cpr': lstm_cpr\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "6340ab52",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_decoder.round(2).T.to_dict()\n",
    "gnn_gmr = ablation_decoder.round(2).T.to_dict()[('no-frozen', 'ArThumbGNN-soft')]['gmr']\n",
    "gnn_cpr = ablation_decoder.round(2).T.to_dict()[('no-frozen', 'ArThumbGNN-soft')]['cpr']\n",
    "\n",
    "lstm_gmr = ablation_decoder.round(2).T.to_dict()[('frozen', 'ArThumbLSTM')]['gmr']\n",
    "lstm_cpr = ablation_decoder.round(2).T.to_dict()[('frozen', 'ArThumbLSTM')]['cpr']\n",
    "\n",
    "table_ablation[\"Without Ar decoder\"] = {\n",
    "    'FFThumbGNN-soft': {\n",
    "        'gmr': gnn_gmr,\n",
    "        'cpr': gnn_cpr\n",
    "    },\n",
    "    'FFThumbLSTM-frozen': {\n",
    "        'gmr': lstm_gmr,\n",
    "        'cpr': lstm_cpr\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99baec6c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "00d9782b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_beam_decoder.T.round(2).to_dict()\n",
    "\n",
    "ablation_decoder.round(2).T.to_dict()\n",
    "gnn_gmr = ablation_beam_decoder.round(2).T.to_dict()[('no-frozen', 'ArThumbGNN-soft')]['gmr']\n",
    "gnn_cpr = ablation_beam_decoder.round(2).T.to_dict()[('no-frozen', 'ArThumbGNN-soft')]['cpr']\n",
    "\n",
    "lstm_gmr = ablation_beam_decoder.round(2).T.to_dict()[('frozen', 'ArThumbLSTM')]['gmr']\n",
    "lstm_cpr = ablation_beam_decoder.round(2).T.to_dict()[('frozen', 'ArThumbLSTM')]['cpr']\n",
    "\n",
    "table_ablation[\"Without beam decoding\"] = {\n",
    "    'ArThumbGNN-soft': {\n",
    "        'gmr': gnn_gmr,\n",
    "        'cpr': gnn_cpr\n",
    "    },\n",
    "    'ARThumbLSTM-frozen': {\n",
    "        'gmr': lstm_gmr,\n",
    "        'cpr': lstm_cpr\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f35d16bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_augmentation.round(2).T.to_dict()\n",
    "gnn_gmr = ablation_augmentation.round(2).T.to_dict()[('no-frozen', 'ArThumbGNN-soft')]['gmr']\n",
    "gnn_cpr = ablation_augmentation.round(2).T.to_dict()[('no-frozen', 'ArThumbGNN-soft')]['cpr']\n",
    "\n",
    "lstm_gmr = ablation_augmentation.round(2).T.to_dict()[('frozen', 'ArThumbLSTM')]['gmr']\n",
    "lstm_cpr = ablation_augmentation.round(2).T.to_dict()[('frozen', 'ArThumbLSTM')]['cpr']\n",
    "\n",
    "table_ablation[\"Without data augmentation\"] = {\n",
    "    'ArThumbGNN-soft': {\n",
    "        'gmr': gnn_gmr,\n",
    "        'cpr': gnn_cpr\n",
    "    },\n",
    "    'ARThumbLSTM-frozen': {\n",
    "        'gmr': lstm_gmr,\n",
    "        'cpr': lstm_cpr\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e53b04a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ablation frozen\n",
    "gnn_gmr = finetuning_results_frozen.round(2).to_dict()[('ArThumbGNN-soft', 'gmr')]['full']\n",
    "gnn_cpr = finetuning_results_frozen.round(2).to_dict()[('ArThumbGNN-soft', 'cpr')]['full']\n",
    "\n",
    "lstm_gmr = finetuning_results.round(2).to_dict()[('ArThumbLSTM', 'gmr')]['full']\n",
    "lstm_cpr = finetuning_results.round(2).to_dict()[('ArThumbLSTM', 'cpr')]['full']\n",
    "\n",
    "table_ablation[\"Freeze encoder ablation\"] = {\n",
    "    'ArThumbGNN-soft-frozen': {\n",
    "        'gmr': gnn_gmr,\n",
    "        'cpr': gnn_cpr\n",
    "    },\n",
    "    'ArThumbLSTM': {\n",
    "        'gmr': lstm_gmr,\n",
    "        'cpr': lstm_cpr\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8265c638",
   "metadata": {},
   "outputs": [],
   "source": [
    "only_PIG.round(2).to_dict()\n",
    "table_ablation[\"Only PIG training\"] = {\n",
    "    'ArThumbGNN': {\n",
    "        'gmr': only_PIG.round(2).to_dict()['gmr']['ArThumbGNN'],\n",
    "        'cpr': only_PIG.round(2).to_dict()['cpr']['ArThumbGNN']\n",
    "    },\n",
    "    'ArThumbLSTM': {\n",
    "        'gmr': only_PIG.round(2).to_dict()['gmr']['ArThumbLSTM'],\n",
    "        'cpr': only_PIG.round(2).to_dict()['cpr']['ArThumbLSTM']\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6f87b48",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "5327da66",
   "metadata": {},
   "outputs": [],
   "source": [
    "# only_thumbset.round(2).to_dict()\n",
    "only_thumbset.round(2).to_dict()\n",
    "table_ablation[\"Only ThumbSet training\"] = {\n",
    "    'ArThumbGNN-soft': {\n",
    "        'gmr': only_thumbset.round(2).to_dict()['gmr']['ArThumbGNN-soft'],\n",
    "        'cpr': only_thumbset.round(2).to_dict()['cpr']['ArThumbGNN-soft']\n",
    "    },\n",
    "    'ArThumbLSTM': {\n",
    "        'gmr': only_thumbset.round(2).to_dict()['gmr']['ArThumbLSTM'],\n",
    "        'cpr': only_thumbset.round(2).to_dict()['cpr']['ArThumbLSTM']\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "e8140ec2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Models proposed\n",
      "ArThumbGNN-soft\t66.84\t0.00\t0.99\t0.00\t\n",
      "ArThumbLSTM-frozen\t65.34\t0.00\t1.01\t0.00\t\n",
      "Without data augmentation\n",
      "ArThumbGNN-soft\t60.36\t-6.48\t0.94\t-0.05\t\n",
      "ARThumbLSTM-frozen\t64.69\t-0.65\t1.01\t0.00\t\n",
      "Without Ar decoder\n",
      "FFThumbGNN-soft\t63.55\t-3.29\t0.83\t-0.16\t\n",
      "FFThumbLSTM-frozen\t27.39\t-37.95\t0.31\t-0.70\t\n",
      "Without beam decoding\n",
      "ArThumbGNN-soft\t64.7\t-2.14\t0.95\t-0.04\t\n",
      "ARThumbLSTM-frozen\t64.25\t-1.09\t1.02\t0.01\t\n",
      "Soft labels ablation\n",
      "ArThumbGNN\t66.48\t-0.36\t0.97\t-0.02\t\n",
      "ArThumbLSTM-soft-frozen\t65.32\t-0.02\t1.02\t0.01\t\n",
      "Freeze encoder ablation\n",
      "ArThumbGNN-soft-frozen\t65.99\t-0.85\t0.94\t-0.05\t\n",
      "ArThumbLSTM\t64.01\t-1.33\t1.05\t0.04\t\n",
      "Only PIG training\n",
      "ArThumbGNN\t60.53\t-6.31\t0.87\t-0.12\t\n",
      "ArThumbLSTM\t26.82\t-38.52\t0.44\t-0.57\t\n",
      "Only ThumbSet training\n",
      "ArThumbGNN-soft\t65.24\t-1.60\t0.97\t-0.02\t\n",
      "ArThumbLSTM\t64.38\t-0.96\t1.01\t0.00\t\n"
     ]
    }
   ],
   "source": [
    "order_ablation = ['Models proposed', \n",
    "                  'Without data augmentation',\n",
    "                  'Without Ar decoder',\n",
    "                  'Without beam decoding', \n",
    "                  'Soft labels ablation', \n",
    "                  'Freeze encoder ablation', \n",
    "                  'Only PIG training', \n",
    "                  'Only ThumbSet training']\n",
    "gmr_gnn = table_ablation[\"Models proposed\"]['ArThumbGNN-soft']['gmr']\n",
    "cpr_gnn = table_ablation[\"Models proposed\"]['ArThumbGNN-soft']['cpr']\n",
    "gmr_lstm = table_ablation[\"Models proposed\"]['ArThumbLSTM-frozen']['gmr']\n",
    "cpr_lstm = table_ablation[\"Models proposed\"]['ArThumbLSTM-frozen']['cpr']\n",
    "best = {\n",
    "    'GNN':{\n",
    "        'gmr': gmr_gnn,\n",
    "        'cpr': cpr_gnn\n",
    "    },\n",
    "    'LSTM':{\n",
    "        'gmr': gmr_lstm,\n",
    "        'cpr': cpr_lstm\n",
    "    }\n",
    "}\n",
    "for ablation_experiment_name in order_ablation:\n",
    "    print(ablation_experiment_name)\n",
    "    for model_name, model_results in table_ablation[ablation_experiment_name].items():\n",
    "        print(f'{model_name}\\t', end='')\n",
    "        for metric in ['gmr', 'cpr']:\n",
    "            diff = model_results[metric] - best[\"GNN\" if \"GNN\" in model_name else \"LSTM\"][metric]\n",
    "            print(f'{model_results[metric]}\\t{diff:.2f}\\t', end='')\n",
    "        print()\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}