05_evaluate_ud.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Re-evaluation\n",
    "Perform two evaluations:\n",
    "1. Strict morpheme evaluation\n",
    "1. Token evaluation (morpheme labels are extended to the token level heuristically)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.034028Z",
     "start_time": "2019-03-13T07:20:22.687626Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.382406Z",
     "start_time": "2019-03-13T07:20:24.037019Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:27.996727Z",
     "start_time": "2019-03-13T07:20:24.385088Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_context('paper')\n",
    "sns.set_style('white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/nlp/danb/NER')\n",
    "\n",
    "import bclm\n",
    "import ne_evaluate_mentions as nem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create BIOSE files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def jsonl_to_biose(in_path, out_path, bioul_to_biose=True):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            sent = json.loads(line)\n",
    "            for word, tag in zip(sent['words'], sent['tags']):\n",
    "                if bioul_to_biose:\n",
    "                    tag = tag.replace('L-', 'E-').replace('U-', 'S-')\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            of.write('\\n')\n",
    "            sents+=1\n",
    "    print (sents)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<DirEntry 'morph_54360'>\n",
      "484\n",
      "<DirEntry 'morph_54360'>\n",
      "491\n",
      "<DirEntry 'multi_54360'>\n",
      "484\n",
      "<DirEntry 'multi_54360'>\n",
      "491\n",
      "<DirEntry 'single_54360'>\n",
      "484\n",
      "<DirEntry 'single_54360'>\n",
      "491\n",
      "<DirEntry 'morph_44184'>\n",
      "484\n",
      "<DirEntry 'morph_44184'>\n",
      "491\n",
      "<DirEntry 'multi_44184'>\n",
      "484\n",
      "<DirEntry 'multi_44184'>\n",
      "491\n",
      "<DirEntry 'single_44184'>\n",
      "484\n",
      "<DirEntry 'single_44184'>\n",
      "491\n",
      "<DirEntry 'morph_20423'>\n",
      "484\n",
      "<DirEntry 'morph_20423'>\n",
      "491\n",
      "<DirEntry 'multi_20423'>\n",
      "484\n",
      "<DirEntry 'multi_20423'>\n",
      "491\n",
      "<DirEntry 'single_20423'>\n",
      "484\n",
      "<DirEntry 'single_20423'>\n",
      "491\n",
      "<DirEntry 'morph_80520'>\n",
      "484\n",
      "<DirEntry 'morph_80520'>\n",
      "491\n",
      "<DirEntry 'multi_80520'>\n",
      "484\n",
      "<DirEntry 'multi_80520'>\n",
      "491\n",
      "<DirEntry 'single_80520'>\n",
      "484\n",
      "<DirEntry 'single_80520'>\n",
      "491\n",
      "<DirEntry 'morph_27916'>\n",
      "484\n",
      "<DirEntry 'morph_27916'>\n",
      "491\n",
      "<DirEntry 'multi_27916'>\n",
      "484\n",
      "<DirEntry 'multi_27916'>\n",
      "491\n",
      "<DirEntry 'single_27916'>\n",
      "484\n",
      "<DirEntry 'single_27916'>\n",
      "491\n"
     ]
    }
   ],
   "source": [
    "for trans in os.scandir('ud/output/predict_alephbert'):\n",
    "    for folder in os.scandir(trans):\n",
    "        if not '.ipynb' in folder.name:\n",
    "            for file in os.scandir(folder):\n",
    "                if '.json' in file.name and not '.ipynb' in file.name:\n",
    "                    output_path = file.path.replace('.json', '.bmes')\n",
    "                    if not os.path.exists(output_path):\n",
    "                        print (folder)\n",
    "                        jsonl_to_biose(file.path, output_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bert-basic-wordpiece-otw-52000-cp3']\n"
     ]
    }
   ],
   "source": [
    "include_only = ['bert-basic-wordpiece-otw-52000-cp3',]\n",
    "print(include_only)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SINGLE + MULTI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-basic-wordpiece-otw-52000-cp3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">dev</th>\n",
       "      <th>morph</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>0.817602</td>\n",
       "      <td>0.797137</td>\n",
       "      <td>0.807200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.833291</td>\n",
       "      <td>0.793047</td>\n",
       "      <td>0.812664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.832301</td>\n",
       "      <td>0.815133</td>\n",
       "      <td>0.823587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">test</th>\n",
       "      <th>morph</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>0.767949</td>\n",
       "      <td>0.833761</td>\n",
       "      <td>0.799462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.799619</td>\n",
       "      <td>0.813248</td>\n",
       "      <td>0.806368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.784595</td>\n",
       "      <td>0.823932</td>\n",
       "      <td>0.803713</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                p         r         f\n",
       "set  eval_unit variant prediction align                              \n",
       "dev  morph     morph   gold       -      0.817602  0.797137  0.807200\n",
       "     token     multi   tokens     -      0.833291  0.793047  0.812664\n",
       "               single  tokens     -      0.832301  0.815133  0.823587\n",
       "test morph     morph   gold       -      0.767949  0.833761  0.799462\n",
       "     token     multi   tokens     -      0.799619  0.813248  0.806368\n",
       "               single  tokens     -      0.784595  0.823932  0.803713"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = []\n",
    "for trans in os.scandir('ud/output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if '.ipynb' in folder.name:\n",
    "            continue\n",
    "\n",
    "        variant, seed = folder.name.split('_')\n",
    "\n",
    "        if 'single' in folder.name:    \n",
    "            p,r,f = nem.evaluate_files('../NER/data/ud_ner/token_gold_test_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_test_fix.bmes'), str_join_char='')\n",
    "            res.append(('test', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "\n",
    "        if 'multi' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/ud_ner/token_gold_test_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_test_dummy_o.bmes'), str_join_char='')\n",
    "            res.append(('test', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "        if 'morph' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/ud_ner/morph_gold_test.bmes', \n",
    "                                       os.path.join(folder.path,'morph_gold_test.bmes'), str_join_char='')\n",
    "            res.append(('test', 'morph', variant, 'gold', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "\n",
    "        #dev\n",
    "\n",
    "        if 'single' in folder.name:    \n",
    "            p,r,f = nem.evaluate_files('../NER/data/ud_ner/token_gold_dev_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_dev_fix.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "\n",
    "        if 'multi' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/ud_ner/token_gold_dev_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_dev_dummy_o.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "        if 'morph' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/ud_ner/morph_gold_dev.bmes', \n",
    "                                       os.path.join(folder.path,'morph_gold_dev.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'morph', variant, 'gold', '-', trans_name, seed, p, r, f))\n",
    "    \n",
    "    \n",
    "\n",
    "ne_df = pd.DataFrame(res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'trans_name', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "ne_df.groupby(['set', 'eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Alignments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Token Level Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "o_re = re.compile('^O+$') \n",
    "s_re = re.compile('^O*SO*$|^O*BI*EO*$')\n",
    "b_re = re.compile('^O*BI*$')\n",
    "i_re = re.compile('^I+$')\n",
    "e_re = re.compile('^I*EO*$')\n",
    "def get_fixed_for_valid_biose(bio_seq):\n",
    "    if o_re.match(bio_seq):\n",
    "        return 'O'\n",
    "    if s_re.match(bio_seq):\n",
    "        return 'S'\n",
    "    if b_re.match(bio_seq):\n",
    "        return 'B'\n",
    "    if i_re.match(bio_seq):\n",
    "        return 'I'\n",
    "    if e_re.match(bio_seq):\n",
    "        return 'E'\n",
    "    raise ValueError\n",
    "    \n",
    "\n",
    "def get_fixed_for_invalid_biose(parts):\n",
    "    bio = 'O'\n",
    "    if 'S' in parts:\n",
    "        bio = 'S'\n",
    "    elif 'B' in parts and 'E' in parts:\n",
    "        bio='S'\n",
    "    elif 'E' in parts:\n",
    "        bio = 'E'\n",
    "    elif 'B' in parts:\n",
    "        bio = 'B'\n",
    "    elif 'I' in parts:\n",
    "        bio = 'I'\n",
    "    return bio\n",
    "\n",
    "valid_bio_re = re.compile('^O*BI*$|^O*BI*EO*$|^I+$|^I*EO*$|^O*SO*$')\n",
    "\n",
    "from functools import lru_cache\n",
    "\n",
    "\n",
    "def validate_biose_sequence(full_bio_seq):\n",
    "    #print(full_bio_seq)\n",
    "    bio_seq, type_seq = zip(*[('O', None) if b=='O' else b.split('-') for b in full_bio_seq])\n",
    "    bio_seq = ''.join(bio_seq)\n",
    "    valid_bio = valid_bio_re.match(bio_seq)\n",
    "    type_seq = list(filter(lambda x: x is not None, type_seq))\n",
    "    type_seq_set = set(type_seq)\n",
    "\n",
    "    if valid_bio:\n",
    "        fixed_bio = get_fixed_for_valid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "            \n",
    "    else:\n",
    "        #take the first BIOSE tag which is not O:\n",
    "        #fixed_bio = list(filter(lambda x: x!='O', full_bio_seq))[0]\n",
    "        #rough BIOSE and first category:\n",
    "        fixed_bio = get_fixed_for_invalid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "        \n",
    "    return valid_bio is not None, len(type_seq_set)<=1, fixed_bio\n",
    "\n",
    "\n",
    "@lru_cache(1000)\n",
    "def get_fixed_bio_sequence(full_bio_seq):\n",
    "    return validate_biose_sequence(full_bio_seq)[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re \n",
    "sent_id_re = re.compile('# sent_id = (\\d+)')\n",
    "\n",
    "import bz2\n",
    "def get_sent_ids_lat(path):\n",
    "    sent_ids = []\n",
    "    with bz2.open(path, 'rt', encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            if line[0]=='#':\n",
    "                sid = sent_id_re.match(line)\n",
    "                if sid:\n",
    "                    sent_id =  int(sid.group(1))\n",
    "                sent_ids.append(sent_id)\n",
    "    return sent_ids\n",
    "\n",
    "dev_sent_ids = get_sent_ids_lat('../UL_Hebrew-HTB/he_htb-ud-dev.heblex.conllul.bz2')\n",
    "train_sent_ids = get_sent_ids_lat('../UL_Hebrew-HTB/he_htb-ud-train.heblex.conllul.bz2')\n",
    "test_sent_ids = get_sent_ids_lat('../UL_Hebrew-HTB/he_htb-ud-test.heblex.conllul.bz2')\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "uddf = bclm.read_dataframe('ud')\n",
    "uddf = uddf[(~uddf.global_sent_id.isin(dropped))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ud_set</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dev</th>\n",
       "      <td>1</td>\n",
       "      <td>484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>5726</td>\n",
       "      <td>6216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>485</td>\n",
       "      <td>5725</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         min   max\n",
       "ud_set            \n",
       "dev        1   484\n",
       "test    5726  6216\n",
       "train    485  5725"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_set_from_sent_id(gsi):\n",
    "    if gsi>=min(dev_sent_ids) and gsi<=max(dev_sent_ids):\n",
    "        return 'dev'\n",
    "    elif gsi>=min(train_sent_ids) and gsi<=max(train_sent_ids):\n",
    "        return 'train'\n",
    "    elif gsi>=min(test_sent_ids) and gsi<=max(test_sent_ids):\n",
    "        return 'test'\n",
    "    \n",
    "uddf['ud_set'] = uddf.sent_id.apply(get_set_from_sent_id)\n",
    "\n",
    "uddf.groupby(['ud_set']).sent_id.agg(['min', 'max'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "dev_gold = uddf[uddf.ud_set=='dev']\n",
    "test_gold = uddf[uddf.ud_set=='test']\n",
    "test_gold['sent_id'] = test_gold.sent_id.rank(method='dense').astype(int)\n",
    "dev_yap = bclm.read_yap_output(treebank_set='dev')\n",
    "test_yap = bclm.read_yap_output(treebank_set='test')\n",
    "dev_gold_sents = bclm.get_sentences_list(dev_gold, fields=['token_id', 'token_str'])\n",
    "test_gold_sents = bclm.get_sentences_list(test_gold, fields=['token_id', 'token_str'])\n",
    "dev_yap_sents = bclm.get_sentences_list(dev_yap, fields=['token_id', 'token_str'])\n",
    "test_yap_sents = bclm.get_sentences_list(test_yap, fields=['token_id', 'token_str'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/pandas/core/frame.py:3509: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "dev_gold_tok = (bclm.get_token_df(dev_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok = (bclm.get_token_df(test_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok['sent_id'] = test_gold_tok.sent_id.rank(method='dense').astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fixed_tok(path, orig_sents=dev_yap_sents):\n",
    "    x = nem.read_file_sents(path, fix_multi_tag=False)\n",
    "    new_sents = []\n",
    "    for (i, ner_sent), (sent_id, yap_sent) in zip(x.iteritems(), orig_sents.iteritems()):\n",
    "        for (form, bio), (token_id, token_str) in zip(ner_sent, yap_sent):\n",
    "            new_sents.append((sent_id, token_id, token_str, form, bio))\n",
    "    new_sents = pd.DataFrame(new_sents, columns=['sent_id', 'token_id', 'token_str', 'form', 'bio'])\n",
    "    new_toks = bclm.get_token_df(new_sents, fields=['bio'])\n",
    "    new_toks['fixed_bio'] = new_toks.bio.apply(lambda x: get_fixed_bio_sequence(tuple(x.split('^'))))\n",
    "    return new_toks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sents_from_df(df, sent_id_col='sent_id', \n",
    "                  group_cols=['token_str'], \n",
    "                  val_cols=['fixed_bio']):\n",
    "    sents = bclm.get_sentences_list(df, fields=group_cols+val_cols)\n",
    "    return sents\n",
    "\n",
    "def evaluate_dataframes(gold_df, pred_df, fix_multi_tag_pred=True, truncate=None, ignore_cat=False, str_join_char=' '):\n",
    "    gold_sents = sents_from_df(gold_df)\n",
    "    pred_sents = sents_from_df(pred_df)\n",
    "    gold_mentions = nem.sents_to_mentions(gold_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    pred_mentions = nem.sents_to_mentions(pred_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    return nem.evaluate_mentions(gold_mentions, pred_mentions, verbose=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run on all pruned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "@lru_cache(512)\n",
    "def get_sent_list(ds, dp, mp):\n",
    "    prun_yo = get_prun_yo(ds, dp, mp)\n",
    "    return bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run all gold and YAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-basic-wordpiece-otw-52000-cp3\n"
     ]
    }
   ],
   "source": [
    "align_tok_res_yg = []\n",
    "for trans in os.scandir('ud/output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "            ## dev \n",
    "            ## - gold\n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_gold_dev.bmes')\n",
    "            out_path = os.path.join(folder.path, 'morph_gold_dev_align_tokens.bmes')\n",
    "\n",
    "            new_toks = get_fixed_tok(file, orig_sents=dev_gold_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "\n",
    "            p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res_yg.append(('dev', 'token', 'morph', 'gold', 'tokens', trans_name, seed, p, r, f))\n",
    "\n",
    "\n",
    "            ## test \n",
    "            ## - gold\n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_gold_test.bmes')\n",
    "            out_path = os.path.join(folder.path, 'morph_gold_test_align_tokens.bmes')\n",
    "\n",
    "            new_toks = get_fixed_tok(file, orig_sents=test_gold_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "\n",
    "            p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res_yg.append(('test', 'token', 'morph', 'gold', 'tokens', trans_name, seed, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>trans_name</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dev</th>\n",
       "      <th>bert-basic-wordpiece-otw-52000-cp3</th>\n",
       "      <th>token</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.824728</td>\n",
       "      <td>0.804090</td>\n",
       "      <td>0.814238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <th>bert-basic-wordpiece-otw-52000-cp3</th>\n",
       "      <th>token</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.769358</td>\n",
       "      <td>0.834615</td>\n",
       "      <td>0.800617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                    p  \\\n",
       "set  trans_name                         eval_unit variant prediction align              \n",
       "dev  bert-basic-wordpiece-otw-52000-cp3 token     morph   gold       tokens  0.824728   \n",
       "test bert-basic-wordpiece-otw-52000-cp3 token     morph   gold       tokens  0.769358   \n",
       "\n",
       "                                                                                    r  \\\n",
       "set  trans_name                         eval_unit variant prediction align              \n",
       "dev  bert-basic-wordpiece-otw-52000-cp3 token     morph   gold       tokens  0.804090   \n",
       "test bert-basic-wordpiece-otw-52000-cp3 token     morph   gold       tokens  0.834615   \n",
       "\n",
       "                                                                                    f  \n",
       "set  trans_name                         eval_unit variant prediction align             \n",
       "dev  bert-basic-wordpiece-otw-52000-cp3 token     morph   gold       tokens  0.814238  \n",
       "test bert-basic-wordpiece-otw-52000-cp3 token     morph   gold       tokens  0.800617  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res_yg, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'trans_name', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morpheme Level Eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Token Multi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### YAP + GOLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_merge_bio_labels(multitok_sents, tokmorph_sents, verbose=False):\n",
    "    new_sents = []\n",
    "    for (i, mt_sent), (sent_id, mor_sent) in zip(multitok_sents.iteritems(), tokmorph_sents.iteritems()):\n",
    "        new_sent = []\n",
    "        for (form, bio), (token_id, token_str, forms) in zip(mt_sent, mor_sent):\n",
    "            forms = forms.split('^')\n",
    "            bio = bio.split('^')\n",
    "            if len(forms) == len(bio):\n",
    "                new_forms = (1, list(zip(forms,bio)))\n",
    "            elif len(forms)>len(bio):\n",
    "                dif = len(forms) - len(bio)\n",
    "                new_forms = (2, list(zip(forms[:dif],['O']*dif)) + list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            else:\n",
    "                new_forms = (3, list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            new_sent.extend(new_forms[1])\n",
    "        new_sents.append(new_sent)\n",
    "    return new_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "decode_sets = {\n",
    "    'token': {\n",
    "        'dev': '../NER/data/ud_ner/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/ud_ner/morph_gold_test.bmes',\n",
    "    },\n",
    "    'multitok': {\n",
    "        'dev': '../NER/data/ud_ner/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/ud_ner/morph_gold_test.bmes',\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok_yg(ner_pred_path, prun_sents, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_morph = {'dev': dev_gold, 'test': test_gold}\n",
    "def get_sents_for_mult(treebank_set, gold=False, pred_set=None, \n",
    "                       dep_path=None, map_path=None):\n",
    "    if treebank_set is None:\n",
    "        prun_yo = get_prun_yo(pred_set, dep_path, map_path)\n",
    "    else:\n",
    "        if not gold:\n",
    "            prun_yo = bclm.read_yap_output(treebank_set=treebank_set)\n",
    "        else:\n",
    "            prun_yo = gold_morph[treebank_set]\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    return prun_sents\n",
    "\n",
    "# dev_yap_sents_m = get_sents_for_mult('dev')\n",
    "# test_yap_sents_m = get_sents_for_mult('test')\n",
    "dev_gold_sents_m = get_sents_for_mult('dev', gold=True)\n",
    "test_gold_sents_m = get_sents_for_mult('test', gold=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-basic-wordpiece-otw-52000-cp3\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_gold = []\n",
    "for trans in os.scandir('ud/output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            #dev\n",
    "            gold_ner_path=os.path.join(folder.path, 'morph_gold_dev.bmes')\n",
    "\n",
    "            align_multitok_yg(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                               dev_gold_sents_m,\n",
    "                               gold_ner_path\n",
    "                              )\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], gold_ner_path, str_join_char='')\n",
    "            align_morph_res_gold.append(('dev', 'morph', 'multi', 'tokens', 'gold', trans_name, seed, p, r, f))\n",
    "\n",
    "            #test\n",
    "            gold_ner_path=os.path.join(folder.path, 'morph_gold_test.bmes')\n",
    "\n",
    "            align_multitok_yg(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                               test_gold_sents_m,\n",
    "                               gold_ner_path\n",
    "                              )\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], gold_ner_path, str_join_char='')\n",
    "            align_morph_res_gold.append(('test', 'morph', 'multi', 'tokens', 'gold', trans_name, seed, p, r, f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ALL SCORES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "at_df = pd.DataFrame(align_tok_res_yg+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', \n",
    "                                                                                                                           'prediction', 'align', 'trans_name', \n",
    "                                                                                                                           'seed', 'p', 'r', 'f'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALL_DF_PATH = 'ud/output/all_results_alephbert.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pd.read_csv(ALL_DF_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "min    5\n",
       "max    5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df = pd.concat([all_df, at_df, ne_df])\n",
    "all_df.groupby(['set', 'eval_unit','variant', \n",
    "                'prediction', 'align','trans_name']).size().agg([min,max])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoints = ['heBERT', \n",
    "#                '2k', '4k', '8k',\n",
    "#                                              '16k', '32k', '52k', '64k',\n",
    "#                                              '128k', 'unichar_improved_52k', \n",
    "#                                              'unichar_improved_with_hash_52k',\n",
    "               'bert-distilled-wordpiece-oscar-2000',\n",
    "                'bert-distilled-wordpiece-oscar-16000',\n",
    "                'bert-distilled-wordpiece-oscar-52000',\n",
    "                'bert-basic-wordpiece-otw-52000-cp3',\n",
    "              ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>trans_name</th>\n",
       "      <th>heBERT</th>\n",
       "      <th>bert-distilled-wordpiece-oscar-2000</th>\n",
       "      <th>bert-distilled-wordpiece-oscar-16000</th>\n",
       "      <th>bert-distilled-wordpiece-oscar-52000</th>\n",
       "      <th>bert-basic-wordpiece-otw-52000-cp3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">dev</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">morph</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>81.247928</td>\n",
       "      <td>74.418670</td>\n",
       "      <td>78.209848</td>\n",
       "      <td>79.302769</td>\n",
       "      <td>80.719959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>80.100053</td>\n",
       "      <td>74.279756</td>\n",
       "      <td>76.922571</td>\n",
       "      <td>80.941309</td>\n",
       "      <td>80.386946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>81.669913</td>\n",
       "      <td>74.632728</td>\n",
       "      <td>78.803933</td>\n",
       "      <td>79.933183</td>\n",
       "      <td>81.423755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>81.012373</td>\n",
       "      <td>74.052065</td>\n",
       "      <td>77.097025</td>\n",
       "      <td>81.770413</td>\n",
       "      <td>81.266358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>83.009847</td>\n",
       "      <td>74.620953</td>\n",
       "      <td>78.509327</td>\n",
       "      <td>83.504978</td>\n",
       "      <td>82.358662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">morph</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>81.133317</td>\n",
       "      <td>70.221607</td>\n",
       "      <td>77.655903</td>\n",
       "      <td>77.573463</td>\n",
       "      <td>79.946230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>79.751971</td>\n",
       "      <td>69.463298</td>\n",
       "      <td>74.504304</td>\n",
       "      <td>76.010495</td>\n",
       "      <td>79.848530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th>morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>81.217558</td>\n",
       "      <td>70.262844</td>\n",
       "      <td>77.925595</td>\n",
       "      <td>78.027602</td>\n",
       "      <td>80.061666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>80.043647</td>\n",
       "      <td>69.515953</td>\n",
       "      <td>74.716462</td>\n",
       "      <td>76.963570</td>\n",
       "      <td>80.636783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>79.515853</td>\n",
       "      <td>67.686947</td>\n",
       "      <td>76.241035</td>\n",
       "      <td>76.413808</td>\n",
       "      <td>80.371276</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "trans_name                                   heBERT  \\\n",
       "set  eval_unit variant prediction align               \n",
       "dev  morph     morph   gold       -       81.247928   \n",
       "               multi   tokens     gold    80.100053   \n",
       "     token     morph   gold       tokens  81.669913   \n",
       "               multi   tokens     -       81.012373   \n",
       "               single  tokens     -       83.009847   \n",
       "test morph     morph   gold       -       81.133317   \n",
       "               multi   tokens     gold    79.751971   \n",
       "     token     morph   gold       tokens  81.217558   \n",
       "               multi   tokens     -       80.043647   \n",
       "               single  tokens     -       79.515853   \n",
       "\n",
       "trans_name                                bert-distilled-wordpiece-oscar-2000  \\\n",
       "set  eval_unit variant prediction align                                         \n",
       "dev  morph     morph   gold       -                                 74.418670   \n",
       "               multi   tokens     gold                              74.279756   \n",
       "     token     morph   gold       tokens                            74.632728   \n",
       "               multi   tokens     -                                 74.052065   \n",
       "               single  tokens     -                                 74.620953   \n",
       "test morph     morph   gold       -                                 70.221607   \n",
       "               multi   tokens     gold                              69.463298   \n",
       "     token     morph   gold       tokens                            70.262844   \n",
       "               multi   tokens     -                                 69.515953   \n",
       "               single  tokens     -                                 67.686947   \n",
       "\n",
       "trans_name                                bert-distilled-wordpiece-oscar-16000  \\\n",
       "set  eval_unit variant prediction align                                          \n",
       "dev  morph     morph   gold       -                                  78.209848   \n",
       "               multi   tokens     gold                               76.922571   \n",
       "     token     morph   gold       tokens                             78.803933   \n",
       "               multi   tokens     -                                  77.097025   \n",
       "               single  tokens     -                                  78.509327   \n",
       "test morph     morph   gold       -                                  77.655903   \n",
       "               multi   tokens     gold                               74.504304   \n",
       "     token     morph   gold       tokens                             77.925595   \n",
       "               multi   tokens     -                                  74.716462   \n",
       "               single  tokens     -                                  76.241035   \n",
       "\n",
       "trans_name                                bert-distilled-wordpiece-oscar-52000  \\\n",
       "set  eval_unit variant prediction align                                          \n",
       "dev  morph     morph   gold       -                                  79.302769   \n",
       "               multi   tokens     gold                               80.941309   \n",
       "     token     morph   gold       tokens                             79.933183   \n",
       "               multi   tokens     -                                  81.770413   \n",
       "               single  tokens     -                                  83.504978   \n",
       "test morph     morph   gold       -                                  77.573463   \n",
       "               multi   tokens     gold                               76.010495   \n",
       "     token     morph   gold       tokens                             78.027602   \n",
       "               multi   tokens     -                                  76.963570   \n",
       "               single  tokens     -                                  76.413808   \n",
       "\n",
       "trans_name                                bert-basic-wordpiece-otw-52000-cp3  \n",
       "set  eval_unit variant prediction align                                       \n",
       "dev  morph     morph   gold       -                                80.719959  \n",
       "               multi   tokens     gold                             80.386946  \n",
       "     token     morph   gold       tokens                           81.423755  \n",
       "               multi   tokens     -                                81.266358  \n",
       "               single  tokens     -                                82.358662  \n",
       "test morph     morph   gold       -                                79.946230  \n",
       "               multi   tokens     gold                             79.848530  \n",
       "     token     morph   gold       tokens                           80.061666  \n",
       "               multi   tokens     -                                80.636783  \n",
       "               single  tokens     -                                80.371276  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_scores = (all_df\n",
    "               .groupby(['set', 'eval_unit','variant', \n",
    "                         'prediction', 'align','trans_name'])\n",
    "               .f.mean().unstack().mul(100)[checkpoints]\n",
    "              )\n",
    "mean_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', \n",
    "                                                                                                                           'prediction', 'align', 'trans_name', \n",
    "                                                                                                                           'seed', 'p', 'r', 'f'])\n",
    "\n",
    "(at_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "(ne_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df.to_csv(ALL_DF_PATH, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_scores.reset_index().to_csv('ud/output/mean_results_alephbert.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}