notebooks/2-metrics_computation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e45ecd50-c2b5-4eab-a89a-be9d295c5c7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "from tqdm import tqdm\n",
    "\n",
    "# from bert_score import BERTScorer\n",
    "\n",
    "from easse.bleu import corpus_bleu\n",
    "from easse.fkgl import corpus_fkgl\n",
    "from easse.samsa import get_samsa_sentence_scores\n",
    "from easse.sari import corpus_sari\n",
    "from easse.bertscore import corpus_bertscore\n",
    "\n",
    "from utils import (\n",
    "    read_test_set,\n",
    "    collect_references,\n",
    "    sigmoid,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f8cba4d-1614-4378-8afe-5c253b71685f",
   "metadata": {},
   "source": [
    "## Experimental Setting"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff456ff8-e983-4c9f-95f6-07eee0ba7b8a",
   "metadata": {},
   "source": [
    "Read the datasets with original sentences and references"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e324050f-c0c1-40e4-b2e3-69610dc272bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "asset_orig, asset_refs = read_test_set(\"asset_test\", as_lists=True)\n",
    "turk_orig, turk_refs = read_test_set(\"turkcorpus_test\", as_lists=True)\n",
    "hsplit_orig, hsplit_refs = read_test_set(\"hsplit_test\", as_lists=True)\n",
    "\n",
    "# We create a dataset composed of all references together\n",
    "all_orig = asset_orig\n",
    "all_refs = asset_refs + turk_refs + hsplit_refs\n",
    "\n",
    "EVAL_DATASETS = {\n",
    "    \"asset\": (asset_orig, asset_refs, 10),  # (original, references, number of references)\n",
    "    \"turk\": (turk_orig, turk_refs, 8),\n",
    "    \"hsplit\": (hsplit_orig, hsplit_refs, 4),\n",
    "    \"all\": (all_orig, all_refs, 22)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc973cc2-ec3f-4cfc-be9a-53f8f39b3c07",
   "metadata": {},
   "source": [
    "Same pre-processing parameters for all metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4772f390-736a-4619-8882-47b2ae8c6f78",
   "metadata": {},
   "outputs": [],
   "source": [
    "lowercase = False  # case-insensitive\n",
    "tokenizer = \"moses\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2fd29210-e263-4360-87d1-025126b8b70f",
   "metadata": {},
   "source": [
    "## Compute metrics for the Simplicity-DA dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9cd2ac48-9262-4aa4-9cd5-7fb4a49d8ab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_simplicityDA = pd.read_csv(\"../data/simplicity_DA.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "54401659-e37c-4d70-a4b4-7117d86db985",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>orig_sent</th>\n",
       "      <th>simp_sent</th>\n",
       "      <th>sys_type</th>\n",
       "      <th>fluency</th>\n",
       "      <th>fluency_zscore</th>\n",
       "      <th>meaning</th>\n",
       "      <th>meaning_zscore</th>\n",
       "      <th>simplicity</th>\n",
       "      <th>simplicity_zscore</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sys_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACCESS</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DMASS-DCSS</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dress-Ls</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hybrid</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PBMT-R</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SBMT-SARI</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            sent_id  orig_sent  simp_sent  sys_type  fluency  fluency_zscore  \\\n",
       "sys_name                                                                       \n",
       "ACCESS          100        100        100       100      100             100   \n",
       "DMASS-DCSS      100        100        100       100      100             100   \n",
       "Dress-Ls        100        100        100       100      100             100   \n",
       "Hybrid          100        100        100       100      100             100   \n",
       "PBMT-R          100        100        100       100      100             100   \n",
       "SBMT-SARI       100        100        100       100      100             100   \n",
       "\n",
       "            meaning  meaning_zscore  simplicity  simplicity_zscore  \n",
       "sys_name                                                            \n",
       "ACCESS          100             100         100                100  \n",
       "DMASS-DCSS      100             100         100                100  \n",
       "Dress-Ls        100             100         100                100  \n",
       "Hybrid          100             100         100                100  \n",
       "PBMT-R          100             100         100                100  \n",
       "SBMT-SARI       100             100         100                100  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_simplicityDA.groupby(\"sys_name\").count()  # 100 sentences per system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4524d7ef-3e62-45bc-9a86-c4775c54232d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# bertscore_rescale = BERTScorer(lang=\"en\", rescale_with_baseline=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "79bb15f7-fc32-4c9e-b683-8c0289df314b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "194it [00:52,  3.81it/s]/Users/fernandoalvamanchego/opt/miniconda3/envs/ats/lib/python3.7/site-packages/scipy/stats/stats.py:275: RuntimeWarning: divide by zero encountered in log\n",
      "  log_a = np.log(np.array(a, dtype=dtype))\n",
      "600it [02:38,  3.78it/s]\n"
     ]
    }
   ],
   "source": [
    "metrics = []\n",
    "for _, row in tqdm(df_simplicityDA.iterrows()):\n",
    "    for test_set, (test_set_orig, test_set_refs, num_refs) in EVAL_DATASETS.items():\n",
    "        orig_sents, ref_sents = collect_references(\n",
    "            [row[\"sent_id\"]], test_set_orig, test_set_refs, num_refs\n",
    "        )\n",
    "        \n",
    "        # BLEU\n",
    "        bleu_sys_refs = corpus_bleu(\n",
    "            [row[\"simp_sent\"]],\n",
    "            ref_sents,\n",
    "            smooth_method=\"floor\",\n",
    "            tokenizer=tokenizer,\n",
    "            lowercase=lowercase,\n",
    "            effective_order=True,\n",
    "        )\n",
    "        \n",
    "        # SARI\n",
    "        sari_score = corpus_sari(\n",
    "            orig_sents,\n",
    "            [row[\"simp_sent\"]],\n",
    "            ref_sents,\n",
    "            tokenizer=tokenizer,\n",
    "            lowercase=lowercase,\n",
    "            use_f1_for_deletion=False,\n",
    "        )\n",
    "        \n",
    "        # iBLEU (alpha = 0.9)\n",
    "        bleu_sys_orig = corpus_bleu(\n",
    "            [row[\"simp_sent\"]],\n",
    "            [orig_sents],\n",
    "            force=True,\n",
    "            tokenizer=tokenizer,\n",
    "            lowercase=lowercase,\n",
    "        )\n",
    "        ibleu_score = 0.9 * bleu_sys_refs - (1 - 0.9) * bleu_sys_orig\n",
    "        \n",
    "        # Avg. of BLEU and SARI\n",
    "        amean_bleu_sari = np.mean([bleu_sys_refs, sari_score])\n",
    "        gmean_bleu_sari = scipy.stats.gmean([bleu_sys_refs, sari_score])\n",
    "        \n",
    "        # Flesch\n",
    "        fkgl_sys = corpus_fkgl([row[\"simp_sent\"]], tokenizer=tokenizer)\n",
    "        \n",
    "        # FKBLEU\n",
    "        fkgl_orig = corpus_fkgl(orig_sents, tokenizer=tokenizer)\n",
    "        fk_diff = sigmoid(fkgl_sys - fkgl_orig)\n",
    "        fkbleu_score = ibleu_score * fk_diff\n",
    "                \n",
    "        # BERTScore\n",
    "        # ref_sents = [ref for [ref] in ref_sents]\n",
    "        # bertscore_rescale_scores = bertscore_rescale.score([row[\"simp_sent\"]], [ref_sents])\n",
    "        # bertscores = corpus_bertscore([\n",
    "        #     row[\"simp_sent\"]], \n",
    "        #     ref_sents, \n",
    "        #     tokenizer=tokenizer,\n",
    "        #     lowercase=lowercase\n",
    "        # )\n",
    "        \n",
    "        metrics.append(\n",
    "            {\n",
    "                \"sent_id\": row[\"sent_id\"],\n",
    "                \"sys_name\": row[\"sys_name\"],\n",
    "                \"test_set\": test_set,\n",
    "                \"bleu\": bleu_sys_refs,\n",
    "                \"sari\": sari_score,\n",
    "                \"ibleu\": ibleu_score,\n",
    "                \"amean_bleu_sari\": amean_bleu_sari,\n",
    "                \"gmean_bleu_sari\": gmean_bleu_sari,\n",
    "                \"fkgl\": fkgl_sys,\n",
    "                \"fkbleu\": fkbleu_score,\n",
    "                # \"bertscore_P\": bertscore_rescale_scores[0].cpu().item(),\n",
    "                # \"bertscore_R\": bertscore_rescale_scores[1].cpu().item(),\n",
    "                # \"bertscore_F1\": bertscore_rescale_scores[2].cpu().item(),\n",
    "                # \"bertscore_P\": bertscores[0],\n",
    "                # \"bertscore_R\": bertscores[1],\n",
    "                # \"bertscore_F1\": bertscores[2],\n",
    "            }\n",
    "        )\n",
    "\n",
    "df_metrics_segment = pd.DataFrame(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "830c7653-671c-47f0-99c0-64145f6c0f27",
   "metadata": {},
   "source": [
    "### Compute SAMSA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d372a727-9713-474d-b318-543b97717b52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: SAMSA metric is long to compute (120 sentences ~ 4min), disable it if you need fast evaluation.\n",
      "Loading spaCy model 'en_core_web_md'... \u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the model via spacy.load('en_core_web_md')\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "Failed to get spaCy model. Download it manually using `python -m spacy download en_core_web_md`.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/ucca/textutil.py\u001b[0m in \u001b[0;36mget_nlp\u001b[0;34m(lang)\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m                 \u001b[0mnlp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/spacy/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mdeprecation_warning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mWarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mW001\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepr_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m    135\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 136\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE050\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model 'en_core_web_md'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/ucca/textutil.py\u001b[0m in \u001b[0;36mget_nlp\u001b[0;34m(lang)\u001b[0m\n\u001b[1;32m     81\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m                     \u001b[0mnlp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/spacy/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mdeprecation_warning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mWarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mW001\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepr_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m    135\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 136\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE050\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model 'en_core_web_md'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory.",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-3f12c31e41b5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mdf_simplicityDA\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"simp_sent\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mtokenizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mlowercase\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlowercase\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m )\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Tools/easse/easse/samsa.py\u001b[0m in \u001b[0;36mget_samsa_sentence_scores\u001b[0;34m(orig_sents, sys_sents, lowercase, tokenizer, verbose)\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    275\u001b[0m     \u001b[0morig_sents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mutils_prep\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlowercase\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0morig_sents\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 276\u001b[0;31m     \u001b[0morig_ucca_passages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mucca_parse_texts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_sents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    277\u001b[0m     orig_synt_scenes = syntactic_parse_ucca_scenes(\n\u001b[1;32m    278\u001b[0m         \u001b[0morig_ucca_passages\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Tools/easse/easse/utils/ucca_utils.py\u001b[0m in \u001b[0;36mucca_parse_texts\u001b[0;34m(texts)\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0mpassages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtexts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mpassages\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mucca\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenized\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_parser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0mparsed_passages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mpassage\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mpassage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0m_\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpassages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/ucca/convert.py\u001b[0m in \u001b[0;36mfrom_text\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    766\u001b[0m                 \u001b[0mlayer1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLayer1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    767\u001b[0m                 \u001b[0mparagraph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 768\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mlex\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtextutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_tokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokenized\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    769\u001b[0m                 \u001b[0ml0\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_terminal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0morth_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpunct\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_punct\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparagraph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparagraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    770\u001b[0m             \u001b[0mparagraph\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/ucca/textutil.py\u001b[0m in \u001b[0;36mget_tokenizer\u001b[0;34m(tokenized, lang)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_tokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokenized\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"en\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m     \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_nlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtokenized\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/miniconda3/envs/ats/lib/python3.7/site-packages/ucca/textutil.py\u001b[0m in \u001b[0;36mget_nlp\u001b[0;34m(lang)\u001b[0m\n\u001b[1;32m     83\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m                     raise OSError(\"Failed to get spaCy model. Download it manually using \"\n\u001b[0;32m---> 85\u001b[0;31m                                   \"`python -m spacy download %s`.\" % model) from e\n\u001b[0m\u001b[1;32m     86\u001b[0m             \u001b[0mtokenizer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDoc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: Failed to get spaCy model. Download it manually using `python -m spacy download en_core_web_md`."
     ]
    }
   ],
   "source": [
    "samsa_scores = get_samsa_sentence_scores(\n",
    "    df_simplicityDA[\"orig_sent\"],\n",
    "    df_simplicityDA[\"simp_sent\"],\n",
    "    tokenizer=tokenizer,\n",
    "    lowercase=lowercase,\n",
    ")\n",
    "\n",
    "# Since SAMSA is reference-less, this reformating is only done so that it can appear in thae same dataframe as the other metrics\n",
    "df_metrics_segment[\"samsa\"] = [\n",
    "    s for s in samsa_scores for _ in range(len(EVAL_DATASETS))\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81bea8bf-a310-47a1-9831-64355cc13068",
   "metadata": {},
   "source": [
    "### Compute Additional Averages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cd68dca-3640-4102-92ee-078ac1b9dee9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_metrics_segment['amean_bleu_samsa'] = np.mean(df_metrics_segment[['bleu', 'samsa']], axis=1)\n",
    "df_metrics_segment['amean_sari_samsa'] = np.mean(df_metrics_segment[['sari', 'samsa']], axis=1)\n",
    "df_metrics_segment['amean_bleu_sari_samsa'] = np.mean(df_metrics_segment[['bleu', 'sari', 'samsa']], axis=1)\n",
    "\n",
    "df_metrics_segment['gmean_bleu_samsa'] = scipy.stats.gmean(df_metrics_segment[['bleu', 'samsa']], axis=1)\n",
    "df_metrics_segment['gmean_sari_samsa'] = scipy.stats.gmean(df_metrics_segment[['sari', 'samsa']], axis=1)\n",
    "df_metrics_segment['gmean_bleu_sari_samsa'] = scipy.stats.gmean(df_metrics_segment[['bleu', 'sari', 'samsa']], axis=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ats",
   "language": "python",
   "name": "ats"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}