notebooks/1-annotator_reliability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "specified-nerve",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pingouin as pg\n",
    "from scipy.stats import zscore, spearmanr\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "located-update",
   "metadata": {},
   "source": [
    "# Read Human Ratings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abstract-desire",
   "metadata": {},
   "source": [
    "### Simplicity DA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "suspended-qualification",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings_simp = pd.read_csv(\"../data/ratings_per_annotator/simplicity_DA_ratings.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "partial-update",
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardise_ratings(df, rater_id, aspect):\n",
    "    return df.groupby(by=rater_id)[aspect].transform(lambda x: zscore(x))\n",
    "\n",
    "df_ratings_simp[f\"simplicity_zscore\"] = standardise_ratings(df_ratings_simp, rater_id='rater_id', aspect=\"simplicity\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "german-lemon",
   "metadata": {},
   "source": [
    "### Simplicity Gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "confidential-groove",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings_simpgain = pd.read_csv(\"../data/ratings_per_annotator/simplicity_gain_ratings.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tropical-reynolds",
   "metadata": {},
   "source": [
    "### Structural Simplicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "outstanding-marks",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings_struct = pd.read_csv(\"../data/ratings_per_annotator/structural_simplicity_ratings.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "orange-semiconductor",
   "metadata": {},
   "source": [
    "# Compute ICC"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acoustic-language",
   "metadata": {},
   "source": [
    "### Simplicity-DA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "swedish-latitude",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>ICC</th>\n",
       "      <th>F</th>\n",
       "      <th>df1</th>\n",
       "      <th>df2</th>\n",
       "      <th>pval</th>\n",
       "      <th>CI95%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ICC1</td>\n",
       "      <td>Single raters absolute</td>\n",
       "      <td>0.386</td>\n",
       "      <td>10.436</td>\n",
       "      <td>599</td>\n",
       "      <td>8400</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.36, 0.42]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ICC2</td>\n",
       "      <td>Single random raters</td>\n",
       "      <td>0.386</td>\n",
       "      <td>10.438</td>\n",
       "      <td>599</td>\n",
       "      <td>8386</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.36, 0.42]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ICC3</td>\n",
       "      <td>Single fixed raters</td>\n",
       "      <td>0.386</td>\n",
       "      <td>10.438</td>\n",
       "      <td>599</td>\n",
       "      <td>8386</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.36, 0.42]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ICC1k</td>\n",
       "      <td>Average raters absolute</td>\n",
       "      <td>0.904</td>\n",
       "      <td>10.436</td>\n",
       "      <td>599</td>\n",
       "      <td>8400</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.89, 0.92]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ICC2k</td>\n",
       "      <td>Average random raters</td>\n",
       "      <td>0.904</td>\n",
       "      <td>10.438</td>\n",
       "      <td>599</td>\n",
       "      <td>8386</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.89, 0.92]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ICC3k</td>\n",
       "      <td>Average fixed raters</td>\n",
       "      <td>0.904</td>\n",
       "      <td>10.438</td>\n",
       "      <td>599</td>\n",
       "      <td>8386</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.89, 0.92]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Type              Description    ICC       F  df1   df2  pval  \\\n",
       "0   ICC1   Single raters absolute  0.386  10.436  599  8400   0.0   \n",
       "1   ICC2     Single random raters  0.386  10.438  599  8386   0.0   \n",
       "2   ICC3      Single fixed raters  0.386  10.438  599  8386   0.0   \n",
       "3  ICC1k  Average raters absolute  0.904  10.436  599  8400   0.0   \n",
       "4  ICC2k    Average random raters  0.904  10.438  599  8386   0.0   \n",
       "5  ICC3k     Average fixed raters  0.904  10.438  599  8386   0.0   \n",
       "\n",
       "          CI95%  \n",
       "0  [0.36, 0.42]  \n",
       "1  [0.36, 0.42]  \n",
       "2  [0.36, 0.42]  \n",
       "3  [0.89, 0.92]  \n",
       "4  [0.89, 0.92]  \n",
       "5  [0.89, 0.92]  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reformat the dataset\n",
    "df_ratings_simp['segment_id'] = df_ratings_simp['sent_id'].astype(str) + df_ratings_simp['sys_name']\n",
    "df_ratings_simp[\"rater_num\"] = df_ratings_simp.groupby([\"segment_id\"]).cumcount()\n",
    "\n",
    "# Compute Intraclass Correlation Coeficient (ICC)\n",
    "icc = pg.intraclass_corr(data=df_ratings_simp, targets='segment_id', raters='rater_num', ratings='simplicity_zscore').round(3)\n",
    "\n",
    "icc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "editorial-encounter",
   "metadata": {},
   "source": [
    "### Simplicity Gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "genuine-doctrine",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>ICC</th>\n",
       "      <th>F</th>\n",
       "      <th>df1</th>\n",
       "      <th>df2</th>\n",
       "      <th>pval</th>\n",
       "      <th>CI95%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ICC1</td>\n",
       "      <td>Single raters absolute</td>\n",
       "      <td>0.176</td>\n",
       "      <td>2.067</td>\n",
       "      <td>371</td>\n",
       "      <td>1488</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.13, 0.22]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ICC2</td>\n",
       "      <td>Single random raters</td>\n",
       "      <td>0.179</td>\n",
       "      <td>2.111</td>\n",
       "      <td>371</td>\n",
       "      <td>1484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.14, 0.23]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ICC3</td>\n",
       "      <td>Single fixed raters</td>\n",
       "      <td>0.182</td>\n",
       "      <td>2.111</td>\n",
       "      <td>371</td>\n",
       "      <td>1484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.14, 0.23]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ICC1k</td>\n",
       "      <td>Average raters absolute</td>\n",
       "      <td>0.516</td>\n",
       "      <td>2.067</td>\n",
       "      <td>371</td>\n",
       "      <td>1488</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.43, 0.59]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ICC2k</td>\n",
       "      <td>Average random raters</td>\n",
       "      <td>0.521</td>\n",
       "      <td>2.111</td>\n",
       "      <td>371</td>\n",
       "      <td>1484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.44, 0.59]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ICC3k</td>\n",
       "      <td>Average fixed raters</td>\n",
       "      <td>0.526</td>\n",
       "      <td>2.111</td>\n",
       "      <td>371</td>\n",
       "      <td>1484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.45, 0.6]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Type              Description    ICC      F  df1   df2  pval         CI95%\n",
       "0   ICC1   Single raters absolute  0.176  2.067  371  1488   0.0  [0.13, 0.22]\n",
       "1   ICC2     Single random raters  0.179  2.111  371  1484   0.0  [0.14, 0.23]\n",
       "2   ICC3      Single fixed raters  0.182  2.111  371  1484   0.0  [0.14, 0.23]\n",
       "3  ICC1k  Average raters absolute  0.516  2.067  371  1488   0.0  [0.43, 0.59]\n",
       "4  ICC2k    Average random raters  0.521  2.111  371  1484   0.0  [0.44, 0.59]\n",
       "5  ICC3k     Average fixed raters  0.526  2.111  371  1484   0.0   [0.45, 0.6]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reformat the dataset\n",
    "df_ratings_simpgain['segment_id'] = df_ratings_simpgain['sent_id'].astype(str) + df_ratings_simpgain['sys_name']\n",
    "df_ratings_simpgain[\"rater_num\"] = df_ratings_simpgain.groupby([\"segment_id\"]).cumcount()\n",
    "\n",
    "# Compute Intraclass Correlation Coeficient (ICC)\n",
    "icc = pg.intraclass_corr(data=df_ratings_simpgain, targets='segment_id', raters='rater_num', ratings='simplicity_gain').round(3)\n",
    "\n",
    "icc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "verified-assembly",
   "metadata": {},
   "source": [
    "### Structural Simplicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "authorized-inspection",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Type</th>\n",
       "      <th>Description</th>\n",
       "      <th>ICC</th>\n",
       "      <th>F</th>\n",
       "      <th>df1</th>\n",
       "      <th>df2</th>\n",
       "      <th>pval</th>\n",
       "      <th>CI95%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ICC1</td>\n",
       "      <td>Single raters absolute</td>\n",
       "      <td>0.465</td>\n",
       "      <td>3.605</td>\n",
       "      <td>1749</td>\n",
       "      <td>3500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.44, 0.49]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ICC2</td>\n",
       "      <td>Single random raters</td>\n",
       "      <td>0.476</td>\n",
       "      <td>4.073</td>\n",
       "      <td>1749</td>\n",
       "      <td>3498</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.41, 0.53]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ICC3</td>\n",
       "      <td>Single fixed raters</td>\n",
       "      <td>0.506</td>\n",
       "      <td>4.073</td>\n",
       "      <td>1749</td>\n",
       "      <td>3498</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.48, 0.53]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ICC1k</td>\n",
       "      <td>Average raters absolute</td>\n",
       "      <td>0.723</td>\n",
       "      <td>3.605</td>\n",
       "      <td>1749</td>\n",
       "      <td>3500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.7, 0.74]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ICC2k</td>\n",
       "      <td>Average random raters</td>\n",
       "      <td>0.731</td>\n",
       "      <td>4.073</td>\n",
       "      <td>1749</td>\n",
       "      <td>3498</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.68, 0.77]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ICC3k</td>\n",
       "      <td>Average fixed raters</td>\n",
       "      <td>0.754</td>\n",
       "      <td>4.073</td>\n",
       "      <td>1749</td>\n",
       "      <td>3498</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[0.73, 0.77]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Type              Description    ICC      F   df1   df2  pval  \\\n",
       "0   ICC1   Single raters absolute  0.465  3.605  1749  3500   0.0   \n",
       "1   ICC2     Single random raters  0.476  4.073  1749  3498   0.0   \n",
       "2   ICC3      Single fixed raters  0.506  4.073  1749  3498   0.0   \n",
       "3  ICC1k  Average raters absolute  0.723  3.605  1749  3500   0.0   \n",
       "4  ICC2k    Average random raters  0.731  4.073  1749  3498   0.0   \n",
       "5  ICC3k     Average fixed raters  0.754  4.073  1749  3498   0.0   \n",
       "\n",
       "          CI95%  \n",
       "0  [0.44, 0.49]  \n",
       "1  [0.41, 0.53]  \n",
       "2  [0.48, 0.53]  \n",
       "3   [0.7, 0.74]  \n",
       "4  [0.68, 0.77]  \n",
       "5  [0.73, 0.77]  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reformat the dataset\n",
    "df_ratings_struct['segment_id'] = df_ratings_struct['sent_id'].astype(str) + df_ratings_struct['sys_name']\n",
    "\n",
    "# Compute Intraclass Correlation Coeficient (ICC)\n",
    "icc = pg.intraclass_corr(data=df_ratings_struct, targets='segment_id', raters='rater_id', ratings='structural_simplicity').round(3)\n",
    "\n",
    "icc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ancient-faith",
   "metadata": {},
   "source": [
    "# Compute Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "pretty-verification",
   "metadata": {},
   "outputs": [],
   "source": [
    "def simulate_two_annotators(ratings, num_ratings_annotatorA=1):\n",
    "    ratings_shuffled = np.random.permutation(ratings)\n",
    "    ratingA = np.mean(ratings_shuffled[:num_ratings_annotatorA])\n",
    "    ratingB = np.mean(ratings_shuffled[num_ratings_annotatorA:])\n",
    "    return [ratingA, ratingB]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "surprised-friendly",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_correlation(df_ratings, segment_id, aspects, n_simulations=1000):\n",
    "    corr_per_aspect = {}\n",
    "    for aspect in aspects:\n",
    "        df_scores = df_ratings[[segment_id, aspect]]\n",
    "        corr_values = []\n",
    "        for _ in tqdm(range(n_simulations)):\n",
    "            ratings_simulation = df_scores.groupby(segment_id)[aspect].apply(simulate_two_annotators).to_list()\n",
    "            raterA, raterB = zip(*ratings_simulation)\n",
    "            corr_values.append(spearmanr(raterA, raterB)[0])\n",
    "        corr_per_aspect[aspect] = (np.mean(corr_values), np.std(corr_values))\n",
    "    return corr_per_aspect"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "protective-isaac",
   "metadata": {},
   "source": [
    "### Simplicity-DA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "integral-venice",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:18<00:00, 54.88it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'simplicity_zscore': (0.6062359392691102, 0.024112537162681632)}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_correlation(df_ratings=df_ratings_simp, segment_id=\"segment_id\", aspects=[\"simplicity_zscore\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "committed-intelligence",
   "metadata": {},
   "source": [
    "### Simplicity Gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "needed-harmony",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:11<00:00, 85.72it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'simplicity_gain': (0.3016893780647777, 0.037100423922126737)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_correlation(df_ratings=df_ratings_simpgain, segment_id=\"segment_id\", aspects=[\"simplicity_gain\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "introductory-maine",
   "metadata": {},
   "source": [
    "### Structural Simplicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "extended-dominican",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:53<00:00, 18.67it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'structural_simplicity': (0.5081216794461466, 0.012956320126838313)}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_correlation(df_ratings=df_ratings_struct, segment_id=\"segment_id\", aspects=[\"structural_simplicity\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mechanical-answer",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ats",
   "language": "python",
   "name": "ats"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}