statistical_tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.stats import ttest_ind\n",
    "import pandas as pd\n",
    "rng = np.random.default_rng()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('results_for_plotting.csv', sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(df, experiment):\n",
    "    # experiment in R, noise, low, ood\n",
    "    e_df = df[(df['dir'].str.contains(experiment)) | (df['dir'] =='images')]\n",
    "    e_df = e_df[['base', 'k', 'dir', \"AUC_\"+experiment+\"_negative\"]]\n",
    "    e_df = e_df.rename(columns={\"AUC_\"+experiment+\"_negative\": \"test_AUC\"})\n",
    "    e_df['test_AUC'] *= 100\n",
    "    return e_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = get_data(df, 'low')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>base</th>\n",
       "      <th>k</th>\n",
       "      <th>dir</th>\n",
       "      <th>test_AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>random</td>\n",
       "      <td>5</td>\n",
       "      <td>images_low_100</td>\n",
       "      <td>71.229506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>random</td>\n",
       "      <td>4</td>\n",
       "      <td>images_low_100</td>\n",
       "      <td>71.548843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>random</td>\n",
       "      <td>3</td>\n",
       "      <td>images_low_100</td>\n",
       "      <td>72.042388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>random</td>\n",
       "      <td>2</td>\n",
       "      <td>images_low_100</td>\n",
       "      <td>71.810132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>random</td>\n",
       "      <td>1</td>\n",
       "      <td>images_low_100</td>\n",
       "      <td>26.665700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>random</td>\n",
       "      <td>2</td>\n",
       "      <td>images</td>\n",
       "      <td>71.432722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>ImageNet</td>\n",
       "      <td>2</td>\n",
       "      <td>images</td>\n",
       "      <td>89.301783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>RadImageNet</td>\n",
       "      <td>1</td>\n",
       "      <td>images</td>\n",
       "      <td>89.388877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>414</th>\n",
       "      <td>ImageNet</td>\n",
       "      <td>1</td>\n",
       "      <td>images</td>\n",
       "      <td>84.134132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415</th>\n",
       "      <td>random</td>\n",
       "      <td>1</td>\n",
       "      <td>images</td>\n",
       "      <td>78.501964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>88 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            base  k             dir   test_AUC\n",
       "181       random  5  images_low_100  71.229506\n",
       "183       random  4  images_low_100  71.548843\n",
       "184       random  3  images_low_100  72.042388\n",
       "186       random  2  images_low_100  71.810132\n",
       "191       random  1  images_low_100  26.665700\n",
       "..           ... ..             ...        ...\n",
       "411       random  2          images  71.432722\n",
       "412     ImageNet  2          images  89.301783\n",
       "413  RadImageNet  1          images  89.388877\n",
       "414     ImageNet  1          images  84.134132\n",
       "415       random  1          images  78.501964\n",
       "\n",
       "[88 rows x 4 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stat_test(df, dir):\n",
    "    imnet = df[(df['base'] == 'ImageNet') & (df['dir'] == dir)]['test_AUC']\n",
    "    print(imnet)\n",
    "    radnet = df[(df['base'] == 'RadImageNet') & (df['dir'] == dir)]['test_AUC']\n",
    "    ts, p = ttest_ind(imnet, radnet, permutations=10000,random_state=rng)\n",
    "    return ts, p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "258    15.038468\n",
      "259    20.939177\n",
      "260    28.480187\n",
      "261    14.254609\n",
      "262     6.873277\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-12.886045842331825, 0.007936507936507936)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_low_100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404    83.205122\n",
      "409    88.256645\n",
      "412    89.301783\n",
      "414    84.134132\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.08422552182012047, 0.8857142857142857)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "278    55.058783\n",
      "279    85.498625\n",
      "280    89.091307\n",
      "281    87.617940\n",
      "282    82.203519\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.46356747290221395, 0.7380952380952381)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_low_10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "273    55.247498\n",
      "274    80.476123\n",
      "275    82.450277\n",
      "276    54.797506\n",
      "277    77.195537\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-1.7211483507512213, 0.14285714285714285)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_low_20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "268    60.494995\n",
      "269    60.734504\n",
      "270    59.246623\n",
      "271    54.318476\n",
      "272    54.173321\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-6.569391681063537, 0.007936507936507936)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_low_50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = get_data(df, 'noise')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "309    65.031213\n",
      "310    79.191464\n",
      "311    77.006823\n",
      "312    78.313255\n",
      "313    77.631003\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.05765055238078454, 0.9603174603174603)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_noise_10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "304    63.986063\n",
      "305    72.158510\n",
      "306    74.626213\n",
      "307    70.735955\n",
      "308    73.668164\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.04010286692153065, 0.9761904761904762)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_noise_20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "296    51.589489\n",
      "298    58.121645\n",
      "299    57.265210\n",
      "300    48.642767\n",
      "302    63.898969\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.5674838068558793, 0.5634920634920635)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_noise_50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "290    33.183336\n",
      "291    33.727682\n",
      "292    32.588184\n",
      "294    23.464944\n",
      "295    41.123530\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.8626746467148496, 0.12698412698412698)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_noise_80')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "283     8.941790\n",
      "285    10.117579\n",
      "286     7.286979\n",
      "287     9.384526\n",
      "288     9.820003\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.1314056988988457, 0.9920634920634921)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_noise_100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = get_data(df, 'R')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "384    63.347369\n",
      "386    82.377702\n",
      "387    80.055159\n",
      "388    75.925392\n",
      "390    77.594721\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.7794722897004762, 0.47619047619047616)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_R_20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "376    43.692845\n",
      "378    43.475106\n",
      "379    12.643343\n",
      "382    36.805052\n",
      "383    52.402383\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-5.018592481723199, 0.007936507936507936)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'images_R_50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gender_data(df, experiment):\n",
    "    # experiment in R, noise, low, ood\n",
    "    e_df = df[(df['dir'].str.contains(experiment))]\n",
    "    e_df['dir_n'] = e_df['dir'].apply(lambda x: 0 if x=='images' else int(x[1+len(experiment):]))\n",
    "    e_df = e_df[['base', 'k', 'dir', 'dir_n', \"AUC_gender\"]]\n",
    "    e_df = e_df.rename(columns={\"AUC_gender\": \"test_AUC\"})\n",
    "    e_df['test_AUC'] *= 100\n",
    "    return e_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2116362/1917380332.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  e_df['dir_n'] = e_df['dir'].apply(lambda x: 0 if x=='images' else int(x[1+len(experiment):]))\n"
     ]
    }
   ],
   "source": [
    "new_df = get_gender_data(df, 'F')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "469    70.360315\n",
      "479    68.962497\n",
      "488    69.876558\n",
      "496    68.555939\n",
      "505    69.061250\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.45885143410083534, 0.8174603174603174)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'F_0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "468    63.270932\n",
      "478    59.902191\n",
      "486    71.036875\n",
      "495    63.546246\n",
      "503    66.689068\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-4.063586890183325, 0.007936507936507936)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'F_10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "467    65.910000\n",
      "476    47.644371\n",
      "485    52.112186\n",
      "493    47.995630\n",
      "502    46.004063\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-2.773375350758132, 0.031746031746031744)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'F_20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "465    50.811875\n",
      "474    37.830311\n",
      "484    30.120629\n",
      "492    35.050935\n",
      "501    36.045623\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-4.1647992918848455, 0.007936507936507936)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'F_50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "463    25.080624\n",
      "473    18.915625\n",
      "482    23.653750\n",
      "491    23.233128\n",
      "499    28.279686\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-4.770834983111328, 0.007936507936507936)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'F_80')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "462    12.316562\n",
      "471     7.491249\n",
      "481    11.356252\n",
      "489    11.949375\n",
      "498    10.546875\n",
      "Name: test_AUC, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-5.932602048011992, 0.007936507936507936)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_test(new_df, 'F_100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_diff(df, dir):\n",
    "    imnet = df[(df['base'] == 'ImageNet') & (df['dir'] == dir)]['test_AUC']\n",
    "    radnet = df[(df['base'] == 'RadImageNet') & (df['dir'] == dir)]['test_AUC']\n",
    "    return np.mean(radnet) - np.mean(imnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.567437060000003\n"
     ]
    }
   ],
   "source": [
    "dif = []\n",
    "for d in ['F_10', 'F_20', 'F_50', 'F_80', 'F_100']:\n",
    "    dif.append(mean_diff(new_df, 'F_80'))\n",
    "print(np.mean(dif))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "doju_robust",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}