score_correlation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import blockers\n",
    "import pruners\n",
    "import torch_subspace\n",
    "from experiments.models import get_model\n",
    "from experiments.data import get_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load model\n",
    "We compute different pruning scores on the same model with the same blocking scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:7\")\n",
    "train_data, test_data, num_classes = get_data(\n",
    "    \"cifar10\", batch_size=256, data_path=\"./data\"\n",
    ")\n",
    "model = get_model(\"vgg16\", num_classes=num_classes, device=device)\n",
    "model.load_state_dict(torch.load(\"./checkpoint/vgg16_cifar10_epoch_160\"))\n",
    "torch_subspace.convert_model_to_lr(model)\n",
    "blockers.square.make_blocks(model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pruning score\n",
    "We calculate scores for two different pruning methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating rel error scores\n",
      "Scoring module: 146 / 146\n",
      "Calculating magnitude scores\n",
      "Scoring module: 146 / 146\n",
      "Calculating alignment output scores\n",
      "Scoring module 146 / 146 (mask    9 /   10)\n",
      "Calculating alignment output sampling scores\n",
      "Iter: 999\n",
      "Calculating alignment variance scores\n",
      "Scoring module 146 / 146 (mask    9 /   10)\n"
     ]
    }
   ],
   "source": [
    "scores = {}\n",
    "print(\"Calculating rel error scores\")\n",
    "scores[\"rel_error\"] = pruners.rel_error._compute_scores(model=model)\n",
    "print(\"Calculating magnitude scores\")\n",
    "scores[\"magnitude\"] = pruners.magnitude._compute_scores(model=model)\n",
    "print(\"Calculating alignment output scores\")\n",
    "scores[\"alignment_output\"] = pruners.alignment_output._compute_scores(model=model, train_data=train_data, device=device)\n",
    "print(\"Calculating alignment output sampling scores\")\n",
    "scores[\"alignment_output_sampling\"] = pruners.alignment_output_sampling._compute_scores(model=model, train_data=train_data, sparsity=0.95, proportional_sampling=False, device=device)\n",
    "print(\"Calculating alignment variance scores\")\n",
    "scores[\"alignment_variance\"] = pruners.alignment_variance._compute_scores(model, train_data, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate correlations\n",
    "First we look at block by block correlation in scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rel_error</th>\n",
       "      <th>magnitude</th>\n",
       "      <th>alignment_output</th>\n",
       "      <th>alignment_output_sampling</th>\n",
       "      <th>alignment_variance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rel_error</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.906422</td>\n",
       "      <td>0.872102</td>\n",
       "      <td>0.008642</td>\n",
       "      <td>0.303540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magnitude</th>\n",
       "      <td>0.906422</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.899813</td>\n",
       "      <td>0.011311</td>\n",
       "      <td>0.277830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alignment_output</th>\n",
       "      <td>0.872102</td>\n",
       "      <td>0.899813</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.008240</td>\n",
       "      <td>0.290989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alignment_output_sampling</th>\n",
       "      <td>0.008642</td>\n",
       "      <td>0.011311</td>\n",
       "      <td>0.008240</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.051718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alignment_variance</th>\n",
       "      <td>0.303540</td>\n",
       "      <td>0.277830</td>\n",
       "      <td>0.290989</td>\n",
       "      <td>0.051718</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           rel_error  magnitude  alignment_output  \\\n",
       "rel_error                   1.000000   0.906422          0.872102   \n",
       "magnitude                   0.906422   1.000000          0.899813   \n",
       "alignment_output            0.872102   0.899813          1.000000   \n",
       "alignment_output_sampling   0.008642   0.011311          0.008240   \n",
       "alignment_variance          0.303540   0.277830          0.290989   \n",
       "\n",
       "                           alignment_output_sampling  alignment_variance  \n",
       "rel_error                                   0.008642            0.303540  \n",
       "magnitude                                   0.011311            0.277830  \n",
       "alignment_output                            0.008240            0.290989  \n",
       "alignment_output_sampling                   1.000000            0.051718  \n",
       "alignment_variance                          0.051718            1.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_flat = {}\n",
    "for key, val in scores.items():\n",
    "    scores_flat[key] = np.concatenate(val)\n",
    "scores_flat = pd.DataFrame(scores_flat)\n",
    "scores_flat.corr(method=\"spearman\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5441360298008848, pvalue=0.0033454791078566955)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5875879087686505, pvalue=0.0012699092296497645)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.313646563942557, pvalue=0.011610939343114749)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.41593113866797005, pvalue=0.000630482071238406)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.566877717681815, pvalue=1.0427563360972673e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.41582201335282687, pvalue=0.0006327740524635211)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.40959016839883955, pvalue=0.0007769108711224999)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5237831667945594, pvalue=8.932518668208072e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.33077274140773455, pvalue=0.007593547307188323)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3152716756728294, pvalue=0.011163536910865057)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.36615017368981645, pvalue=0.002924686088070503)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.44312307925983707, pvalue=1.6232210140808373e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3138031132510888, pvalue=0.0003097558336007699)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2683139172991724, pvalue=0.0021971542592444846)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3121074163788046, pvalue=0.0003351811308893507)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.1448781989967031, pvalue=0.1027530318814004)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.35965022628172805, pvalue=3.0532968069587144e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3569206492664434, pvalue=3.541192075065005e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3895920776525574, pvalue=5.482705841514589e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3519047240846283, pvalue=4.63377124578301e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.32649327943626083, pvalue=0.00016908353044654966)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3628935020483373, pvalue=2.5556810576032098e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.39826589522976485, pvalue=3.227850814931473e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.49286006120827314, pvalue=3.423093515058128e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.278453918826584, pvalue=6.090081378235204e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.36884381762525065, pvalue=1.1386688887664503e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3283817077696896, pvalue=7.520138071415935e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3129887509661332, pvalue=3.181982836158819e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.4059044106037585, pvalue=1.421010502169001e-11)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.38831701703438287, pvalue=1.218255191654417e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3278088389713716, pvalue=7.946409643690405e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.33069098205254593, pvalue=6.014517936579406e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.25862742343952677, pvalue=2.798377214010773e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3438985525280503, pvalue=1.6168958814400216e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.35086848798943865, pvalue=7.883472514827671e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3372492564659127, pvalue=3.156810957532193e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3586196894004602, pvalue=3.4734135155433794e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3489602302399785, pvalue=9.613619337348916e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2717350394910293, pvalue=1.0349455040938983e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.34156056980598504, pvalue=2.0494055208649718e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.44388488036462775, pvalue=8.7441767266702e-14)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.29639612662968196, pvalue=1.378728681556066e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.26929492269271804, pvalue=1.250425021599885e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.31327944672905594, pvalue=3.0987700889656343e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2720903741159721, pvalue=1.006676255836672e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.36484817691786026, pvalue=1.7689629489590453e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.4288414466011923, pvalue=2.5459148077672783e-24)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.38042992208537363, pvalue=4.461851737642954e-19)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2695466347399826, pvalue=5.669923450539396e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.35360374727353894, pvalue=1.5893717222411191e-16)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.26104821435244635, pvalue=2.013703338627354e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.25874553209754203, pvalue=2.8170932231071047e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.27919097133523013, pvalue=1.2741079508196693e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.25957069323378934, pvalue=2.4987018642196428e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.22927010308402693, pvalue=1.5616637647029195e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2558881678518831, pvalue=4.253729384132114e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.23828857970818967, pvalue=4.8292889261133265e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2935890212875804, pvalue=1.2284032423346224e-11)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2685552797864819, pvalue=6.588513063998333e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2773703572210694, pvalue=1.6965015068915015e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2661711680152903, pvalue=9.430260358220895e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.23747303253556548, pvalue=5.380592208706142e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.20824435473799238, pvalue=2.007011900149323e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2841124648271574, pvalue=5.8140765417541795e-11)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.23497943728182388, pvalue=7.469937576213097e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2256006597753386, pvalue=2.483544451948922e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2665134304005415, pvalue=8.959020551162912e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.271427451849513, pvalue=4.257224364172894e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2760476340006713, pvalue=2.0860484147775665e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.20768520160415796, pvalue=2.1406700359811483e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.24293631299154747, pvalue=2.5886353429284263e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.24816116398134555, pvalue=1.264609358878376e-08)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2143572964030236, pvalue=9.804856440065999e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.18183603311209456, pvalue=3.488563461501077e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.21870439629684993, pvalue=5.815597806668617e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.26110575108486783, pvalue=1.9967981396215566e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.17561255041623597, pvalue=6.468484181399866e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.15153457911691004, pvalue=0.0005809782585512425)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2016568794554587, pvalue=4.24211918391214e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.16797722213726918, pvalue=0.0001341197378418305)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.20369210059000623, pvalue=3.3751191829634335e-06)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.18540019997980078, pvalue=2.42652321760973e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.21785643853901118, pvalue=6.444831488047224e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.22759613103796064, pvalue=1.9316317230376625e-07)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.18557366107514048, pvalue=2.3836085787175903e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.19347204635308152, pvalue=1.039468292267269e-05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2999544746210928, pvalue=4.183811335569883e-12)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2665456063715782, pvalue=8.915915786763641e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2754196086572566, pvalue=2.3002702822404057e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2826367776297655, pvalue=7.368023811893617e-11)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.26373978032699125, pvalue=1.3544745178315421e-09)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.29465667690434394, pvalue=1.0272822830043584e-11)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.32335715611645593, pvalue=6.318285299307479e-14)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3169976492375636, pvalue=2.0484895704112253e-13)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.26944344710093343, pvalue=5.759399134879173e-10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6636434090439899, pvalue=0.036411567166174705)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6833490732084478, pvalue=0.02937668347961381)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5449492609130662, pvalue=0.10331591735410979)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7531234192521683, pvalue=0.011914859098813412)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.18164975363768868, pvalue=0.6154951256738583)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.49959672366232943, pvalue=0.1414857456452181)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.4061811972299616, pvalue=0.2441286818964912)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5890767338705079, pvalue=0.07314571054139454)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7006490497453708, pvalue=0.02401850924875498)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.058025885318565944, pvalue=0.8734948923712951)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5222329678670935, pvalue=0.12150291881711318)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.2901294265928297, pvalue=0.4161202535070595)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3504633733153655, pvalue=0.3207871171095401)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5449492609130662, pvalue=0.10331591735410979)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3719494955438387, pvalue=0.2898886711708534)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5222329678670935, pvalue=0.12150291881711318)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5443367287664187, pvalue=0.10378216581521554)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.8876253645985944, pvalue=0.0006079255649158426)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6227991553292185, pvalue=0.054438696140665634)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.8469257208241591, pvalue=0.001988368670679039)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.4061811972299616, pvalue=0.2441286818964912)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6189034039399006, pvalue=0.05641658384254792)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.812776759390954, pvalue=0.004259472986524341)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5276492843761433, pvalue=0.11699832200012014)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7006490497453708, pvalue=0.02401850924875498)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6691329671589404, pvalue=0.034348320459470706)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5222329678670935, pvalue=0.12150291881711318)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6833490732084478, pvalue=0.02937668347961381)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5222329678670935, pvalue=0.12150291881711318)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5871983181190702, pvalue=0.07429561413550047)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.45485671855824034, pvalue=0.18656494869982135)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5443367287664187, pvalue=0.10378216581521554)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.4061811972299616, pvalue=0.2441286818964912)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7978634243562575, pvalue=0.005677181810793317)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5881992022553729, pvalue=0.07368145588384303)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7305839539388431, pvalue=0.016402333784353674)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5222329678670935, pvalue=0.12150291881711318)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.36537670835006186, pvalue=0.299164843623496)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.3892494720807615, pvalue=0.2662240730692272)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7006490497453708, pvalue=0.02401850924875498)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7006490497453708, pvalue=0.02401850924875498)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.35464951900691605, pvalue=0.3146377079404608)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.7382100842174719, pvalue=0.014771359347092073)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.058025885318565944, pvalue=0.8734948923712951)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5052636690792, pvalue=0.13630677217453738)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6227991553292185, pvalue=0.054438696140665634)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5276492843761433, pvalue=0.11699832200012014)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.4697700535929367, pvalue=0.1707074545917416)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.5276492843761433, pvalue=0.11699832200012014)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.6227991553292185, pvalue=0.054438696140665634)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.06919990614769093, pvalue=0.8493480024065823)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(len(scores[\"rel_error\"])):\n",
    "    display(stats.spearmanr(scores[\"rel_error\"][i], scores[\"alignment_variance\"][i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8dc0c735115da02fdb521651a839b43a7b353576a1e7c935415b1c42e93294ff"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 ('torch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}