term_project_task3.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8vjcWSnMX87X"
      },
      "source": [
        "# Term Project: Question quality assessment\n",
        "### 20182650 Jang Jaehoon"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### load data using Google Drive"
      ],
      "metadata": {
        "id": "HRsF8P_M8BAJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# train data load\n",
        "#from google.colab import drive\n",
        "#drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "zO8_HwJtPrw1"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import pandas as pd\n",
        "#data = pd.read_csv('/content/drive/MyDrive/2023-1_ML/Term_Project/data/train_data/train_task_3_4.csv')\n",
        "#data.head()"
      ],
      "metadata": {
        "id": "HtAyZdPc_5nv"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### load data using Github"
      ],
      "metadata": {
        "id": "85guCjRd8Edw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/leehahoon/question_quality_assessment.git git_dir"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OGQV8P057r9U",
        "outputId": "cd7a25ae-9eb9-41b9-e99f-de8c9f7b33f1"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'git_dir' already exists and is not an empty directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv('git_dir/data/train_data/train_task_3_4.csv')\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "dntKU2ya8Ib6",
        "outputId": "03bd70c0-7865-4aa0-9d99-224fa8d8cbd3"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   QuestionId  UserId  AnswerId  IsCorrect  CorrectAnswer  AnswerValue\n",
              "0         898    2111    280203          1              2            2\n",
              "1         767    3062     55638          1              3            3\n",
              "2         165    1156    386475          1              2            2\n",
              "3         490    1653    997498          1              4            4\n",
              "4         298    3912    578636          1              3            3"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a8ec4496-ca3b-440f-b857-cc2af262ec6c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>QuestionId</th>\n",
              "      <th>UserId</th>\n",
              "      <th>AnswerId</th>\n",
              "      <th>IsCorrect</th>\n",
              "      <th>CorrectAnswer</th>\n",
              "      <th>AnswerValue</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>898</td>\n",
              "      <td>2111</td>\n",
              "      <td>280203</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>767</td>\n",
              "      <td>3062</td>\n",
              "      <td>55638</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>165</td>\n",
              "      <td>1156</td>\n",
              "      <td>386475</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>490</td>\n",
              "      <td>1653</td>\n",
              "      <td>997498</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>298</td>\n",
              "      <td>3912</td>\n",
              "      <td>578636</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a8ec4496-ca3b-440f-b857-cc2af262ec6c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a8ec4496-ca3b-440f-b857-cc2af262ec6c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a8ec4496-ca3b-440f-b857-cc2af262ec6c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Overview"
      ],
      "metadata": {
        "id": "Eqll4JAm-WLE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(data.isnull().sum())\n",
        "# Null value check\n",
        "\n",
        "print(data.describe(include='all'))\n",
        "# data summary"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OoVbRgdTZx61",
        "outputId": "a5115989-a657-4807-fb1f-fdcc5eaafdb1"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "QuestionId       0\n",
            "UserId           0\n",
            "AnswerId         0\n",
            "IsCorrect        0\n",
            "CorrectAnswer    0\n",
            "AnswerValue      0\n",
            "dtype: int64\n",
            "         QuestionId        UserId      AnswerId     IsCorrect  CorrectAnswer  \\\n",
            "count  1.382727e+06  1.382727e+06  1.382727e+06  1.382727e+06   1.382727e+06   \n",
            "mean   4.682276e+02  3.036283e+03  7.544279e+05  5.373317e-01   2.390497e+00   \n",
            "std    2.735797e+02  1.770599e+03  4.356190e+05  4.986046e-01   1.066479e+00   \n",
            "min    0.000000e+00  1.000000e+00  0.000000e+00  0.000000e+00   1.000000e+00   \n",
            "25%    2.330000e+02  1.515000e+03  3.772935e+05  0.000000e+00   1.000000e+00   \n",
            "50%    4.680000e+02  3.009000e+03  7.544530e+05  1.000000e+00   2.000000e+00   \n",
            "75%    7.030000e+02  4.565000e+03  1.131772e+06  1.000000e+00   3.000000e+00   \n",
            "max    9.470000e+02  6.147000e+03  1.508916e+06  1.000000e+00   4.000000e+00   \n",
            "\n",
            "        AnswerValue  \n",
            "count  1.382727e+06  \n",
            "mean   2.437992e+00  \n",
            "std    1.067624e+00  \n",
            "min    1.000000e+00  \n",
            "25%    2.000000e+00  \n",
            "50%    2.000000e+00  \n",
            "75%    3.000000e+00  \n",
            "max    4.000000e+00  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "n_questions = len(data['QuestionId'].unique())\n",
        "print(n_questions)\n",
        "# question number"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cmIc7aHX4y--",
        "outputId": "3bead3db-1dea-40e2-b960-5304df634d91"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "948\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Question Correct Answer Rate"
      ],
      "metadata": {
        "id": "XTgu-PkOL45Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "result_df = pd.DataFrame()\n",
        "# store result"
      ],
      "metadata": {
        "id": "EoNcvfnRDpVj"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result_df['Correct_Rate'] = data.groupby('QuestionId').agg(['mean'])['IsCorrect']\n",
        "# Average of correct answers per QuestionID\n",
        "\n",
        "result_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "LwW9J6qJLzBv",
        "outputId": "c0c13913-943b-4126-ceb0-f657fb4e5fb1"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Correct_Rate\n",
              "QuestionId              \n",
              "0               0.443457\n",
              "1               0.571429\n",
              "2               0.385214\n",
              "3               0.808757\n",
              "4               0.401408\n",
              "...                  ...\n",
              "943             0.566528\n",
              "944             0.142857\n",
              "945             0.422336\n",
              "946             0.459459\n",
              "947             0.617124\n",
              "\n",
              "[948 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0e51dc20-538d-4b76-a90d-5d2b57c501d6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Correct_Rate</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.443457</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.571429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.385214</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.808757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.401408</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>0.566528</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>0.142857</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>945</th>\n",
              "      <td>0.422336</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>946</th>\n",
              "      <td>0.459459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>947</th>\n",
              "      <td>0.617124</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>948 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0e51dc20-538d-4b76-a90d-5d2b57c501d6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0e51dc20-538d-4b76-a90d-5d2b57c501d6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0e51dc20-538d-4b76-a90d-5d2b57c501d6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Question Correct Entropy"
      ],
      "metadata": {
        "id": "c9oVYx0oE46U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import scipy as sp\n",
        "from scipy.stats import multinomial\n",
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import argparse"
      ],
      "metadata": {
        "id": "SgmsYzPQEKxk"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "result_df['IsCorrect_Entropy'] = data.groupby('QuestionId')['IsCorrect'].agg(lambda x:multinomial.entropy(1,x.value_counts(normalize=True)).mean())\n",
        "# Calculates the entropy of the polynomial distribution of the IsCorrect column for each question\n",
        "# A smaller value means that the correct answer for that question is easier to predict.\n",
        "\n",
        "result_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "s3-8EnJdPnoT",
        "outputId": "71db1c1a-4e29-4459-e789-199c3ea96607"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Correct_Rate  IsCorrect_Entropy\n",
              "QuestionId                                 \n",
              "0               0.443457           0.686739\n",
              "1               0.571429           0.682908\n",
              "2               0.385214           0.666559\n",
              "3               0.808757           0.488020\n",
              "4               0.401408           0.673579\n",
              "...                  ...                ...\n",
              "943             0.566528           0.684269\n",
              "944             0.142857           0.410116\n",
              "945             0.422336           0.681035\n",
              "946             0.459459           0.689856\n",
              "947             0.617124           0.665454\n",
              "\n",
              "[948 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a5eba447-1765-45dc-823c-67b627c2f9e1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Correct_Rate</th>\n",
              "      <th>IsCorrect_Entropy</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.443457</td>\n",
              "      <td>0.686739</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.682908</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.385214</td>\n",
              "      <td>0.666559</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.808757</td>\n",
              "      <td>0.488020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.401408</td>\n",
              "      <td>0.673579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>0.566528</td>\n",
              "      <td>0.684269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.410116</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>945</th>\n",
              "      <td>0.422336</td>\n",
              "      <td>0.681035</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>946</th>\n",
              "      <td>0.459459</td>\n",
              "      <td>0.689856</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>947</th>\n",
              "      <td>0.617124</td>\n",
              "      <td>0.665454</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>948 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a5eba447-1765-45dc-823c-67b627c2f9e1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a5eba447-1765-45dc-823c-67b627c2f9e1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a5eba447-1765-45dc-823c-67b627c2f9e1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Answer Value Entropy"
      ],
      "metadata": {
        "id": "U5Mj1kjUD8Tl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "result_df['AnswerValue_Entropy'] = data.groupby('QuestionId')['AnswerValue'].agg(lambda x:multinomial.entropy(1,x.value_counts(normalize=True)).mean())\n",
        "# reference : starter_kit - sample_model_entropy\n",
        "# A smaller value means the answer to the question is easier to predict.\n",
        "\n",
        "result_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "sOaO_PGMEBR0",
        "outputId": "b47a37ff-6487-4734-f798-038a774f43cf"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Correct_Rate  IsCorrect_Entropy  AnswerValue_Entropy\n",
              "QuestionId                                                      \n",
              "0               0.443457           0.686739             1.194456\n",
              "1               0.571429           0.682908             0.876006\n",
              "2               0.385214           0.666559             1.331663\n",
              "3               0.808757           0.488020             0.695423\n",
              "4               0.401408           0.673579             1.324321\n",
              "...                  ...                ...                  ...\n",
              "943             0.566528           0.684269             0.958544\n",
              "944             0.142857           0.410116             1.312404\n",
              "945             0.422336           0.681035             1.260934\n",
              "946             0.459459           0.689856             1.236733\n",
              "947             0.617124           0.665454             1.082614\n",
              "\n",
              "[948 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bfac289d-e7e6-441a-8ac0-dae646e15960\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Correct_Rate</th>\n",
              "      <th>IsCorrect_Entropy</th>\n",
              "      <th>AnswerValue_Entropy</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.443457</td>\n",
              "      <td>0.686739</td>\n",
              "      <td>1.194456</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.682908</td>\n",
              "      <td>0.876006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.385214</td>\n",
              "      <td>0.666559</td>\n",
              "      <td>1.331663</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.808757</td>\n",
              "      <td>0.488020</td>\n",
              "      <td>0.695423</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.401408</td>\n",
              "      <td>0.673579</td>\n",
              "      <td>1.324321</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>0.566528</td>\n",
              "      <td>0.684269</td>\n",
              "      <td>0.958544</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.410116</td>\n",
              "      <td>1.312404</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>945</th>\n",
              "      <td>0.422336</td>\n",
              "      <td>0.681035</td>\n",
              "      <td>1.260934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>946</th>\n",
              "      <td>0.459459</td>\n",
              "      <td>0.689856</td>\n",
              "      <td>1.236733</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>947</th>\n",
              "      <td>0.617124</td>\n",
              "      <td>0.665454</td>\n",
              "      <td>1.082614</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>948 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bfac289d-e7e6-441a-8ac0-dae646e15960')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bfac289d-e7e6-441a-8ac0-dae646e15960 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bfac289d-e7e6-441a-8ac0-dae646e15960');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Subject"
      ],
      "metadata": {
        "id": "Eya7VXuia8hi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "question_metadata = pd.read_csv('git_dir/data/metadata/question_metadata_task_3_4.csv')\n",
        "question_metadata.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "SkvItpO-bjeE",
        "outputId": "7ff778c5-5e43-406a-a0f1-36131b7bb57d"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   QuestionId         SubjectId\n",
              "0         684  [3, 71, 98, 209]\n",
              "1          87  [3, 71, 98, 209]\n",
              "2         679  [3, 71, 98, 209]\n",
              "3         685  [3, 71, 98, 209]\n",
              "4         222  [3, 71, 98, 209]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fc7365e4-9f65-48b3-86b1-fef6e6b6c8ae\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>QuestionId</th>\n",
              "      <th>SubjectId</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>684</td>\n",
              "      <td>[3, 71, 98, 209]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>87</td>\n",
              "      <td>[3, 71, 98, 209]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>679</td>\n",
              "      <td>[3, 71, 98, 209]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>685</td>\n",
              "      <td>[3, 71, 98, 209]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>222</td>\n",
              "      <td>[3, 71, 98, 209]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fc7365e4-9f65-48b3-86b1-fef6e6b6c8ae')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fc7365e4-9f65-48b3-86b1-fef6e6b6c8ae button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fc7365e4-9f65-48b3-86b1-fef6e6b6c8ae');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "subj_metadata = pd.read_csv('git_dir/data/metadata/subject_metadata.csv')\n",
        "subj_metadata.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "-2u45seba7iT",
        "outputId": "7141d988-2343-4fcb-f74f-e6a72da418a1"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   SubjectId                    Name  ParentId  Level\n",
              "0          3                   Maths       NaN      0\n",
              "1         32                  Number       3.0      1\n",
              "2         33                  BIDMAS     144.0      3\n",
              "3         34  Upper and Lower Bounds     141.0      3\n",
              "4         35          Calculator Use      32.0      2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-aa8640fe-0ac9-43be-a3bc-3b96849d5cec\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SubjectId</th>\n",
              "      <th>Name</th>\n",
              "      <th>ParentId</th>\n",
              "      <th>Level</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Maths</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>32</td>\n",
              "      <td>Number</td>\n",
              "      <td>3.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>33</td>\n",
              "      <td>BIDMAS</td>\n",
              "      <td>144.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>34</td>\n",
              "      <td>Upper and Lower Bounds</td>\n",
              "      <td>141.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>35</td>\n",
              "      <td>Calculator Use</td>\n",
              "      <td>32.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-aa8640fe-0ac9-43be-a3bc-3b96849d5cec')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-aa8640fe-0ac9-43be-a3bc-3b96849d5cec button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-aa8640fe-0ac9-43be-a3bc-3b96849d5cec');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import ast\n",
        "question_data = question_metadata.sort_values('QuestionId')\n",
        "question_data['SubjectList'] = question_data['SubjectId'].apply(lambda x: ast.literal_eval(x))\n",
        "question_data['SubjectLen'] = question_data['SubjectList'].apply(len)\n",
        "question_data.set_index('QuestionId', inplace=True)\n",
        "question_data.head()\n",
        "\n",
        "# get SubjectList's length"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "ZvewzJ1uc3f0",
        "outputId": "7022b9c7-1a49-4a7e-a0b3-7be1f677bbc7"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            SubjectId                SubjectList  SubjectLen\n",
              "QuestionId                                                                  \n",
              "0                    [3, 32, 40, 245]           [3, 32, 40, 245]           4\n",
              "1                    [3, 32, 39, 228]           [3, 32, 39, 228]           4\n",
              "2           [3, 32, 33, 42, 144, 212]  [3, 32, 33, 42, 144, 212]           6\n",
              "3                    [3, 32, 42, 211]           [3, 32, 42, 211]           4\n",
              "4                    [3, 71, 88, 174]           [3, 71, 88, 174]           4"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2f648acf-5897-4ba4-b199-0e693d8eefa1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SubjectId</th>\n",
              "      <th>SubjectList</th>\n",
              "      <th>SubjectLen</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[3, 32, 40, 245]</td>\n",
              "      <td>[3, 32, 40, 245]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[3, 32, 39, 228]</td>\n",
              "      <td>[3, 32, 39, 228]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[3, 32, 33, 42, 144, 212]</td>\n",
              "      <td>[3, 32, 33, 42, 144, 212]</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[3, 32, 42, 211]</td>\n",
              "      <td>[3, 32, 42, 211]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[3, 71, 88, 174]</td>\n",
              "      <td>[3, 71, 88, 174]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2f648acf-5897-4ba4-b199-0e693d8eefa1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2f648acf-5897-4ba4-b199-0e693d8eefa1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2f648acf-5897-4ba4-b199-0e693d8eefa1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "result_df['Subject_Score'] = 10-question_data['SubjectLen']\n",
        "result_df\n",
        "\n",
        "# The longer the length, the more diverse the problem area.\n",
        "# so, 10 - (SubjectLen)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "E7vqMr37ENNH",
        "outputId": "deaed9b2-9224-4814-d323-f83b64e3c389"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Correct_Rate  IsCorrect_Entropy  AnswerValue_Entropy  \\\n",
              "QuestionId                                                         \n",
              "0               0.443457           0.686739             1.194456   \n",
              "1               0.571429           0.682908             0.876006   \n",
              "2               0.385214           0.666559             1.331663   \n",
              "3               0.808757           0.488020             0.695423   \n",
              "4               0.401408           0.673579             1.324321   \n",
              "...                  ...                ...                  ...   \n",
              "943             0.566528           0.684269             0.958544   \n",
              "944             0.142857           0.410116             1.312404   \n",
              "945             0.422336           0.681035             1.260934   \n",
              "946             0.459459           0.689856             1.236733   \n",
              "947             0.617124           0.665454             1.082614   \n",
              "\n",
              "            Subject_Score  \n",
              "QuestionId                 \n",
              "0                       6  \n",
              "1                       6  \n",
              "2                       4  \n",
              "3                       6  \n",
              "4                       6  \n",
              "...                   ...  \n",
              "943                     6  \n",
              "944                     6  \n",
              "945                     6  \n",
              "946                     6  \n",
              "947                     6  \n",
              "\n",
              "[948 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b45e7144-1030-482f-8ebe-5a46d1a2152d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Correct_Rate</th>\n",
              "      <th>IsCorrect_Entropy</th>\n",
              "      <th>AnswerValue_Entropy</th>\n",
              "      <th>Subject_Score</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.443457</td>\n",
              "      <td>0.686739</td>\n",
              "      <td>1.194456</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.682908</td>\n",
              "      <td>0.876006</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.385214</td>\n",
              "      <td>0.666559</td>\n",
              "      <td>1.331663</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.808757</td>\n",
              "      <td>0.488020</td>\n",
              "      <td>0.695423</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.401408</td>\n",
              "      <td>0.673579</td>\n",
              "      <td>1.324321</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>0.566528</td>\n",
              "      <td>0.684269</td>\n",
              "      <td>0.958544</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.410116</td>\n",
              "      <td>1.312404</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>945</th>\n",
              "      <td>0.422336</td>\n",
              "      <td>0.681035</td>\n",
              "      <td>1.260934</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>946</th>\n",
              "      <td>0.459459</td>\n",
              "      <td>0.689856</td>\n",
              "      <td>1.236733</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>947</th>\n",
              "      <td>0.617124</td>\n",
              "      <td>0.665454</td>\n",
              "      <td>1.082614</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>948 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b45e7144-1030-482f-8ebe-5a46d1a2152d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b45e7144-1030-482f-8ebe-5a46d1a2152d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b45e7144-1030-482f-8ebe-5a46d1a2152d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rank_df = pd.DataFrame()\n",
        "rank_df['Correct_Rate'] = result_df['Correct_Rate']\n",
        "rank_df['CR_inverse'] = 1-result_df['Correct_Rate']\n",
        "rank_df['CR_inverse+SS'] = rank_df['CR_inverse'] + result_df['Subject_Score']\n",
        "rank_df['IC'] = result_df['IsCorrect_Entropy']\n",
        "rank_df['IC+AV'] = result_df['IsCorrect_Entropy'] + result_df['AnswerValue_Entropy']\n",
        "rank_df['IC+AV+SS'] = result_df['IsCorrect_Entropy'] + result_df['AnswerValue_Entropy'] + result_df['Subject_Score']\n",
        "rank_df['IC+AV+SS+CR_inv'] = result_df['IsCorrect_Entropy'] + result_df['AnswerValue_Entropy'] + result_df['Subject_Score'] + rank_df['CR_inverse']\n",
        "\n",
        "rank_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "bYQmoSTLpaYN",
        "outputId": "3e09bc5a-cfa2-4cfe-e938-5c912a0096b0"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Correct_Rate  CR_inverse  CR_inverse+SS        IC     IC+AV  \\\n",
              "QuestionId                                                                \n",
              "0               0.443457    0.556543       6.556543  0.686739  1.881195   \n",
              "1               0.571429    0.428571       6.428571  0.682908  1.558914   \n",
              "2               0.385214    0.614786       4.614786  0.666559  1.998222   \n",
              "3               0.808757    0.191243       6.191243  0.488020  1.183442   \n",
              "4               0.401408    0.598592       6.598592  0.673579  1.997900   \n",
              "...                  ...         ...            ...       ...       ...   \n",
              "943             0.566528    0.433472       6.433472  0.684269  1.642812   \n",
              "944             0.142857    0.857143       6.857143  0.410116  1.722520   \n",
              "945             0.422336    0.577664       6.577664  0.681035  1.941969   \n",
              "946             0.459459    0.540541       6.540541  0.689856  1.926589   \n",
              "947             0.617124    0.382876       6.382876  0.665454  1.748069   \n",
              "\n",
              "            IC+AV+SS  IC+AV+SS+CR_inv  \n",
              "QuestionId                             \n",
              "0           7.881195         8.437739  \n",
              "1           7.558914         7.987485  \n",
              "2           5.998222         6.613008  \n",
              "3           7.183442         7.374685  \n",
              "4           7.997900         8.596491  \n",
              "...              ...              ...  \n",
              "943         7.642812         8.076284  \n",
              "944         7.722520         8.579663  \n",
              "945         7.941969         8.519633  \n",
              "946         7.926589         8.467130  \n",
              "947         7.748069         8.130944  \n",
              "\n",
              "[948 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a5f92117-3d2c-4a1d-b1dc-2acb8ff4757c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Correct_Rate</th>\n",
              "      <th>CR_inverse</th>\n",
              "      <th>CR_inverse+SS</th>\n",
              "      <th>IC</th>\n",
              "      <th>IC+AV</th>\n",
              "      <th>IC+AV+SS</th>\n",
              "      <th>IC+AV+SS+CR_inv</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.443457</td>\n",
              "      <td>0.556543</td>\n",
              "      <td>6.556543</td>\n",
              "      <td>0.686739</td>\n",
              "      <td>1.881195</td>\n",
              "      <td>7.881195</td>\n",
              "      <td>8.437739</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>6.428571</td>\n",
              "      <td>0.682908</td>\n",
              "      <td>1.558914</td>\n",
              "      <td>7.558914</td>\n",
              "      <td>7.987485</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.385214</td>\n",
              "      <td>0.614786</td>\n",
              "      <td>4.614786</td>\n",
              "      <td>0.666559</td>\n",
              "      <td>1.998222</td>\n",
              "      <td>5.998222</td>\n",
              "      <td>6.613008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.808757</td>\n",
              "      <td>0.191243</td>\n",
              "      <td>6.191243</td>\n",
              "      <td>0.488020</td>\n",
              "      <td>1.183442</td>\n",
              "      <td>7.183442</td>\n",
              "      <td>7.374685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.401408</td>\n",
              "      <td>0.598592</td>\n",
              "      <td>6.598592</td>\n",
              "      <td>0.673579</td>\n",
              "      <td>1.997900</td>\n",
              "      <td>7.997900</td>\n",
              "      <td>8.596491</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>0.566528</td>\n",
              "      <td>0.433472</td>\n",
              "      <td>6.433472</td>\n",
              "      <td>0.684269</td>\n",
              "      <td>1.642812</td>\n",
              "      <td>7.642812</td>\n",
              "      <td>8.076284</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>6.857143</td>\n",
              "      <td>0.410116</td>\n",
              "      <td>1.722520</td>\n",
              "      <td>7.722520</td>\n",
              "      <td>8.579663</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>945</th>\n",
              "      <td>0.422336</td>\n",
              "      <td>0.577664</td>\n",
              "      <td>6.577664</td>\n",
              "      <td>0.681035</td>\n",
              "      <td>1.941969</td>\n",
              "      <td>7.941969</td>\n",
              "      <td>8.519633</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>946</th>\n",
              "      <td>0.459459</td>\n",
              "      <td>0.540541</td>\n",
              "      <td>6.540541</td>\n",
              "      <td>0.689856</td>\n",
              "      <td>1.926589</td>\n",
              "      <td>7.926589</td>\n",
              "      <td>8.467130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>947</th>\n",
              "      <td>0.617124</td>\n",
              "      <td>0.382876</td>\n",
              "      <td>6.382876</td>\n",
              "      <td>0.665454</td>\n",
              "      <td>1.748069</td>\n",
              "      <td>7.748069</td>\n",
              "      <td>8.130944</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>948 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a5f92117-3d2c-4a1d-b1dc-2acb8ff4757c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a5f92117-3d2c-4a1d-b1dc-2acb8ff4757c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a5f92117-3d2c-4a1d-b1dc-2acb8ff4757c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rank_df['CR_ranking'] = rank_df['Correct_Rate'].rank(method='first', ascending=False).astype('int16')\n",
        "rank_df['CR_inverse_ranking'] = rank_df['CR_inverse'].rank(method='first', ascending=False).astype('int16')\n",
        "rank_df['CR_inverse+SS_ranking']  = rank_df['CR_inverse+SS'].rank(method='first', ascending=False).astype('int16')\n",
        "rank_df['IC_ranking'] = rank_df['IC'].rank(method='first', ascending=False).astype('int16')\n",
        "rank_df['IC+AV_ranking'] = rank_df['IC+AV'].rank(method='first', ascending=False).astype('int16')\n",
        "rank_df['IC+AV+SS_ranking'] = rank_df['IC+AV+SS'].rank(method='first', ascending=False).astype('int16')\n",
        "rank_df['IC+AV+SS_+CR_inv_ranking'] = rank_df['IC+AV+SS+CR_inv'].rank(method='first', ascending=False).astype('int16')\n",
        "\n",
        "rank_df\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 519
        },
        "id": "rrLj8_GBvTyA",
        "outputId": "22a66645-59d8-46fc-ae8c-c6ff33cef457"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            Correct_Rate  CR_inverse  CR_inverse+SS        IC     IC+AV  \\\n",
              "QuestionId                                                                \n",
              "0               0.443457    0.556543       6.556543  0.686739  1.881195   \n",
              "1               0.571429    0.428571       6.428571  0.682908  1.558914   \n",
              "2               0.385214    0.614786       4.614786  0.666559  1.998222   \n",
              "3               0.808757    0.191243       6.191243  0.488020  1.183442   \n",
              "4               0.401408    0.598592       6.598592  0.673579  1.997900   \n",
              "...                  ...         ...            ...       ...       ...   \n",
              "943             0.566528    0.433472       6.433472  0.684269  1.642812   \n",
              "944             0.142857    0.857143       6.857143  0.410116  1.722520   \n",
              "945             0.422336    0.577664       6.577664  0.681035  1.941969   \n",
              "946             0.459459    0.540541       6.540541  0.689856  1.926589   \n",
              "947             0.617124    0.382876       6.382876  0.665454  1.748069   \n",
              "\n",
              "            IC+AV+SS  IC+AV+SS+CR_inv  CR_ranking  CR_inverse_ranking  \\\n",
              "QuestionId                                                              \n",
              "0           7.881195         8.437739         594                 355   \n",
              "1           7.558914         7.987485         328                 619   \n",
              "2           5.998222         6.613008         721                 228   \n",
              "3           7.183442         7.374685          27                 922   \n",
              "4           7.997900         8.596491         676                 273   \n",
              "...              ...              ...         ...                 ...   \n",
              "943         7.642812         8.076284         336                 613   \n",
              "944         7.722520         8.579663         948                   1   \n",
              "945         7.941969         8.519633         639                 310   \n",
              "946         7.926589         8.467130         554                 395   \n",
              "947         7.748069         8.130944         239                 710   \n",
              "\n",
              "            CR_inverse+SS_ranking  IC_ranking  IC+AV_ranking  \\\n",
              "QuestionId                                                     \n",
              "0                             345         247            455   \n",
              "1                             608         293            802   \n",
              "2                             948         481             40   \n",
              "3                             908         915            915   \n",
              "4                             264         405             43   \n",
              "...                           ...         ...            ...   \n",
              "943                           602         274            750   \n",
              "944                             1         938            684   \n",
              "945                           300         324            284   \n",
              "946                           385         178            328   \n",
              "947                           697         489            649   \n",
              "\n",
              "            IC+AV+SS_ranking  IC+AV+SS_+CR_inv_ranking  \n",
              "QuestionId                                              \n",
              "0                        447                       425  \n",
              "1                        788                       747  \n",
              "2                        948                       945  \n",
              "3                        901                       912  \n",
              "4                         41                       180  \n",
              "...                      ...                       ...  \n",
              "943                      736                       706  \n",
              "944                      671                       210  \n",
              "945                      279                       309  \n",
              "946                      322                       382  \n",
              "947                      636                       675  \n",
              "\n",
              "[948 rows x 14 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-566b3732-93dc-4d91-b2f4-bc02125ea57c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Correct_Rate</th>\n",
              "      <th>CR_inverse</th>\n",
              "      <th>CR_inverse+SS</th>\n",
              "      <th>IC</th>\n",
              "      <th>IC+AV</th>\n",
              "      <th>IC+AV+SS</th>\n",
              "      <th>IC+AV+SS+CR_inv</th>\n",
              "      <th>CR_ranking</th>\n",
              "      <th>CR_inverse_ranking</th>\n",
              "      <th>CR_inverse+SS_ranking</th>\n",
              "      <th>IC_ranking</th>\n",
              "      <th>IC+AV_ranking</th>\n",
              "      <th>IC+AV+SS_ranking</th>\n",
              "      <th>IC+AV+SS_+CR_inv_ranking</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>QuestionId</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.443457</td>\n",
              "      <td>0.556543</td>\n",
              "      <td>6.556543</td>\n",
              "      <td>0.686739</td>\n",
              "      <td>1.881195</td>\n",
              "      <td>7.881195</td>\n",
              "      <td>8.437739</td>\n",
              "      <td>594</td>\n",
              "      <td>355</td>\n",
              "      <td>345</td>\n",
              "      <td>247</td>\n",
              "      <td>455</td>\n",
              "      <td>447</td>\n",
              "      <td>425</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>6.428571</td>\n",
              "      <td>0.682908</td>\n",
              "      <td>1.558914</td>\n",
              "      <td>7.558914</td>\n",
              "      <td>7.987485</td>\n",
              "      <td>328</td>\n",
              "      <td>619</td>\n",
              "      <td>608</td>\n",
              "      <td>293</td>\n",
              "      <td>802</td>\n",
              "      <td>788</td>\n",
              "      <td>747</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.385214</td>\n",
              "      <td>0.614786</td>\n",
              "      <td>4.614786</td>\n",
              "      <td>0.666559</td>\n",
              "      <td>1.998222</td>\n",
              "      <td>5.998222</td>\n",
              "      <td>6.613008</td>\n",
              "      <td>721</td>\n",
              "      <td>228</td>\n",
              "      <td>948</td>\n",
              "      <td>481</td>\n",
              "      <td>40</td>\n",
              "      <td>948</td>\n",
              "      <td>945</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.808757</td>\n",
              "      <td>0.191243</td>\n",
              "      <td>6.191243</td>\n",
              "      <td>0.488020</td>\n",
              "      <td>1.183442</td>\n",
              "      <td>7.183442</td>\n",
              "      <td>7.374685</td>\n",
              "      <td>27</td>\n",
              "      <td>922</td>\n",
              "      <td>908</td>\n",
              "      <td>915</td>\n",
              "      <td>915</td>\n",
              "      <td>901</td>\n",
              "      <td>912</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.401408</td>\n",
              "      <td>0.598592</td>\n",
              "      <td>6.598592</td>\n",
              "      <td>0.673579</td>\n",
              "      <td>1.997900</td>\n",
              "      <td>7.997900</td>\n",
              "      <td>8.596491</td>\n",
              "      <td>676</td>\n",
              "      <td>273</td>\n",
              "      <td>264</td>\n",
              "      <td>405</td>\n",
              "      <td>43</td>\n",
              "      <td>41</td>\n",
              "      <td>180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>0.566528</td>\n",
              "      <td>0.433472</td>\n",
              "      <td>6.433472</td>\n",
              "      <td>0.684269</td>\n",
              "      <td>1.642812</td>\n",
              "      <td>7.642812</td>\n",
              "      <td>8.076284</td>\n",
              "      <td>336</td>\n",
              "      <td>613</td>\n",
              "      <td>602</td>\n",
              "      <td>274</td>\n",
              "      <td>750</td>\n",
              "      <td>736</td>\n",
              "      <td>706</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>0.142857</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>6.857143</td>\n",
              "      <td>0.410116</td>\n",
              "      <td>1.722520</td>\n",
              "      <td>7.722520</td>\n",
              "      <td>8.579663</td>\n",
              "      <td>948</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>938</td>\n",
              "      <td>684</td>\n",
              "      <td>671</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>945</th>\n",
              "      <td>0.422336</td>\n",
              "      <td>0.577664</td>\n",
              "      <td>6.577664</td>\n",
              "      <td>0.681035</td>\n",
              "      <td>1.941969</td>\n",
              "      <td>7.941969</td>\n",
              "      <td>8.519633</td>\n",
              "      <td>639</td>\n",
              "      <td>310</td>\n",
              "      <td>300</td>\n",
              "      <td>324</td>\n",
              "      <td>284</td>\n",
              "      <td>279</td>\n",
              "      <td>309</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>946</th>\n",
              "      <td>0.459459</td>\n",
              "      <td>0.540541</td>\n",
              "      <td>6.540541</td>\n",
              "      <td>0.689856</td>\n",
              "      <td>1.926589</td>\n",
              "      <td>7.926589</td>\n",
              "      <td>8.467130</td>\n",
              "      <td>554</td>\n",
              "      <td>395</td>\n",
              "      <td>385</td>\n",
              "      <td>178</td>\n",
              "      <td>328</td>\n",
              "      <td>322</td>\n",
              "      <td>382</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>947</th>\n",
              "      <td>0.617124</td>\n",
              "      <td>0.382876</td>\n",
              "      <td>6.382876</td>\n",
              "      <td>0.665454</td>\n",
              "      <td>1.748069</td>\n",
              "      <td>7.748069</td>\n",
              "      <td>8.130944</td>\n",
              "      <td>239</td>\n",
              "      <td>710</td>\n",
              "      <td>697</td>\n",
              "      <td>489</td>\n",
              "      <td>649</td>\n",
              "      <td>636</td>\n",
              "      <td>675</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>948 rows × 14 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-566b3732-93dc-4d91-b2f4-bc02125ea57c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-566b3732-93dc-4d91-b2f4-bc02125ea57c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-566b3732-93dc-4d91-b2f4-bc02125ea57c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#result_df['ranking'].to_csv('rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['CR_ranking'].to_csv('CR_rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['CR_inverse_ranking'].to_csv('CR_inverse_rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['CR_inverse+SS_ranking'].to_csv('CR_inverse+SS_rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['IC_ranking'].to_csv('IC_rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['IC+AV_ranking'].to_csv('IC_AV_rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['IC+AV+SS_ranking'].to_csv('IC_AV_SS_rank.csv', header=True, index_label='QuestionId')\n",
        "rank_df['IC+AV+SS_+CR_inv_ranking'].to_csv('IC_AV_SS_CR_inv_rank.csv', header=True, index_label='QuestionId')\n"
      ],
      "metadata": {
        "id": "2E5WDhyW6vnj"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluate"
      ],
      "metadata": {
        "id": "Zeb67epWnMsd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load evaluators' score on \"validation\" set\n",
        "eval_validation = pd.read_csv('git_dir/data/test_data/quality_response_remapped_public.csv')\n",
        "print(len(eval_validation))\n",
        "eval_validation"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 852
        },
        "id": "zQRSEHNwnQSM",
        "outputId": "3d6f7ba4-dead-452b-dec9-fe0fe44caa43"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "25\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Unnamed: 0.1  Unnamed: 0  left  right  T1_ALR  T2_CL  T3_GF  T4_MQ  T5_NS\n",
              "0              0           0   909    318       2      2      2    1.0      2\n",
              "1             16          16   285    609       2      2      2    2.0      2\n",
              "2             48          48   132    271       1      2      2    2.0      1\n",
              "3             38          38   704    127       2      2      1    2.0      2\n",
              "4             33          33    16    586       1      1      2    2.0      2\n",
              "5             17          17    56    865       2      1      2    2.0      2\n",
              "6             27          27   233    478       1      1      1    1.0      2\n",
              "7             41          41   877    620       2      2      2    2.0      2\n",
              "8              5           5   246    733       1      1      1    2.0      1\n",
              "9             35          35   668    741       1      1      1    1.0      1\n",
              "10            11          11   691      4       2      2      2    1.0      1\n",
              "11            47          47   403    830       2      2      2    2.0      1\n",
              "12            10          10   404     27       2      2      1    2.0      1\n",
              "13            49          49   202    761       1      2      2    2.0      2\n",
              "14             2           2   555    815       1      1      1    1.0      1\n",
              "15            18          18   527    876       2      2      2    1.0      2\n",
              "16            20          20   225    558       2      2      2    2.0      2\n",
              "17            12          12   316    121       1      2      2    2.0      2\n",
              "18            46          46   363    780       2      1      2    2.0      2\n",
              "19            29          29   118    116       1      1      1    1.0      1\n",
              "20             8           8   676    153       1      1      1    2.0      1\n",
              "21            24          24   527    129       2      2      2    2.0      2\n",
              "22            36          36   327    721       2      1      2    2.0      1\n",
              "23            14          14   158    139       2      1      1    1.0      2\n",
              "24            22          22   547    578       1      1      1    1.0      1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1785f88f-dd45-4f38-aab0-0756c425918d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0.1</th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>left</th>\n",
              "      <th>right</th>\n",
              "      <th>T1_ALR</th>\n",
              "      <th>T2_CL</th>\n",
              "      <th>T3_GF</th>\n",
              "      <th>T4_MQ</th>\n",
              "      <th>T5_NS</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>909</td>\n",
              "      <td>318</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>16</td>\n",
              "      <td>16</td>\n",
              "      <td>285</td>\n",
              "      <td>609</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>48</td>\n",
              "      <td>48</td>\n",
              "      <td>132</td>\n",
              "      <td>271</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>38</td>\n",
              "      <td>38</td>\n",
              "      <td>704</td>\n",
              "      <td>127</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>33</td>\n",
              "      <td>33</td>\n",
              "      <td>16</td>\n",
              "      <td>586</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>17</td>\n",
              "      <td>17</td>\n",
              "      <td>56</td>\n",
              "      <td>865</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>27</td>\n",
              "      <td>27</td>\n",
              "      <td>233</td>\n",
              "      <td>478</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>41</td>\n",
              "      <td>41</td>\n",
              "      <td>877</td>\n",
              "      <td>620</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>246</td>\n",
              "      <td>733</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>35</td>\n",
              "      <td>35</td>\n",
              "      <td>668</td>\n",
              "      <td>741</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>11</td>\n",
              "      <td>11</td>\n",
              "      <td>691</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>47</td>\n",
              "      <td>47</td>\n",
              "      <td>403</td>\n",
              "      <td>830</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "      <td>404</td>\n",
              "      <td>27</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>202</td>\n",
              "      <td>761</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>555</td>\n",
              "      <td>815</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>527</td>\n",
              "      <td>876</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>20</td>\n",
              "      <td>20</td>\n",
              "      <td>225</td>\n",
              "      <td>558</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>12</td>\n",
              "      <td>12</td>\n",
              "      <td>316</td>\n",
              "      <td>121</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>46</td>\n",
              "      <td>46</td>\n",
              "      <td>363</td>\n",
              "      <td>780</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>29</td>\n",
              "      <td>29</td>\n",
              "      <td>118</td>\n",
              "      <td>116</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>8</td>\n",
              "      <td>8</td>\n",
              "      <td>676</td>\n",
              "      <td>153</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>527</td>\n",
              "      <td>129</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>36</td>\n",
              "      <td>36</td>\n",
              "      <td>327</td>\n",
              "      <td>721</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>14</td>\n",
              "      <td>14</td>\n",
              "      <td>158</td>\n",
              "      <td>139</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>22</td>\n",
              "      <td>22</td>\n",
              "      <td>547</td>\n",
              "      <td>578</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1785f88f-dd45-4f38-aab0-0756c425918d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1785f88f-dd45-4f38-aab0-0756c425918d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1785f88f-dd45-4f38-aab0-0756c425918d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# take the average of 5 evaulators` scores\n",
        "eval_validation['score'] = eval_validation.filter(regex='^T', axis = 1).mean(axis=1)\n",
        "eval_validation['score'].hist()\n",
        "print(eval_validation)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "aM3d7k9enesL",
        "outputId": "5332ac8d-1344-4b5b-e2f2-30762c50500d"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    Unnamed: 0.1  Unnamed: 0  left  right  T1_ALR  T2_CL  T3_GF  T4_MQ  T5_NS  \\\n",
            "0              0           0   909    318       2      2      2    1.0      2   \n",
            "1             16          16   285    609       2      2      2    2.0      2   \n",
            "2             48          48   132    271       1      2      2    2.0      1   \n",
            "3             38          38   704    127       2      2      1    2.0      2   \n",
            "4             33          33    16    586       1      1      2    2.0      2   \n",
            "5             17          17    56    865       2      1      2    2.0      2   \n",
            "6             27          27   233    478       1      1      1    1.0      2   \n",
            "7             41          41   877    620       2      2      2    2.0      2   \n",
            "8              5           5   246    733       1      1      1    2.0      1   \n",
            "9             35          35   668    741       1      1      1    1.0      1   \n",
            "10            11          11   691      4       2      2      2    1.0      1   \n",
            "11            47          47   403    830       2      2      2    2.0      1   \n",
            "12            10          10   404     27       2      2      1    2.0      1   \n",
            "13            49          49   202    761       1      2      2    2.0      2   \n",
            "14             2           2   555    815       1      1      1    1.0      1   \n",
            "15            18          18   527    876       2      2      2    1.0      2   \n",
            "16            20          20   225    558       2      2      2    2.0      2   \n",
            "17            12          12   316    121       1      2      2    2.0      2   \n",
            "18            46          46   363    780       2      1      2    2.0      2   \n",
            "19            29          29   118    116       1      1      1    1.0      1   \n",
            "20             8           8   676    153       1      1      1    2.0      1   \n",
            "21            24          24   527    129       2      2      2    2.0      2   \n",
            "22            36          36   327    721       2      1      2    2.0      1   \n",
            "23            14          14   158    139       2      1      1    1.0      2   \n",
            "24            22          22   547    578       1      1      1    1.0      1   \n",
            "\n",
            "    score  \n",
            "0     1.8  \n",
            "1     2.0  \n",
            "2     1.6  \n",
            "3     1.8  \n",
            "4     1.6  \n",
            "5     1.8  \n",
            "6     1.2  \n",
            "7     2.0  \n",
            "8     1.2  \n",
            "9     1.0  \n",
            "10    1.6  \n",
            "11    1.8  \n",
            "12    1.6  \n",
            "13    1.8  \n",
            "14    1.0  \n",
            "15    1.8  \n",
            "16    2.0  \n",
            "17    1.8  \n",
            "18    1.8  \n",
            "19    1.0  \n",
            "20    1.2  \n",
            "21    2.0  \n",
            "22    1.6  \n",
            "23    1.4  \n",
            "24    1.0  \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAfoUlEQVR4nO3dX3BU9f3/8deGLJvEZlEMCIEgqEUlCowiTEpbRAkZjan0QqmxQwaU0RqqNCNW2iLJIMU/LYNWB7W1aKcuCFbUqqhRCxSRkVDoiO2g1H+IIg3WLCG6nmbP76KT/JpvEsiJ73PWkz4fM3uxx8/5nDfv/eTsy7ObnIjruq4AAAAMZGW6AAAA0HcQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGAmO+gDptNpffjhh8rPz1ckEgn68AAAoBdc19Xhw4dVWFiorKzur0sEHiw+/PBDFRUVBX1YAABgYN++fRo+fHi3/z3wYJGfny/pP4XF43GzeR3H0QsvvKDp06crGo2azYuO6HNw6HUw6HMw6HMw/OxzMplUUVFR+/t4dwIPFm0ff8TjcfNgkZeXp3g8zqL1EX0ODr0OBn0OBn0ORhB9PtbXGPjyJgAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZjwFi9bWVi1atEijRo1Sbm6uTj31VC1ZskSu6/pVHwAACBFP9wq5/fbbtXLlSj388MMqLi5WQ0ODZs+erQEDBuj666/3q0YAABASnoLF1q1bdemll6q8vFySNHLkSK1evVqvvfaaL8UBAIBw8RQsvvGNb+iBBx7Qm2++qdGjR+uvf/2rtmzZouXLl3e7TyqVUiqVan+eTCYl/ecObI7j9LLsztrmspwTndHn4NDrYNDnYNDnYPjZ557OGXE9fEEinU7rJz/5ie644w7169dPra2tWrp0qRYuXNjtPrW1taqrq+u0PZFIKC8vr6eHBgAAGdTS0qLKyko1NTUpHo93O85TsFizZo0WLFigO++8U8XFxdq1a5fmz5+v5cuXq6qqqst9urpiUVRUpMbGxqMW5pXjOKqvr1dpaalv96AHfQ4SvQ4GfQ5GW58XNWQplY5kupwe211blukSPPFzPSeTSRUUFBwzWHj6KGTBggW6+eab9b3vfU+SdPbZZ+u9997TsmXLug0WsVhMsVis0/ZoNOrLD7Ff86Ij+hwceh0M+hyMVDqiVGt4gkVY14Qf67mn83n6ddOWlhZlZXXcpV+/fkqn016mAQAAfZSnKxYVFRVaunSpRowYoeLiYu3cuVPLly/XnDlz/KoPAACEiKdg8atf/UqLFi3Sddddp4MHD6qwsFDXXHONbrnlFr/qAwAAIeIpWOTn52vFihVasWKFT+UAAIAw414hAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADAjKdgMXLkSEUikU6P6upqv+oDAAAhku1l8Pbt29Xa2tr+fPfu3SotLdVll11mXhgAAAgfT8Fi0KBBHZ7fdtttOvXUUzVlyhTTogAAQDh5Chb/7YsvvtDvf/971dTUKBKJdDsulUoplUq1P08mk5Ikx3HkOE5vD99J21yWc6Iz+hwceh0M+hyMtv7GstwMV+JN2NaFn+u5p3NGXNft1au8du1aVVZW6v3331dhYWG342pra1VXV9dpeyKRUF5eXm8ODQAAAtbS0qLKyko1NTUpHo93O67XwaKsrEz9+/fXH//4x6OO6+qKRVFRkRobG49amFeO46i+vl6lpaWKRqNm86Ij+hwceh0M+hyMtj4vashSKt39Ve6vmt21ZZkuwRM/13MymVRBQcExg0WvPgp577339OKLL+rxxx8/5thYLKZYLNZpezQa9eWH2K950RF9Dg69DgZ9DkYqHVGqNTzBIqxrwo/13NP5evV3LFatWqXBgwervLy8N7sDAIA+ynOwSKfTWrVqlaqqqpSd3evvfgIAgD7Ic7B48cUX9f7772vOnDl+1AMAAELM8yWH6dOnq5ff9wQAAH0c9woBAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmPAeL/fv36/vf/75OPPFE5ebm6uyzz1ZDQ4MftQEAgJDJ9jL4X//6lyZPnqypU6dqw4YNGjRokN566y2dcMIJftUHAABCxFOwuP3221VUVKRVq1a1bxs1apR5UQAAIJw8BYunnnpKZWVluuyyy7Rp0yYNGzZM1113nebOndvtPqlUSqlUqv15MpmUJDmOI8dxell2Z21zWc6JzuhzcOh1MOhzMNr6G8tyM1yJN2FbF36u557OGXFdt8evck5OjiSppqZGl112mbZv364bbrhB9913n6qqqrrcp7a2VnV1dZ22JxIJ5eXl9fTQAAAgg1paWlRZWammpibF4/Fux3kKFv3799eECRO0devW9m3XX3+9tm/frldffbXLfbq6YlFUVKTGxsajFuaV4ziqr69XaWmpotGo2bzoiD4Hh14Hgz4Ho63PixqylEpHMl1Oj+2uLct0CZ74uZ6TyaQKCgqOGSw8fRQydOhQjRkzpsO2M888U3/4wx+63ScWiykWi3XaHo1Gffkh9mtedESfg0Ovg0Gfg5FKR5RqDU+wCOua8GM993Q+T79uOnnyZO3Zs6fDtjfffFMnn3yyl2kAAEAf5SlY/OhHP9K2bdv085//XHv37lUikdADDzyg6upqv+oDAAAh4ilYnHfeeVq/fr1Wr16ts846S0uWLNGKFSt05ZVX+lUfAAAIEU/fsZCkSy65RJdccokftQAAgJDjXiEAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMCMp2BRW1urSCTS4XHGGWf4VRsAAAiZbK87FBcX68UXX/z/E2R7ngIAAPRRnlNBdna2hgwZ4kctAAAg5DwHi7feekuFhYXKyclRSUmJli1bphEjRnQ7PpVKKZVKtT9PJpOSJMdx5DhOL0ruWttclnOiM/ocHHodDPocjLb+xrLcDFfiTdjWhZ/ruadzRlzX7fGrvGHDBjU3N+v000/XRx99pLq6Ou3fv1+7d+9Wfn5+l/vU1taqrq6u0/ZEIqG8vLyeHhoAAGRQS0uLKisr1dTUpHg83u04T8Hi//r000918skna/ny5brqqqu6HNPVFYuioiI1NjYetTCvHMdRfX29SktLFY1GzeZFR/Q5OPQ6GG19XtSQpVQ6kulyemx3bVmmS/CEPgfDz/NGMplUQUHBMYPFl/rm5fHHH6/Ro0dr79693Y6JxWKKxWKdtkejUV9Oln7Ni47oc3DodTBS6YhSreF5wwvrmqDPwfDjvNHT+b7U37Fobm7WP/7xDw0dOvTLTAMAAPoIT8Hixhtv1KZNm/Tuu+9q69at+u53v6t+/frpiiuu8Ks+AAAQIp4+Cvnggw90xRVX6NChQxo0aJC++c1vatu2bRo0aJBf9QEAgBDxFCzWrFnjVx0AAKAP4F4hAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADAzJcKFrfddpsikYjmz59vVA4AAAizXgeL7du36/7779fYsWMt6wEAACHWq2DR3NysK6+8Ur/+9a91wgknWNcEAABCKrs3O1VXV6u8vFzTpk3TrbfeetSxqVRKqVSq/XkymZQkOY4jx3F6c/gutc1lOSc6o8/BodfBaOtvLMvNcCXehG1d0Odg+Hne6OmcEdd1Pb3Ka9as0dKlS7V9+3bl5OTo/PPP1/jx47VixYoux9fW1qqurq7T9kQioby8PC+HBgAAGdLS0qLKyko1NTUpHo93O85TsNi3b58mTJig+vr69u9WHCtYdHXFoqioSI2NjUctzCvHcVRfX69FDVlKpSNm8/ptd21ZpkvwpK3PpaWlikajmS6nT6PXweDcEYyw9jlsYlmulkxI+3LeSCaTKigoOGaw8PRRyI4dO3Tw4EGdc8457dtaW1u1efNm3XPPPUqlUurXr1+HfWKxmGKxWKe5otGoLyfLVDqiVGt4Fm1Y3zD8ev3QGb0OBueOYIStz2Hlx3mjp/N5ChYXXnihXn/99Q7bZs+erTPOOEM//vGPO4UKAADwv8VTsMjPz9dZZ53VYdtxxx2nE088sdN2AADwv4e/vAkAAMz06tdN/9vGjRsNygAAAH0BVywAAIAZggUAADBDsAAAAGYIFgAAwAzBAgAAmCFYAAAAMwQLAABghmABAADMECwAAIAZggUAADBDsAAAAGYIFgAAwAzBAgAAmCFYAAAAMwQLAABghmABAADMECwAAIAZggUAADBDsAAAAGYIFgAAwAzBAgAAmCFYAAAAMwQLAABghmABAADMECwAAIAZggUAADBDsAAAAGYIFgAAwAzBAgAAmCFYAAAAMwQLAABghmABAADMECwAAIAZT8Fi5cqVGjt2rOLxuOLxuEpKSrRhwwa/agMAACHjKVgMHz5ct912m3bs2KGGhgZdcMEFuvTSS/XGG2/4VR8AAAiRbC+DKyoqOjxfunSpVq5cqW3btqm4uNi0MAAAED6egsV/a21t1bp163TkyBGVlJR0Oy6VSimVSrU/TyaTkiTHceQ4Tm8P30nbXLEs12zOIFj2IAht9Yat7jCi18Hg3BGMsPY5bNr668f66OmcEdd1Pb3Kr7/+ukpKSvT555/ra1/7mhKJhC6++OJux9fW1qqurq7T9kQioby8PC+HBgAAGdLS0qLKyko1NTUpHo93O85zsPjiiy/0/vvvq6mpSY899ph+85vfaNOmTRozZkyX47u6YlFUVKTGxsajFuaV4ziqr6/XooYspdIRs3n9tru2LNMleEKfg9PW69LSUkWj0UyX02expoMR1j6HTSzL1ZIJaV/OG8lkUgUFBccMFp4/Cunfv79OO+00SdK5556r7du366677tL999/f5fhYLKZYLNZpezQa9eVkmUpHlGoNz6IN6xsGfQ6OXz8r6Ig1HYyw9Tms/Dhv9HS+L/13LNLpdIcrEgAA4H+XpysWCxcu1EUXXaQRI0bo8OHDSiQS2rhxo55//nm/6gMAACHiKVgcPHhQs2bN0kcffaQBAwZo7Nixev7551VaWupXfQAAIEQ8BYsHH3zQrzoAAEAfwL1CAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGU/BYtmyZTrvvPOUn5+vwYMHa8aMGdqzZ49ftQEAgJDxFCw2bdqk6upqbdu2TfX19XIcR9OnT9eRI0f8qg8AAIRItpfBzz33XIfnDz30kAYPHqwdO3bo29/+tmlhAAAgfDwFi/+rqalJkjRw4MBux6RSKaVSqfbnyWRSkuQ4jhzH+TKH76BtrliWazZnECx7EAT6HJy2msNYe5iwpoMR1j6HTVt//VgfPZ0z4rpur17ldDqt73znO/r000+1ZcuWbsfV1taqrq6u0/ZEIqG8vLzeHBoAAASspaVFlZWVampqUjwe73Zcr4PFD37wA23YsEFbtmzR8OHDux3X1RWLoqIiNTY2HrUwrxzHUX19vRY1ZCmVjpjN67fdtWWZLsET+hyctl6XlpYqGo1mupw+izUdjLD2OWxiWa6WTEj7ct5IJpMqKCg4ZrDo1Uch8+bN09NPP63NmzcfNVRIUiwWUywW67Q9Go36crJMpSNKtYZn0Yb1DYM+B8evnxV0xJoORtj6HFZ+nDd6Op+nYOG6rn74wx9q/fr12rhxo0aNGtWr4gAAQN/kKVhUV1crkUjoySefVH5+vg4cOCBJGjBggHJzc30pEAAAhIenv2OxcuVKNTU16fzzz9fQoUPbH48++qhf9QEAgBDx/FEIAABAd7hXCAAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMOM5WGzevFkVFRUqLCxUJBLRE0884UNZAAAgjDwHiyNHjmjcuHG69957/agHAACEWLbXHS666CJddNFFftQCAABCznOw8CqVSimVSrU/TyaTkiTHceQ4jtlx2uaKZblmcwbBsgdBoM/Baas5jLWHCWs6GGHtc9i09deP9dHTOSOu6/b6VY5EIlq/fr1mzJjR7Zja2lrV1dV12p5IJJSXl9fbQwMAgAC1tLSosrJSTU1Nisfj3Y7zPVh0dcWiqKhIjY2NRy3MK8dxVF9fr0UNWUqlI2bz+m13bVmmS/CEPgeHXgeDPgcjrH0Om1iWqyUT0iotLVU0GjWdO5lMqqCg4JjBwvePQmKxmGKxWKft0WjU/B8tSal0RKnW8CxaP3oQBPocHHodDPocjLD1Oaz8eI/t6Xz8HQsAAGDG8xWL5uZm7d27t/35O++8o127dmngwIEaMWKEaXEAACBcPAeLhoYGTZ06tf15TU2NJKmqqkoPPfSQWWEAACB8PAeL888/X1/i+54AAKAP4zsWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADADMECAACYIVgAAAAzBAsAAGCGYAEAAMwQLAAAgBmCBQAAMEOwAAAAZggWAADATK+Cxb333quRI0cqJydHkyZN0muvvWZdFwAACCHPweLRRx9VTU2NFi9erL/85S8aN26cysrKdPDgQT/qAwAAIeI5WCxfvlxz587V7NmzNWbMGN13333Ky8vTb3/7Wz/qAwAAIZLtZfAXX3yhHTt2aOHChe3bsrKyNG3aNL366qtd7pNKpZRKpdqfNzU1SZI++eQTOY7Tm5q75DiOWlpalO1kqTUdMZvXb4cOHcp0CZ7Q5+DQ62DQ52CEtc9hk5121dKS1qFDhxSNRk3nPnz4sCTJdd2jD3Q92L9/vyvJ3bp1a4ftCxYscCdOnNjlPosXL3Yl8eDBgwcPHjz6wGPfvn1HzQqerlj0xsKFC1VTU9P+PJ1O65NPPtGJJ56oSMQutSaTSRUVFWnfvn2Kx+Nm86Ij+hwceh0M+hwM+hwMP/vsuq4OHz6swsLCo47zFCwKCgrUr18/ffzxxx22f/zxxxoyZEiX+8RiMcVisQ7bjj/+eC+H9SQej7NoA0Cfg0Ovg0Gfg0Gfg+FXnwcMGHDMMZ6+vNm/f3+de+65eumll9q3pdNpvfTSSyopKfFeIQAA6FM8fxRSU1OjqqoqTZgwQRMnTtSKFSt05MgRzZ4924/6AABAiHgOFjNnztQ///lP3XLLLTpw4IDGjx+v5557TieddJIf9fVYLBbT4sWLO33sAlv0OTj0Ohj0ORj0ORhfhT5H3GP+3ggAAEDPcK8QAABghmABAADMECwAAIAZggUAADATmmCxefNmVVRUqLCwUJFIRE888cQx99m4caPOOeccxWIxnXbaaXrooYd8rzPsvPb58ccfV2lpqQYNGqR4PK6SkhI9//zzwRQbYr1Zz21eeeUVZWdna/z48b7V11f0ps+pVEo//elPdfLJJysWi2nkyJHcZPEYetPnRx55ROPGjVNeXp6GDh2qOXPmhO7+J0FbtmyZzjvvPOXn52vw4MGaMWOG9uzZc8z91q1bpzPOOEM5OTk6++yz9eyzz/paZ2iCxZEjRzRu3Djde++9PRr/zjvvqLy8XFOnTtWuXbs0f/58XX311bzpHYPXPm/evFmlpaV69tlntWPHDk2dOlUVFRXauXOnz5WGm9c+t/n00081a9YsXXjhhT5V1rf0ps+XX365XnrpJT344IPas2ePVq9erdNPP93HKsPPa59feeUVzZo1S1dddZXeeOMNrVu3Tq+99prmzp3rc6XhtmnTJlVXV2vbtm2qr6+X4ziaPn26jhw50u0+W7du1RVXXKGrrrpKO3fu1IwZMzRjxgzt3r3bv0K93ITsq0KSu379+qOOuemmm9zi4uIO22bOnOmWlZX5WFnf0pM+d2XMmDFuXV2dfUF9lJc+z5w50/3Zz37mLl682B03bpyvdfU1Penzhg0b3AEDBriHDh0Kpqg+qCd9vvPOO91TTjmlw7a7777bHTZsmI+V9T0HDx50JbmbNm3qdszll1/ulpeXd9g2adIk95prrvGtrtBcsfDq1Vdf1bRp0zpsKysr6/b27rCRTqd1+PBhDRw4MNOl9DmrVq3S22+/rcWLF2e6lD7rqaee0oQJE3THHXdo2LBhGj16tG688UZ99tlnmS6tTykpKdG+ffv07LPPynVdffzxx3rsscd08cUXZ7q0UGlqapKko55vM/Fe6PvdTTPlwIEDnf4a6EknnaRkMqnPPvtMubm5Gaqsb/vFL36h5uZmXX755ZkupU956623dPPNN+vPf/6zsrP77I9txr399tvasmWLcnJytH79ejU2Nuq6667ToUOHtGrVqkyX12dMnjxZjzzyiGbOnKnPP/9c//73v1VRUeH5o8H/Zel0WvPnz9fkyZN11llndTuuu/fCAwcO+FZbn71igeAlEgnV1dVp7dq1Gjx4cKbL6TNaW1tVWVmpuro6jR49OtPl9GnpdFqRSESPPPKIJk6cqIsvvljLly/Xww8/zFULQ3/72990ww036JZbbtGOHTv03HPP6d1339W1116b6dJCo7q6Wrt379aaNWsyXUonffZ/fYYMGdLl7d3j8ThXK3ywZs0aXX311Vq3bl2ny274cg4fPqyGhgbt3LlT8+bNk/SfN0DXdZWdna0XXnhBF1xwQYar7BuGDh2qYcOGdbg19JlnninXdfXBBx/o61//egar6zuWLVumyZMna8GCBZKksWPH6rjjjtO3vvUt3XrrrRo6dGiGK/xqmzdvnp5++mlt3rxZw4cPP+rY7t4LhwwZ4lt9ffaKRUlJSYfbu0tSfX09t3f3werVqzV79mytXr1a5eXlmS6nz4nH43r99de1a9eu9se1116r008/Xbt27dKkSZMyXWKfMXnyZH344Ydqbm5u3/bmm28qKyvrmCdw9FxLS4uysjq+/fTr10+S5HL7qm65rqt58+Zp/fr1evnllzVq1Khj7pOJ98LQXLFobm7W3r1725+/88472rVrlwYOHKgRI0Zo4cKF2r9/v373u99Jkq699lrdc889uummmzRnzhy9/PLLWrt2rZ555plM/RNCwWufE4mEqqqqdNddd2nSpEntn9vl5uZ2+L8+dOSlz1lZWZ0+Qx08eLBycnKO+tkqvK/nyspKLVmyRLNnz1ZdXZ0aGxu1YMECzZkzhyudR+G1zxUVFZo7d65WrlypsrIyffTRR5o/f74mTpyowsLCTP0zvvKqq6uVSCT05JNPKj8/v/18O2DAgPb1OWvWLA0bNkzLli2TJN1www2aMmWKfvnLX6q8vFxr1qxRQ0ODHnjgAf8K9e33TYz96U9/ciV1elRVVbmu67pVVVXulClTOu0zfvx4t3///u4pp5zirlq1KvC6w8Zrn6dMmXLU8ehab9bzf+PXTXumN33++9//7k6bNs3Nzc11hw8f7tbU1LgtLS3BFx8ivenz3Xff7Y4ZM8bNzc11hw4d6l555ZXuBx98EHzxIdJVjyV1eG+bMmVKp/Pv2rVr3dGjR7v9+/d3i4uL3WeeecbXOrltOgAAMNNnv2MBAACCR7AAAABmCBYAAMAMwQIAAJghWAAAADMECwAAYIZgAQAAzBAsAACAGYIFAAAwQ7AAAABmCBYAAMAMwQIAAJj5fxW6ZQeCCDn1AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# decide preference\n",
        "import numpy as np\n",
        "def calc_preference(scores):\n",
        "    preference = np.ones(len(scores), dtype=int)\n",
        "\n",
        "    idx_two = scores > 1.5\n",
        "    preference[idx_two] = 2\n",
        "    \n",
        "    return list(preference)\n",
        "\n",
        "eval_validation['preference'] = calc_preference(eval_validation['score'])\n",
        "# eval_dev[]\n",
        "# eval_dev['score']\n",
        "eval_validation['preference'].hist()\n",
        "print(eval_validation)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "u4OC1f6tnfBr",
        "outputId": "a33fce5f-722d-4af3-e5ca-d2708647ab6a"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    Unnamed: 0.1  Unnamed: 0  left  right  T1_ALR  T2_CL  T3_GF  T4_MQ  T5_NS  \\\n",
            "0              0           0   909    318       2      2      2    1.0      2   \n",
            "1             16          16   285    609       2      2      2    2.0      2   \n",
            "2             48          48   132    271       1      2      2    2.0      1   \n",
            "3             38          38   704    127       2      2      1    2.0      2   \n",
            "4             33          33    16    586       1      1      2    2.0      2   \n",
            "5             17          17    56    865       2      1      2    2.0      2   \n",
            "6             27          27   233    478       1      1      1    1.0      2   \n",
            "7             41          41   877    620       2      2      2    2.0      2   \n",
            "8              5           5   246    733       1      1      1    2.0      1   \n",
            "9             35          35   668    741       1      1      1    1.0      1   \n",
            "10            11          11   691      4       2      2      2    1.0      1   \n",
            "11            47          47   403    830       2      2      2    2.0      1   \n",
            "12            10          10   404     27       2      2      1    2.0      1   \n",
            "13            49          49   202    761       1      2      2    2.0      2   \n",
            "14             2           2   555    815       1      1      1    1.0      1   \n",
            "15            18          18   527    876       2      2      2    1.0      2   \n",
            "16            20          20   225    558       2      2      2    2.0      2   \n",
            "17            12          12   316    121       1      2      2    2.0      2   \n",
            "18            46          46   363    780       2      1      2    2.0      2   \n",
            "19            29          29   118    116       1      1      1    1.0      1   \n",
            "20             8           8   676    153       1      1      1    2.0      1   \n",
            "21            24          24   527    129       2      2      2    2.0      2   \n",
            "22            36          36   327    721       2      1      2    2.0      1   \n",
            "23            14          14   158    139       2      1      1    1.0      2   \n",
            "24            22          22   547    578       1      1      1    1.0      1   \n",
            "\n",
            "    score  preference  \n",
            "0     1.8           2  \n",
            "1     2.0           2  \n",
            "2     1.6           2  \n",
            "3     1.8           2  \n",
            "4     1.6           2  \n",
            "5     1.8           2  \n",
            "6     1.2           1  \n",
            "7     2.0           2  \n",
            "8     1.2           1  \n",
            "9     1.0           1  \n",
            "10    1.6           2  \n",
            "11    1.8           2  \n",
            "12    1.6           2  \n",
            "13    1.8           2  \n",
            "14    1.0           1  \n",
            "15    1.8           2  \n",
            "16    2.0           2  \n",
            "17    1.8           2  \n",
            "18    1.8           2  \n",
            "19    1.0           1  \n",
            "20    1.2           1  \n",
            "21    2.0           2  \n",
            "22    1.6           2  \n",
            "23    1.4           1  \n",
            "24    1.0           1  \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiZklEQVR4nO3dbXBU5f3G8WuTLAthEhQikEBQtAoCCmN5GIoVqIRMxCCdVqihkgGlUkGkaUGpgknBUrFl0MpI7bSgUwJqFbQ+ABkEIgpKgLTaKg8VK0oDDdYsJLrdZs//hf8spgmQxbO/9YTvZ8YXe+/Z+/y8WDgXuxvW5ziOIwAAACNJiR4AAACcWygfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMJWS6AH+VyQS0eHDh5WWliafz5focQAAQAs4jqPjx48rKytLSUmnf23jK1c+Dh8+rOzs7ESPAQAAzsKhQ4fUvXv30x7zlSsfaWlpkj4fPj093dW9w+GwNm7cqNGjR8vv97u6N04iZxvkbIesbZCzjXjlHAwGlZ2dHb2On85Xrnw0vNWSnp4el/KRmpqq9PR0nthxRM42yNkOWdsgZxvxzrklH5ngA6cAAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAqZREDwAAgJdddPeLiR4hJoFkR4sHJ3YGXvkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwFXP5KC8vV35+vrKysuTz+bRu3bomx7zzzjsaO3asOnTooPbt22vQoEH64IMP3JgXAAB4XMzlo7a2Vv3799eyZcuavf/vf/+7rr76avXu3VtbtmzRX/7yF82bN09t27b90sMCAADvS4n1AXl5ecrLyzvl/ffcc4+uu+46LV68OLp2ySWXnN10AACg1Ym5fJxOJBLRiy++qDlz5ig3N1d79uxRz549NXfuXI0bN67Zx4RCIYVCoejtYDAoSQqHwwqHw26OF93P7X3RGDnbIGc7ZG3DqzkHkp1EjxCTQNLn88brGtsSPsdxzjo1n8+ntWvXRotFVVWVMjMzlZqaqoULF2rkyJFav369fvrTn2rz5s0aPnx4kz2Ki4tVUlLSZL20tFSpqalnOxoAADBUV1engoIC1dTUKD09/bTHulo+Dh8+rG7duummm25SaWlp9LixY8eqffv2Wr16dZM9mnvlIzs7W9XV1WccPlbhcFhlZWXKycmR3+93dW+cRM42yNkOWdvwas79ijckeoSYBJIcLRgYcT3nYDCojIyMFpUPV992ycjIUEpKivr06dNo/fLLL9e2bduafUwgEFAgEGiy7vf74/bki+feOImcbZCzHbK24bWcQ/W+RI9wVtzOOZa9XP13Ptq0aaNBgwZp7969jdb37dunCy+80M1TAQAAj4r5lY8TJ07owIED0dsHDx5UZWWlOnbsqB49emj27NmaMGGCrrnmmuhnPv70pz9py5Ytbs4NAAA8KubyUVFRoZEjR0ZvFxUVSZIKCwu1cuVKffvb39by5cu1aNEizZw5U7169dIzzzyjq6++2r2pAQCAZ8VcPkaMGKEzfUZ1ypQpmjJlylkPBQAAWi++2wUAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwFXP5KC8vV35+vrKysuTz+bRu3bpTHjtt2jT5fD4tXbr0S4wIAABak5jLR21trfr3769ly5ad9ri1a9dqx44dysrKOuvhAABA65MS6wPy8vKUl5d32mM++ugj3XHHHdqwYYPGjBlz1sMBAIDWJ+bycSaRSEQ333yzZs+erb59+57x+FAopFAoFL0dDAYlSeFwWOFw2NXZGvZze180Rs42yNkOWdvwas6BZCfRI8QkkPT5vPG6xraE6+XjgQceUEpKimbOnNmi4xctWqSSkpIm6xs3blRqaqrb40mSysrK4rIvGiNnG+Rsh6xteC3nxYMTPcHZcTvnurq6Fh/ravnYtWuXHnroIe3evVs+n69Fj5k7d66Kioqit4PBoLKzszV69Gilp6e7OZ7C4bDKysqUk5Mjv9/v6t44iZxtkLMdsrbh1Zz7FW9I9AgxCSQ5WjAw4nrODe9ctISr5ePVV1/V0aNH1aNHj+hafX29fvzjH2vp0qV6//33mzwmEAgoEAg0Wff7/XF78sVzb5xEzjbI2Q5Z2/BazqH6lv1l+6vG7Zxj2cvV8nHzzTdr1KhRjdZyc3N18803a/LkyW6eCgAAeFTM5ePEiRM6cOBA9PbBgwdVWVmpjh07qkePHurUqVOj4/1+v7p27apevXp9+WkBAIDnxVw+KioqNHLkyOjths9rFBYWauXKla4NBgAAWqeYy8eIESPkOC3/saLmPucBAADOXXy3CwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAEzFXD7Ky8uVn5+vrKws+Xw+rVu3LnpfOBzWXXfdpSuuuELt27dXVlaWJk2apMOHD7s5MwAA8LCYy0dtba369++vZcuWNbmvrq5Ou3fv1rx587R79249++yz2rt3r8aOHevKsAAAwPtSYn1AXl6e8vLymr2vQ4cOKisra7T2yCOPaPDgwfrggw/Uo0ePs5sSAAC0GjGXj1jV1NTI5/PpvPPOa/b+UCikUCgUvR0MBiV9/hZOOBx2dZaG/dzeF42Rsw1ytkPWNryacyDZSfQIMQkkfT5vvK6xLeFzHOesU/P5fFq7dq3GjRvX7P2fffaZhg0bpt69e2vVqlXNHlNcXKySkpIm66WlpUpNTT3b0QAAgKG6ujoVFBSopqZG6enppz02buUjHA7rO9/5jj788ENt2bLllIM098pHdna2qqurzzh8rMLhsMrKypSTkyO/3+/q3jiJnG2Qsx2ytuHVnPsVb0j0CDEJJDlaMDDies7BYFAZGRktKh9xedslHA5r/Pjx+sc//qFXXnnltEMEAgEFAoEm636/P25PvnjujZPI2QY52yFrG17LOVTvS/QIZ8XtnGPZy/Xy0VA89u/fr82bN6tTp05unwIAAHhYzOXjxIkTOnDgQPT2wYMHVVlZqY4dOyozM1Pf/e53tXv3br3wwguqr69XVVWVJKljx45q06aNe5MDAABPirl8VFRUaOTIkdHbRUVFkqTCwkIVFxfr+eeflyQNGDCg0eM2b96sESNGnP2kAACgVYi5fIwYMUKn+4zql/j8KgAAOAfw3S4AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAqZjLR3l5ufLz85WVlSWfz6d169Y1ut9xHM2fP1+ZmZlq166dRo0apf3797s1LwAA8LiYy0dtba369++vZcuWNXv/4sWL9fDDD2v58uV644031L59e+Xm5uqzzz770sMCAADvS4n1AXl5ecrLy2v2PsdxtHTpUt1777264YYbJElPPPGEunTponXr1ul73/vel5sWAAB4Xszl43QOHjyoqqoqjRo1KrrWoUMHDRkyRNu3b2+2fIRCIYVCoejtYDAoSQqHwwqHw26OF93P7X3RGDnbIGc7ZG3DqzkHkp1EjxCTQNLn88brGtsSrpaPqqoqSVKXLl0arXfp0iV63/9atGiRSkpKmqxv3LhRqampbo4XVVZWFpd90Rg52yBnO2Rtw2s5Lx6c6AnOjts519XVtfhYV8vH2Zg7d66Kioqit4PBoLKzszV69Gilp6e7eq5wOKyysjLl5OTI7/e7ujdOImcb5GyHrG14Ned+xRsSPUJMAkmOFgyMuJ5zwzsXLeFq+ejatask6ciRI8rMzIyuHzlyRAMGDGj2MYFAQIFAoMm63++P25MvnnvjJHK2Qc52yNqG13IO1fsSPcJZcTvnWPZy9d/56Nmzp7p27apNmzZF14LBoN544w0NHTrUzVMBAACPivmVjxMnTujAgQPR2wcPHlRlZaU6duyoHj16aNasWVq4cKEuvfRS9ezZU/PmzVNWVpbGjRvn5twAAMCjYi4fFRUVGjlyZPR2w+c1CgsLtXLlSs2ZM0e1tbX6wQ9+oE8++URXX3211q9fr7Zt27o3NQAA8KyYy8eIESPkOKf+sSKfz6ef/exn+tnPfvalBgMAAK0T3+0CAABMUT4AAIApygcAADBF+QAAAKYoHwAAwBTlAwAAmKJ8AAAAU5QPAABgivIBAABMUT4AAIApygcAADBF+QAAAKYoHwAAwBTlAwAAmKJ8AAAAU5QPAABgivIBAABMUT4AAIApygcAADBF+QAAAKYoHwAAwBTlAwAAmKJ8AAAAU5QPAABgivIBAABMUT4AAIApygcAADBF+QAAAKYoHwAAwBTlAwAAmHK9fNTX12vevHnq2bOn2rVrp0suuUQLFiyQ4zhunwoAAHhQitsbPvDAA3r00Uf1+OOPq2/fvqqoqNDkyZPVoUMHzZw50+3TAQAAj3G9fLz++uu64YYbNGbMGEnSRRddpNWrV+vNN990+1QAAMCDXH/b5Rvf+IY2bdqkffv2SZL+/Oc/a9u2bcrLy3P7VAAAwINcf+Xj7rvvVjAYVO/evZWcnKz6+nrdf//9mjhxYrPHh0IhhUKh6O1gMChJCofDCofDrs7WsJ/b+6IxcrZBznbI2oZXcw4ke+szjYGkz+eN1zW2JXyOy58EXbNmjWbPnq0HH3xQffv2VWVlpWbNmqUlS5aosLCwyfHFxcUqKSlpsl5aWqrU1FQ3RwMAAHFSV1engoIC1dTUKD09/bTHul4+srOzdffdd2v69OnRtYULF+oPf/iD3n333SbHN/fKR3Z2tqqrq884fKzC4bDKyso0ryJJoYjP1b3j6e3i3ESPEJOGnHNycuT3+xM9TqtFznbI2oZXc+5XvCHRI8QkkORowcCI6zkHg0FlZGS0qHy4/rZLXV2dkpIaf5QkOTlZkUik2eMDgYACgUCTdb/fH7cnXyjiU6jeO+XDS78Jvyiev4Y4iZztkLUNr+XspevJF7mdcyx7uV4+8vPzdf/996tHjx7q27ev9uzZoyVLlmjKlClunwoAAHiQ6+Xj17/+tebNm6fbb79dR48eVVZWlm677TbNnz/f7VMBAAAPcr18pKWlaenSpVq6dKnbWwMAgFaA73YBAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAqbiUj48++kjf//731alTJ7Vr105XXHGFKioq4nEqAADgMSlub/jvf/9bw4YN08iRI/Xyyy/rggsu0P79+3X++ee7fSoAAOBBrpePBx54QNnZ2VqxYkV0rWfPnm6fBgAAeJTr5eP5559Xbm6ubrzxRm3dulXdunXT7bffrqlTpzZ7fCgUUigUit4OBoOSpHA4rHA47OpsDfsFkhxX9403t3OIt4Z5vTa315CzHbK24dWcA8neuqY0XAPjdY1tCZ/jOK6m1rZtW0lSUVGRbrzxRu3cuVN33nmnli9frsLCwibHFxcXq6SkpMl6aWmpUlNT3RwNAADESV1dnQoKClRTU6P09PTTHut6+WjTpo0GDhyo119/Pbo2c+ZM7dy5U9u3b29yfHOvfGRnZ6u6uvqMw8cqHA6rrKxM8yqSFIr4XN07nt4uzk30CDFpyDknJ0d+vz/R47Ra5GyHrG14Ned+xRsSPUJMAkmOFgyMuJ5zMBhURkZGi8qH62+7ZGZmqk+fPo3WLr/8cj3zzDPNHh8IBBQIBJqs+/3+uD35QhGfQvXeKR9e+k34RfH8NcRJ5GyHrG14LWcvXU++yO2cY9nL9R+1HTZsmPbu3dtobd++fbrwwgvdPhUAAPAg18vHj370I+3YsUM///nPdeDAAZWWluqxxx7T9OnT3T4VAADwINfLx6BBg7R27VqtXr1a/fr104IFC7R06VJNnDjR7VMBAAAPcv0zH5J0/fXX6/rrr4/H1gAAwOP4bhcAAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAVNzLxy9+8Qv5fD7NmjUr3qcCAAAeENfysXPnTv3mN7/RlVdeGc/TAAAAD4lb+Thx4oQmTpyo3/72tzr//PPjdRoAAOAxKfHaePr06RozZoxGjRqlhQsXnvK4UCikUCgUvR0MBiVJ4XBY4XDY1Zka9gskOa7uG29u5xBvDfN6bW6vIWc7ZG3DqzkHkr11TWm4BsbrGtsSPsdxXE9tzZo1uv/++7Vz5061bdtWI0aM0IABA7R06dImxxYXF6ukpKTJemlpqVJTU90eDQAAxEFdXZ0KCgpUU1Oj9PT00x7revk4dOiQBg4cqLKysuhnPU5XPpp75SM7O1vV1dVnHD5W4XBYZWVlmleRpFDE5+re8fR2cW6iR4hJQ845OTny+/2JHqfVImc7ZG3Dqzn3K96Q6BFiEkhytGBgxPWcg8GgMjIyWlQ+XH/bZdeuXTp69Kiuuuqq6Fp9fb3Ky8v1yCOPKBQKKTk5OXpfIBBQIBBoso/f74/bky8U8SlU753y4aXfhF8Uz19DnETOdsjahtdy9tL15IvczjmWvVwvH9dee63eeuutRmuTJ09W7969dddddzUqHgAA4NzjevlIS0tTv379Gq21b99enTp1arIOAADOPfwLpwAAwFTcftT2i7Zs2WJxGgAA4AG88gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTlA8AAGCK8gEAAExRPgAAgCnKBwAAMEX5AAAApigfAADAFOUDAACYonwAAABTrpePRYsWadCgQUpLS1Pnzp01btw47d271+3TAAAAj3K9fGzdulXTp0/Xjh07VFZWpnA4rNGjR6u2ttbtUwEAAA9KcXvD9evXN7q9cuVKde7cWbt27dI111zj9ukAAIDHuF4+/ldNTY0kqWPHjs3eHwqFFAqForeDwaAkKRwOKxwOuzpLw36BJMfVfePN7RzirWFer83tNeRsh6xteDXnQLK3rikN18B4XWNbwuc4TtxSi0QiGjt2rD755BNt27at2WOKi4tVUlLSZL20tFSpqanxGg0AALiorq5OBQUFqqmpUXp6+mmPjWv5+OEPf6iXX35Z27ZtU/fu3Zs9prlXPrKzs1VdXX3G4WMVDodVVlameRVJCkV8ru4dT28X5yZ6hJg05JyTkyO/35/ocVotcrZD1ja8mnO/4g2JHiEmgSRHCwZGXM85GAwqIyOjReUjbm+7zJgxQy+88ILKy8tPWTwkKRAIKBAINFn3+/1xe/KFIj6F6r1TPrz0m/CL4vlriJPI2Q5Z2/Bazl66nnyR2znHspfr5cNxHN1xxx1au3attmzZop49e7p9CgAA4GGul4/p06ertLRUzz33nNLS0lRVVSVJ6tChg9q1a+f26QAAgMe4/u98PProo6qpqdGIESOUmZkZ/e/JJ590+1QAAMCD4vK2CwAAwKnw3S4AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAqbiVj2XLlumiiy5S27ZtNWTIEL355pvxOhUAAPCQuJSPJ598UkVFRbrvvvu0e/du9e/fX7m5uTp69Gg8TgcAADwkLuVjyZIlmjp1qiZPnqw+ffpo+fLlSk1N1e9///t4nA4AAHhIitsb/uc//9GuXbs0d+7c6FpSUpJGjRql7du3Nzk+FAopFApFb9fU1EiSPv74Y4XDYVdnC4fDqqurU0o4SfURn6t7x9OxY8cSPUJMGnI+duyY/H5/osdptcjZDlnb8GrOKf+tTfQIMUmJOKqri7ie8/HjxyVJjuOceQbXzvr/qqurVV9fry5dujRa79Kli959990mxy9atEglJSVN1nv27On2aJ6V8atETwAAaE0K4rj38ePH1aFDh9Me43r5iNXcuXNVVFQUvR2JRPTxxx+rU6dO8vncfXUiGAwqOztbhw4dUnp6uqt74yRytkHOdsjaBjnbiFfOjuPo+PHjysrKOuOxrpePjIwMJScn68iRI43Wjxw5oq5duzY5PhAIKBAINFo777zz3B6rkfT0dJ7YBsjZBjnbIWsb5GwjHjmf6RWPBq5/4LRNmzb6+te/rk2bNkXXIpGINm3apKFDh7p9OgAA4DFxedulqKhIhYWFGjhwoAYPHqylS5eqtrZWkydPjsfpAACAh8SlfEyYMEH/+te/NH/+fFVVVWnAgAFav359kw+hWgsEArrvvvuavM0Dd5GzDXK2Q9Y2yNnGVyFnn9OSn4kBAABwCd/tAgAATFE+AACAKcoHAAAwRfkAAACmWlX5KC8vV35+vrKysuTz+bRu3bozPmbLli266qqrFAgE9LWvfU0rV66M+5xeF2vOzz77rHJycnTBBRcoPT1dQ4cO1YYNG2yG9bCzeT43eO2115SSkqIBAwbEbb7W4mxyDoVCuueee3ThhRcqEAjooosu4oszz+Bscl61apX69++v1NRUZWZmasqUKZ77ritrixYt0qBBg5SWlqbOnTtr3Lhx2rt37xkf9/TTT6t3795q27atrrjiCr300ktxnbNVlY/a2lr1799fy5Yta9HxBw8e1JgxYzRy5EhVVlZq1qxZuvXWW7kwnkGsOZeXlysnJ0cvvfSSdu3apZEjRyo/P1979uyJ86TeFmvODT755BNNmjRJ1157bZwma13OJufx48dr06ZN+t3vfqe9e/dq9erV6tWrVxyn9L5Yc37ttdc0adIk3XLLLfrrX/+qp59+Wm+++aamTp0a50m9bevWrZo+fbp27NihsrIyhcNhjR49WrW1p/7yu9dff1033XSTbrnlFu3Zs0fjxo3TuHHj9Pbbb8dvUKeVkuSsXbv2tMfMmTPH6du3b6O1CRMmOLm5uXGcrHVpSc7N6dOnj1NSUuL+QK1ULDlPmDDBuffee5377rvP6d+/f1znam1akvPLL7/sdOjQwTl27JjNUK1QS3J+8MEHnYsvvrjR2sMPP+x069YtjpO1PkePHnUkOVu3bj3lMePHj3fGjBnTaG3IkCHObbfdFre5WtUrH7Havn27Ro0a1WgtNzdX27dvT9BE54ZIJKLjx4+rY8eOiR6l1VmxYoXee+893XfffYkepdV6/vnnNXDgQC1evFjdunXTZZddpp/85Cf69NNPEz1aqzJ06FAdOnRIL730khzH0ZEjR/THP/5R1113XaJH85SamhpJOu2ft4m4Fib8W20Tqaqqqsm/utqlSxcFg0F9+umnateuXYIma91++ctf6sSJExo/fnyiR2lV9u/fr7vvvluvvvqqUlLO6d/acfXee+9p27Ztatu2rdauXavq6mrdfvvtOnbsmFasWJHo8VqNYcOGadWqVZowYYI+++wz/fe//1V+fn7Mb0OeyyKRiGbNmqVhw4apX79+pzzuVNfCqqqquM12Tr/yAXulpaUqKSnRU089pc6dOyd6nFajvr5eBQUFKikp0WWXXZbocVq1SCQin8+nVatWafDgwbruuuu0ZMkSPf7447z64aK//e1vuvPOOzV//nzt2rVL69ev1/vvv69p06YlejTPmD59ut5++22tWbMm0aM0cU7/9ahr1646cuRIo7UjR44oPT2dVz3iYM2aNbr11lv19NNPN3mJD1/O8ePHVVFRoT179mjGjBmSPr9IOo6jlJQUbdy4Ud/61rcSPGXrkJmZqW7dujX66vDLL79cjuPoww8/1KWXXprA6VqPRYsWadiwYZo9e7Yk6corr1T79u31zW9+UwsXLlRmZmaCJ/xqmzFjhl544QWVl5ere/fupz32VNfCrl27xm2+c/qVj6FDh2rTpk2N1srKyjR06NAETdR6rV69WpMnT9bq1as1ZsyYRI/T6qSnp+utt95SZWVl9L9p06apV69eqqys1JAhQxI9YqsxbNgwHT58WCdOnIiu7du3T0lJSWf8Qx4tV1dXp6Skxpeo5ORkSZLDV5KdkuM4mjFjhtauXatXXnlFPXv2PONjEnEtbFWvfJw4cUIHDhyI3j548KAqKyvVsWNH9ejRQ3PnztVHH32kJ554QpI0bdo0PfLII5ozZ46mTJmiV155RU899ZRefPHFRP0veEKsOZeWlqqwsFAPPfSQhgwZEn0fsV27do3+9ojGYsk5KSmpyXu6nTt3Vtu2bU/7Xi9ifz4XFBRowYIFmjx5skpKSlRdXa3Zs2drypQpvGJ6GrHmnJ+fr6lTp+rRRx9Vbm6u/vnPf2rWrFkaPHiwsrKyEvW/8ZU3ffp0lZaW6rnnnlNaWlr0z9sOHTpEn5+TJk1St27dtGjRIknSnXfeqeHDh+tXv/qVxowZozVr1qiiokKPPfZY/AaN28/RJMDmzZsdSU3+KywsdBzHcQoLC53hw4c3ecyAAQOcNm3aOBdffLGzYsUK87m9Jtachw8fftrj0byzeT5/ET9q2zJnk/M777zjjBo1ymnXrp3TvXt3p6ioyKmrq7Mf3kPOJueHH37Y6dOnj9OuXTsnMzPTmThxovPhhx/aD+8hzWUsqdG1bfjw4U3+/H3qqaecyy67zGnTpo3Tt29f58UXX4zrnL7/HxYAAMDEOf2ZDwAAYI/yAQAATFE+AACAKcoHAAAwRfkAAACmKB8AAMAU5QMAAJiifAAAAFOUDwAAYIryAQAATFE+AACAKcoHAAAw9X/OGpo8GZS9rQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#template = pd.read_csv('/content/drive/MyDrive/2023-1_ML/Term_Project/submission/template.csv')\n",
        "template1 = pd.read_csv('CR_rank.csv')\n",
        "template1.columns = ['QuestionId', 'ranking']\n",
        "\n",
        "template2 = pd.read_csv('CR_inverse_rank.csv')\n",
        "template2.columns = ['QuestionId', 'ranking']\n",
        "\n",
        "template3 = pd.read_csv('IC_AV_rank.csv')\n",
        "template3.columns = ['QuestionId', 'ranking']\n",
        "\n",
        "template4 = pd.read_csv('IC_AV_SS_rank.csv')\n",
        "template4.columns = ['QuestionId', 'ranking']\n",
        "\n",
        "template5 = pd.read_csv('IC_AV_SS_CR_inv_rank.csv')\n",
        "template5.columns = ['QuestionId', 'ranking']\n",
        "\n",
        "template6 = pd.read_csv('CR_inverse+SS_rank.csv')\n",
        "template6.columns = ['QuestionId', 'ranking']\n",
        "\n",
        "template7 = pd.read_csv('IC_rank.csv')\n",
        "template7.columns = ['QuestionId', 'ranking']\n"
      ],
      "metadata": {
        "id": "bqnHt7dfewZE"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "template1.head()"
      ],
      "metadata": {
        "id": "3dBpA1mley-8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "c918ab82-22c3-4b2d-9c8a-72eed8a565cb"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   QuestionId  ranking\n",
              "0           0      594\n",
              "1           1      328\n",
              "2           2      721\n",
              "3           3       27\n",
              "4           4      676"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2b1672a2-6294-40a0-945e-8fa1ca24c62f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>QuestionId</th>\n",
              "      <th>ranking</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>594</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>328</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>676</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2b1672a2-6294-40a0-945e-8fa1ca24c62f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2b1672a2-6294-40a0-945e-8fa1ca24c62f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2b1672a2-6294-40a0-945e-8fa1ca24c62f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(truth, submission):\n",
        "    # extract ranking\n",
        "    left = list(truth.left)\n",
        "    right = list(truth.right)\n",
        "#     if len(left) != len(right):\n",
        "#     message = 'left and right lengths are not the same'\n",
        "#     sys.exit(message)\n",
        "\n",
        "    submission_left = []\n",
        "    submission_right = []\n",
        "    submission_preference = []\n",
        "    for idx in range(len(left)):\n",
        "        submission_left.append(left[idx])\n",
        "        submission_right.append(right[idx])\n",
        "        ranking_left = submission[submission.QuestionId==left[idx]].ranking.values[0]\n",
        "        ranking_right = submission[submission.QuestionId==right[idx]].ranking.values[0]\n",
        "        preference = 1 if ranking_left < ranking_right else 2\n",
        "        submission_preference.append(preference)\n",
        "    print(submission_preference)\n",
        "    return submission_preference\n"
      ],
      "metadata": {
        "id": "6KhgYCyOb_Ns"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "CR_result = evaluate(eval_validation, template1)\n",
        "CR_inverse_result = evaluate(eval_validation, template2)\n",
        "IC_AV_result = evaluate(eval_validation, template3)\n",
        "IC_AV_SS_result = evaluate(eval_validation, template4)\n",
        "IC_AV_SS_CR_inv_result = evaluate(eval_validation, template5)\n",
        "CR_inverse_SS_result = evaluate(eval_validation, template6)\n",
        "IC_result = evaluate(eval_validation, template7)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aCS7-C80obi3",
        "outputId": "8de3f651-dd81-44a8-c930-dda73dcc08e3"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 2]\n",
            "[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 1, 1, 2, 1, 2, 1]\n",
            "[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 1]\n",
            "[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 1, 1, 2, 1, 1, 1]\n",
            "[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 1, 1, 2, 1, 1, 1]\n",
            "[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1]\n",
            "[2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 1, 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "public_result = eval_validation['preference']\n",
        "print(public_result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ShuV12y3KvQ5",
        "outputId": "f4c1531a-e9ae-4057-f31f-726fdd168558"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0     2\n",
            "1     2\n",
            "2     2\n",
            "3     2\n",
            "4     2\n",
            "5     2\n",
            "6     1\n",
            "7     2\n",
            "8     1\n",
            "9     1\n",
            "10    2\n",
            "11    2\n",
            "12    2\n",
            "13    2\n",
            "14    1\n",
            "15    2\n",
            "16    2\n",
            "17    2\n",
            "18    2\n",
            "19    1\n",
            "20    1\n",
            "21    2\n",
            "22    2\n",
            "23    1\n",
            "24    1\n",
            "Name: preference, dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "estimate_df = pd.DataFrame(CR_result, columns=['CR_estimate'])\n",
        "estimate_df['CR_inverse_estimate'] = CR_inverse_result\n",
        "estimate_df['IC_AV_estimate'] = IC_AV_result\n",
        "estimate_df['IC_AV_SS_estimate'] = IC_AV_SS_result\n",
        "estimate_df['IC_AV_SS_CR_inv_estimate'] = IC_AV_SS_CR_inv_result\n",
        "estimate_df['CR_inverse_SS_estimate'] = CR_inverse_SS_result\n",
        "estimate_df['IC_estimate'] = IC_result\n",
        "\n",
        "estimate_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 833
        },
        "id": "1314ppmjqMsv",
        "outputId": "e4550c6c-bbac-418a-c221-6426b7c1bb1e"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    CR_estimate  CR_inverse_estimate  IC_AV_estimate  IC_AV_SS_estimate  \\\n",
              "0             1                    2               2                  2   \n",
              "1             1                    2               2                  2   \n",
              "2             1                    2               2                  2   \n",
              "3             1                    2               2                  2   \n",
              "4             1                    2               2                  2   \n",
              "5             1                    2               2                  2   \n",
              "6             2                    1               1                  1   \n",
              "7             1                    2               2                  2   \n",
              "8             2                    1               1                  1   \n",
              "9             1                    2               2                  2   \n",
              "10            1                    2               2                  2   \n",
              "11            1                    2               2                  2   \n",
              "12            2                    1               2                  2   \n",
              "13            1                    2               2                  2   \n",
              "14            2                    1               1                  1   \n",
              "15            1                    2               2                  2   \n",
              "16            2                    1               1                  1   \n",
              "17            2                    1               1                  2   \n",
              "18            1                    2               2                  2   \n",
              "19            2                    1               1                  1   \n",
              "20            2                    1               1                  1   \n",
              "21            1                    2               2                  2   \n",
              "22            2                    1               1                  1   \n",
              "23            1                    2               1                  1   \n",
              "24            2                    1               1                  1   \n",
              "\n",
              "    IC_AV_SS_CR_inv_estimate  CR_inverse_SS_estimate  IC_estimate  \n",
              "0                          2                       2            2  \n",
              "1                          2                       2            2  \n",
              "2                          2                       2            1  \n",
              "3                          2                       2            2  \n",
              "4                          2                       2            1  \n",
              "5                          2                       2            1  \n",
              "6                          1                       1            1  \n",
              "7                          2                       2            1  \n",
              "8                          1                       1            1  \n",
              "9                          2                       2            2  \n",
              "10                         2                       2            2  \n",
              "11                         2                       2            2  \n",
              "12                         2                       1            2  \n",
              "13                         2                       2            2  \n",
              "14                         1                       1            1  \n",
              "15                         2                       2            2  \n",
              "16                         1                       1            2  \n",
              "17                         2                       2            2  \n",
              "18                         2                       2            2  \n",
              "19                         1                       1            1  \n",
              "20                         1                       1            1  \n",
              "21                         2                       2            2  \n",
              "22                         1                       1            1  \n",
              "23                         1                       2            1  \n",
              "24                         1                       1            1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-00be7a9e-f529-4055-baaa-9725a4f9a409\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CR_estimate</th>\n",
              "      <th>CR_inverse_estimate</th>\n",
              "      <th>IC_AV_estimate</th>\n",
              "      <th>IC_AV_SS_estimate</th>\n",
              "      <th>IC_AV_SS_CR_inv_estimate</th>\n",
              "      <th>CR_inverse_SS_estimate</th>\n",
              "      <th>IC_estimate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-00be7a9e-f529-4055-baaa-9725a4f9a409')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-00be7a9e-f529-4055-baaa-9725a4f9a409 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-00be7a9e-f529-4055-baaa-9725a4f9a409');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def getCompareDF(eval_validation, estimate_df):\n",
        "  df = pd.DataFrame()\n",
        "  df['truth'] = eval_validation['preference']\n",
        "  df['estimate'] = estimate_df\n",
        "  df['evaluate'] = df['truth'] == df['estimate']\n",
        "  df['score'] = df['evaluate'].apply(lambda x: 1 if x else 0)\n",
        "  total_score = df['score'].sum()\n",
        "  total_percent = total_score/df['score'].count()\n",
        "  return (total_score, total_percent)\n",
        "\n",
        "CR_score, CR_percent = getCompareDF(eval_validation, estimate_df['CR_estimate'])\n",
        "CR_inverse_score, CR_inverse_percent = getCompareDF(eval_validation, estimate_df['CR_inverse_estimate'])\n",
        "IC_AV_score, IC_AV_percent = getCompareDF(eval_validation, estimate_df['IC_AV_estimate'])\n",
        "IC_AV_SS_score, IC_AV_SS_percent = getCompareDF(eval_validation, estimate_df['IC_AV_SS_estimate'])\n",
        "IC_AV_SS_CR_inv_score, IC_AV_SS_CR_inv_percent = getCompareDF(eval_validation, estimate_df['IC_AV_SS_CR_inv_estimate'])\n",
        "CR_inverse_SS_score, CR_inverse_SS_percent = getCompareDF(eval_validation, estimate_df['CR_inverse_SS_estimate'])\n",
        "IC_score, IC_percent = getCompareDF(eval_validation, estimate_df['IC_estimate'])\n",
        "\n",
        "print(f'CR_score: {CR_score}, CR_percent: {CR_percent}')\n",
        "print(f'CR_inverse_score: {CR_inverse_score}, CR_inverse_percent: {CR_inverse_percent}')\n",
        "print(f'CR_inverse_SS_score: {CR_inverse_SS_score}, CR_inverse_SS_percent: {CR_inverse_SS_percent}')\n",
        "print(f'IC_score: {IC_score}, IC_percent: {IC_percent}')\n",
        "print(f'IC_AV_score: {IC_AV_score}, IC_AV_percent: {IC_AV_percent}')\n",
        "print(f'IC_AV_SS_score: {IC_AV_SS_score}, IC_AV_SS_percent: {IC_AV_SS_percent}')\n",
        "print(f'IC_AV_SS_CR_inv_score: {IC_AV_SS_CR_inv_score}, IC_AV_SS_CR_inv_percent: {IC_AV_SS_CR_inv_percent}')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7STniUeRpddZ",
        "outputId": "2acc26c2-3125-43f6-aa9e-c0e7cfbda0a8"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CR_score: 6, CR_percent: 0.24\n",
            "CR_inverse_score: 19, CR_inverse_percent: 0.76\n",
            "CR_inverse_SS_score: 20, CR_inverse_SS_percent: 0.8\n",
            "IC_score: 19, IC_percent: 0.76\n",
            "IC_AV_score: 21, IC_AV_percent: 0.84\n",
            "IC_AV_SS_score: 22, IC_AV_SS_percent: 0.88\n",
            "IC_AV_SS_CR_inv_score: 22, IC_AV_SS_CR_inv_percent: 0.88\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluate Test(private)"
      ],
      "metadata": {
        "id": "xczPmLvERzY0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_test = pd.read_csv('git_dir/data/test_data/quality_response_remapped_private.csv')\n",
        "eval_test['score'] = eval_test.filter(regex='^T', axis = 1).mean(axis=1)\n",
        "eval_test['preference'] = calc_preference(eval_test['score'])\n",
        "\n",
        "eval_test.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ENR5md6qOjOs",
        "outputId": "25193df0-ba8f-4b75-9788-aa91fea4bfa4"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0.1  Unnamed: 0  left  right  T1_ALR  T2_CL  T3_GF  T4_MQ  T5_NS  \\\n",
              "0            15          15   145    762       2      2      2    2.0      2   \n",
              "1            31          31   307    208       1      1      1    2.0      1   \n",
              "2            25          25   278    195       1      1      1    1.0      1   \n",
              "3             3           3    13    153       2      2      2    2.0      2   \n",
              "4            40          40    23    100       2      1      1    1.0      1   \n",
              "\n",
              "   score  preference  \n",
              "0    2.0           2  \n",
              "1    1.2           1  \n",
              "2    1.0           1  \n",
              "3    2.0           2  \n",
              "4    1.2           1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f93bca82-01da-4f3e-b85a-39a2289aa1f9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0.1</th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>left</th>\n",
              "      <th>right</th>\n",
              "      <th>T1_ALR</th>\n",
              "      <th>T2_CL</th>\n",
              "      <th>T3_GF</th>\n",
              "      <th>T4_MQ</th>\n",
              "      <th>T5_NS</th>\n",
              "      <th>score</th>\n",
              "      <th>preference</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>145</td>\n",
              "      <td>762</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>31</td>\n",
              "      <td>31</td>\n",
              "      <td>307</td>\n",
              "      <td>208</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1.2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>25</td>\n",
              "      <td>25</td>\n",
              "      <td>278</td>\n",
              "      <td>195</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>13</td>\n",
              "      <td>153</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>23</td>\n",
              "      <td>100</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1.2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f93bca82-01da-4f3e-b85a-39a2289aa1f9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f93bca82-01da-4f3e-b85a-39a2289aa1f9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f93bca82-01da-4f3e-b85a-39a2289aa1f9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "CR_result = evaluate(eval_test, template1)\n",
        "CR_inverse_result = evaluate(eval_test, template2)\n",
        "IC_AV_result = evaluate(eval_test, template3)\n",
        "IC_AV_SS_result = evaluate(eval_test, template4)\n",
        "IC_AV_SS_CR_inv_result = evaluate(eval_test, template5)\n",
        "CR_inverse_SS_result = evaluate(eval_test, template6)\n",
        "IC_result = evaluate(eval_test, template7)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vPemJC9OR4vC",
        "outputId": "52aa446b-9979-494f-97b9-357fe8eeee58"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1, 1, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1]\n",
            "[2, 2, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 2]\n",
            "[2, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2]\n",
            "[2, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2]\n",
            "[2, 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2]\n",
            "[2, 2, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 2]\n",
            "[1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1, 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df = pd.DataFrame(CR_result, columns=['CR_estimate'])\n",
        "test_df['CR_inverse_estimate'] = CR_inverse_result\n",
        "test_df['IC_AV_estimate'] = IC_AV_result\n",
        "test_df['IC_AV_SS_estimate'] = IC_AV_SS_result\n",
        "test_df['IC_AV_SS_CR_inv_estimate'] = IC_AV_SS_CR_inv_result\n",
        "test_df['CR_inverse_SS_estimate'] = CR_inverse_SS_result\n",
        "test_df['IC_estimate'] = IC_result\n",
        "\n",
        "test_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ZG2N8oVdR8QN",
        "outputId": "b0082685-9629-49d5-9383-826397841bd5"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   CR_estimate  CR_inverse_estimate  IC_AV_estimate  IC_AV_SS_estimate  \\\n",
              "0            1                    2               2                  2   \n",
              "1            1                    2               1                  1   \n",
              "2            2                    1               1                  1   \n",
              "3            2                    1               1                  1   \n",
              "4            2                    1               1                  1   \n",
              "\n",
              "   IC_AV_SS_CR_inv_estimate  CR_inverse_SS_estimate  IC_estimate  \n",
              "0                         2                       2            1  \n",
              "1                         1                       2            1  \n",
              "2                         1                       1            1  \n",
              "3                         1                       1            1  \n",
              "4                         1                       1            1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c212d5c9-0b67-401f-b3e2-5ed2bf873363\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CR_estimate</th>\n",
              "      <th>CR_inverse_estimate</th>\n",
              "      <th>IC_AV_estimate</th>\n",
              "      <th>IC_AV_SS_estimate</th>\n",
              "      <th>IC_AV_SS_CR_inv_estimate</th>\n",
              "      <th>CR_inverse_SS_estimate</th>\n",
              "      <th>IC_estimate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c212d5c9-0b67-401f-b3e2-5ed2bf873363')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c212d5c9-0b67-401f-b3e2-5ed2bf873363 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c212d5c9-0b67-401f-b3e2-5ed2bf873363');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "CR_score, CR_percent = getCompareDF(eval_test, test_df['CR_estimate'])\n",
        "CR_inverse_score, CR_inverse_percent = getCompareDF(eval_test, estimate_df['CR_inverse_estimate'])\n",
        "IC_AV_score, IC_AV_percent = getCompareDF(eval_test, test_df['IC_AV_estimate'])\n",
        "IC_AV_SS_score, IC_AV_SS_percent = getCompareDF(eval_test, test_df['IC_AV_SS_estimate'])\n",
        "IC_AV_SS_CR_inv_score, IC_AV_SS_CR_inv_percent = getCompareDF(eval_test, test_df['IC_AV_SS_CR_inv_estimate'])\n",
        "CR_inverse_SS_score, CR_inverse_SS_percent = getCompareDF(eval_test, test_df['CR_inverse_SS_estimate'])\n",
        "IC_score, IC_percent = getCompareDF(eval_test, test_df['IC_estimate'])\n",
        "\n",
        "print(f'CR_score: {CR_score}, CR_percent: {CR_percent}')\n",
        "print(f'CR_inverse_score: {CR_inverse_score}, CR_inverse_percent: {CR_inverse_percent}')\n",
        "print(f'CR_inverse_SS_score: {CR_inverse_SS_score}, CR_inverse_SS_percent: {CR_inverse_SS_percent}')\n",
        "print(f'IC_score: {IC_score}, IC_percent: {IC_percent}')\n",
        "print(f'IC_AV_score: {IC_AV_score}, IC_AV_percent: {IC_AV_percent}')\n",
        "print(f'IC_AV_SS_score: {IC_AV_SS_score}, IC_AV_SS_percent: {IC_AV_SS_percent}')\n",
        "print(f'IC_AV_SS_CR_inv_score: {IC_AV_SS_CR_inv_score}, IC_AV_SS_CR_inv_percent: {IC_AV_SS_CR_inv_percent}')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HWoBkPYASLd3",
        "outputId": "69602dbe-abe4-467d-a48b-a7d958ecef57"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CR_score: 8, CR_percent: 0.32\n",
            "CR_inverse_score: 9, CR_inverse_percent: 0.36\n",
            "CR_inverse_SS_score: 17, CR_inverse_SS_percent: 0.68\n",
            "IC_score: 12, IC_percent: 0.48\n",
            "IC_AV_score: 15, IC_AV_percent: 0.6\n",
            "IC_AV_SS_score: 15, IC_AV_SS_percent: 0.6\n",
            "IC_AV_SS_CR_inv_score: 16, IC_AV_SS_CR_inv_percent: 0.64\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}