trait_based_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "22fcd924-a534-4895-af6e-00b2a627f6c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import plotly.express as px\n",
    "import plotly.io as pio\n",
    "from itertools import combinations\n",
    "from scipy.spatial.distance import euclidean, cdist\n",
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import statistics, math\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "\n",
    "from eval import evaluate, mean_pearsons, calc_pearsons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "17d86813-c34b-493a-a144-2c3a8d55cb41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The path follows the challenges output format: /MuSe-2024/results/prediction_muse/perception\n",
    "PATH_RESULTS = \"PATH_TO_THE_PERCEPTION_RESULT\"\n",
    "\n",
    "# List of model for emsemble\n",
    "MODELS_LIST = [\"CNN-ATTN_[vit-fer]_[0.0005_32]\", \"CRNN_[vit-fer]_[0.0005_32]\", \"CNN_[vit-fer]_[0.0005_32]\",\n",
    "               \"CRNN-ATTN_[vit-fer]_[0.0005_32]\", \"RNN_[vit-fer]_[0.0005_32]\"]\n",
    "\n",
    "# MODELS_LIST = [\"IAF_[egemaps_vit-fer_bert-base-uncased]_[0.0005_32]\", \"CNN-ATTN_[vit-fer]_[0.0005_32]\", \"CNN_[vit-fer]_[0.0005_32]\", \n",
    "#                \"CRNN_[vit-fer]_[0.0005_32]\", \"CRNN-ATTN_[vit-fer]_[0.0005_32]\", \"RNN_[vit-fer]_[0.0005_32]\",\n",
    "#                \"IAF_[w2v-msp_avhubert-large-noise-pt-noise-ft-30h_vit-fer]_[0.0005_32]\", \n",
    "#                \"IAF_[w2v-msp_vit-fer_avhubert-large-noise-pt-noise-ft-30h]_[0.0005_32]\",\n",
    "#                \"IAF_[vit-fer_vit-fer_faus]_[0.0005_32]\",\n",
    "#                \"IAF_[faus+vit-fer+avhubert-large-noise-pt-noise-ft-30h]_[0.0005_32]\"]\n",
    "\n",
    "# MODELS_LIST = [\"CNN-ATTN_[vit-fer]_[0.0005_32]\", \"CRNN_[vit-fer]_[0.0005_32]\", \"CNN_[vit-fer]_[0.0005_32]\",\n",
    "#                \"CRNN-ATTN_[vit-fer]_[0.0005_32]\", \"RNN_[vit-fer]_[0.0005_32]\", \n",
    "#                \"IAF_[w2v-msp_vit-fer_gpt2]_[0.0005_32]\", \"IAF_[ds_vit-fer_xlm-roberta-large]_[0.0005_32]\",\n",
    "#                \"IAF_[w2v-msp_vit-fer_xlm-roberta-large]_[0.0005_32]\", \"IAF_[egemaps_vit-fer_gpt2]_[0.0005_32]\",\n",
    "#                \"IAF_[ds_vit-fer_gpt2]_[0.0005_32]\"]\n",
    "\n",
    "# Trait group dictionary\n",
    "TRAITS_DICT = {1: [\"aggressive\", \"arrogant\", \"dominant\", \"assertiv\"],\n",
    "               2: [\"risk\", \"independent\", \"leader_like\", \"confident\", \"enthusiastic\"],\n",
    "               3: [],\n",
    "               4: [\"friendly\", \"collaborative\", \"sincere\", \"likeable\", \"kind\", \"warm\", \"good_natured\"]\n",
    "              }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e0c41ced-3498-4e7c-a0e9-9781fa05e7bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_traits = [item for sublist in TRAITS_DICT.values() for item in sublist]\n",
    "\n",
    "# REMEMBER to set it to \"test\" for challenge evaluation\n",
    "def load_data(prefer='dev'):\n",
    "    # Choose file name based on preference    \n",
    "    file_name = \"predictions_devel.csv\" if prefer == 'dev' else \"predictions_test.csv\"\n",
    "\n",
    "    # Create an empty dataframe to stack all dataframes loaded from csvs\n",
    "    final_df = pd.DataFrame()\n",
    "\n",
    "    # Iterate over all models\n",
    "    for model in MODELS_LIST:\n",
    "\n",
    "        # Iterate over all trait categories and traits in the traits_dict\n",
    "        for trait_category, traits in TRAITS_DICT.items():\n",
    "\n",
    "            # Iterate over all traits\n",
    "            for trait in traits:\n",
    "                trait_path = os.path.join(PATH_RESULTS, trait, model, file_name)\n",
    "\n",
    "                # Load data using pandas\n",
    "                df = pd.read_csv(trait_path)\n",
    "\n",
    "                # Add 'model' and 'trait' info to loaded df as new columns\n",
    "                df['model'] = model\n",
    "                df['trait'] = trait\n",
    "\n",
    "                # Stack current df to final_df\n",
    "                final_df = pd.concat([final_df, df])\n",
    "\n",
    "    # Return the final dataframe\n",
    "    return final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5fc873b7-9e61-4ec6-b65f-3a3bece90140",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = load_data(prefer='dev')  # load development data\n",
    "\n",
    "traits = ['aggressive', 'arrogant', 'dominant', 'assertiv', 'risk', 'independent', 'leader_like', 'confident', 'enthusiastic', 'friendly', 'collaborative', 'sincere', 'likeable', 'kind', 'warm', 'good_natured']\n",
    "\n",
    "# Create a category type based on your ordered trait list\n",
    "df['trait'] = pd.Categorical(df['trait'], categories=traits, ordered=True)\n",
    "\n",
    "# Sort by trait (in the order specified by your 'traits' list), then by 'meta_col_0'\n",
    "df_sorted = df.sort_values(by=['trait', 'meta_col_0'])\n",
    "\n",
    "df = df_sorted.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cc319d18-70ba-4c78-8ad7-17f02838859c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_col_0</th>\n",
       "      <th>prediction</th>\n",
       "      <th>label</th>\n",
       "      <th>model</th>\n",
       "      <th>trait</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0.377227</td>\n",
       "      <td>0.3529</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>aggressive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0.354732</td>\n",
       "      <td>0.3529</td>\n",
       "      <td>CRNN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>aggressive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0.324465</td>\n",
       "      <td>0.3529</td>\n",
       "      <td>CNN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>aggressive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0.380100</td>\n",
       "      <td>0.3529</td>\n",
       "      <td>CRNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>aggressive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0.390050</td>\n",
       "      <td>0.3529</td>\n",
       "      <td>RNN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>aggressive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>227</td>\n",
       "      <td>0.634571</td>\n",
       "      <td>0.6979</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>good_natured</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>227</td>\n",
       "      <td>0.537733</td>\n",
       "      <td>0.6979</td>\n",
       "      <td>CRNN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>good_natured</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>227</td>\n",
       "      <td>0.645341</td>\n",
       "      <td>0.6979</td>\n",
       "      <td>CNN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>good_natured</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>227</td>\n",
       "      <td>0.645373</td>\n",
       "      <td>0.6979</td>\n",
       "      <td>CRNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>good_natured</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>227</td>\n",
       "      <td>0.621585</td>\n",
       "      <td>0.6979</td>\n",
       "      <td>RNN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>good_natured</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4640 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    meta_col_0  prediction   label                            model  \\\n",
       "0            9    0.377227  0.3529   CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "0            9    0.354732  0.3529       CRNN_[vit-fer]_[0.0005_32]   \n",
       "0            9    0.324465  0.3529        CNN_[vit-fer]_[0.0005_32]   \n",
       "0            9    0.380100  0.3529  CRNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "0            9    0.390050  0.3529        RNN_[vit-fer]_[0.0005_32]   \n",
       "..         ...         ...     ...                              ...   \n",
       "57         227    0.634571  0.6979   CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "57         227    0.537733  0.6979       CRNN_[vit-fer]_[0.0005_32]   \n",
       "57         227    0.645341  0.6979        CNN_[vit-fer]_[0.0005_32]   \n",
       "57         227    0.645373  0.6979  CRNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "57         227    0.621585  0.6979        RNN_[vit-fer]_[0.0005_32]   \n",
       "\n",
       "           trait  \n",
       "0     aggressive  \n",
       "0     aggressive  \n",
       "0     aggressive  \n",
       "0     aggressive  \n",
       "0     aggressive  \n",
       "..           ...  \n",
       "57  good_natured  \n",
       "57  good_natured  \n",
       "57  good_natured  \n",
       "57  good_natured  \n",
       "57  good_natured  \n",
       "\n",
       "[4640 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trait = df.copy()\n",
    "\n",
    "def filter_by_trait(df, traits_dict, target_trait):\n",
    "    for key in traits_dict:\n",
    "        if target_trait in traits_dict[key]:\n",
    "            group_traits = traits_dict[key]\n",
    "            filtered_df = df[df['trait'].isin(group_traits)] # this will create a new df for each group\n",
    "            return filtered_df\n",
    "\n",
    "# call the function\n",
    "#df_trait = filter_by_trait(df_trait, TRAITS_DICT, 'aggressive')\n",
    "df_trait"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3158488d-0c48-44bf-873a-59b57cb2476d",
   "metadata": {},
   "outputs": [],
   "source": [
    "D1 = 0.2\n",
    "D2 = 0.1\n",
    "W_INTER = 1.0\n",
    "\n",
    "def measure_distance_for_model(df, group, model_name, d1=D1, d2=D2):\n",
    "    # Measure distance for each model\n",
    "    # TODO: normalize the distance per items in the list\n",
    "    # the output is a list, so sum of all items in the list can be not good\n",
    "    group_traits = TRAITS_DICT[group]\n",
    "    filter_cond = (df['trait'].isin(group_traits)) & (df['model'] == model_name)\n",
    "    filtered_df = df[filter_cond]\n",
    "    \n",
    "    trait_pairs = list(combinations(group_traits, 2))\n",
    "    distances = []\n",
    "    for pair in trait_pairs:\n",
    "        predictions = filtered_df[filtered_df['trait'].isin(pair)]['prediction']\n",
    "\n",
    "        if len(predictions) == 2:\n",
    "            prediction1, prediction2 = predictions\n",
    "            difference = abs(prediction1 - prediction2)\n",
    "\n",
    "            \n",
    "            acceptable_distance = min(prediction1, prediction2) * ((1-d1) if group == 1 else (1-d2))\n",
    "            \n",
    "            # if distance is within acceptable level, distance is 0\n",
    "            if difference <= acceptable_distance:\n",
    "                normalized_distance = 0\n",
    "            else:\n",
    "                min_possible_difference = acceptable_distance\n",
    "                max_possible_difference = 1  # since predictions lie in the range 0 to 1\n",
    "                normalized_distance = (difference - min_possible_difference) / (max_possible_difference - min_possible_difference)\n",
    "                \n",
    "            distances.append(normalized_distance)\n",
    "    return distances\n",
    "\n",
    "def measure_distance_for_model_centroid(df, group, model_name, radius):\n",
    "    # Measure distance for each model\n",
    "    # TODO: normalize the distance per items in the list\n",
    "    # the output is a list, so sum of all items in the list can be not good\n",
    "    group_traits = TRAITS_DICT[group]\n",
    "    filter_cond = (df['trait'].isin(group_traits)) & (df['model'] == model_name)\n",
    "    filtered_df = df[filter_cond]\n",
    "    variable_list = filtered_df.prediction.tolist()\n",
    "    \n",
    "    # Get the center point\n",
    "    center_point = np.mean(variable_list)    \n",
    "  \n",
    "    # Calculate distances from each point to boundary of circle\n",
    "    dist_to_center = np.abs(variable_list - center_point)\n",
    "    dist_to_boundary = np.where(dist_to_center <= radius, 0, dist_to_center - radius)\n",
    "        \n",
    "    return np.sum(dist_to_boundary)  \n",
    "\n",
    "\n",
    "def calculate_distance_between_groups(df, group1, group2, model_name):\n",
    "    group1_traits = TRAITS_DICT[group1]\n",
    "    group2_traits = TRAITS_DICT[group2]\n",
    "\n",
    "    group1_filtered = df[(df['trait'].isin(group1_traits)) & (df['model'] == model_name)]\n",
    "    group2_filtered = df[(df['trait'].isin(group2_traits)) & (df['model'] == model_name)]\n",
    "    \n",
    "    distances = cdist([[x] for x in group1_filtered['prediction'].tolist()], [[x] for x in group2_filtered['prediction'].tolist()])\n",
    "    avg_distances_group1_to_group2 = distances.mean(axis=1)\n",
    "    \n",
    "    return sum(avg_distances_group1_to_group2)\n",
    "\n",
    "def rank_models(df, model_names, d1=D1, d2=D2, w_inter=W_INTER):\n",
    "    model_scores = []\n",
    "\n",
    "    for model_name in model_names:\n",
    "\n",
    "        group1_distances_model = measure_distance_for_model_centroid(df, 1, model_name, d1)\n",
    "        group4_distances_model = measure_distance_for_model_centroid(df, 4, model_name, d2)\n",
    "        \n",
    "        distances_group1_to_group4 = calculate_distance_between_groups(df, 1, 4, model_name)\n",
    "\n",
    "        ######################## NOTE #####################        \n",
    "        # The smaller the intra_group_sum and the higher the inter_group_sum, the better the model.\n",
    "        #print(intra_group_sum, inter_group_sum)\n",
    "        score = -group1_distances_model -group4_distances_model + w_inter*distances_group1_to_group4 \n",
    "        \n",
    "        model_scores.append((model_name, score))\n",
    "\n",
    "    # Sort the models by their scores in descending order\n",
    "    model_scores.sort(key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    return model_scores\n",
    "\n",
    "def get_best_models(df, d1=D1, d2=D2, w_inter=W_INTER):\n",
    "    results = []\n",
    "    \n",
    "    temp_df = df.groupby('meta_col_0')[['meta_col_0', 'prediction', 'label', 'model', 'trait']].apply(lambda x: x.loc[x['model'] == rank_models(x, MODELS_LIST, d1, d2, w_inter)[0][0]])\n",
    "    result_df = temp_df.reset_index(drop=True)\n",
    "    return result_df\n",
    "\n",
    "def calculate_score(best_df):\n",
    "    # calculate the score \n",
    "    unique_traits = best_df['trait'].unique()\n",
    "    traits_to_idx_map = {trait: idx for idx, trait in enumerate(unique_traits)}\n",
    "    num_classes = len(unique_traits)\n",
    "\n",
    "    # Convert df to list of dictionaries\n",
    "    df_dict = best_df.to_dict('records')\n",
    "\n",
    "    result_prediction, result_label = {}, {}\n",
    "    \n",
    "    for rec in df_dict:\n",
    "        key = (rec['meta_col_0'], rec['model'])\n",
    "        trait_position = traits_to_idx_map[rec['trait']]\n",
    "        \n",
    "        if key not in result_prediction:\n",
    "            result_prediction[key] = np.zeros(num_classes)\n",
    "            result_label[key] = np.zeros(num_classes)\n",
    "\n",
    "        result_prediction[key][trait_position] = rec['prediction']\n",
    "        result_label[key][trait_position] = rec['label'] if 'label' in best_df.columns else 0\n",
    "            \n",
    "    prediction_arrays = np.array(list(result_prediction.values()))\n",
    "    label_arrays = np.array(list(result_label.values()))\n",
    "\n",
    "    result, _data = mean_pearsons(prediction_arrays, label_arrays)    \n",
    "    return result, _data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "80ad127e-4811-498f-8709-740b54ea6ee4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>meta_col_0</th>\n",
       "      <th>prediction</th>\n",
       "      <th>label</th>\n",
       "      <th>model</th>\n",
       "      <th>trait</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0.377227</td>\n",
       "      <td>0.3529</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>aggressive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>0.357250</td>\n",
       "      <td>0.3382</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>arrogant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.502012</td>\n",
       "      <td>0.4902</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>dominant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0.555200</td>\n",
       "      <td>0.6569</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>assertiv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>0.505092</td>\n",
       "      <td>0.5147</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>risk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9</td>\n",
       "      <td>0.651515</td>\n",
       "      <td>0.6520</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>independent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9</td>\n",
       "      <td>0.631275</td>\n",
       "      <td>0.6520</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>leader_like</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9</td>\n",
       "      <td>0.677542</td>\n",
       "      <td>0.7451</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>confident</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0.616529</td>\n",
       "      <td>0.4804</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>enthusiastic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>0.574478</td>\n",
       "      <td>0.5539</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>friendly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9</td>\n",
       "      <td>0.568677</td>\n",
       "      <td>0.5686</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>collaborative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9</td>\n",
       "      <td>0.695462</td>\n",
       "      <td>0.6275</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>sincere</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>9</td>\n",
       "      <td>0.642448</td>\n",
       "      <td>0.6373</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>likeable</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>9</td>\n",
       "      <td>0.585921</td>\n",
       "      <td>0.5784</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>kind</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>9</td>\n",
       "      <td>0.577699</td>\n",
       "      <td>0.5294</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>warm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>9</td>\n",
       "      <td>0.663584</td>\n",
       "      <td>0.6667</td>\n",
       "      <td>CNN-ATTN_[vit-fer]_[0.0005_32]</td>\n",
       "      <td>good_natured</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    meta_col_0  prediction   label                           model  \\\n",
       "0            9    0.377227  0.3529  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "1            9    0.357250  0.3382  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "2            9    0.502012  0.4902  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "3            9    0.555200  0.6569  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "4            9    0.505092  0.5147  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "5            9    0.651515  0.6520  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "6            9    0.631275  0.6520  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "7            9    0.677542  0.7451  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "8            9    0.616529  0.4804  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "9            9    0.574478  0.5539  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "10           9    0.568677  0.5686  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "11           9    0.695462  0.6275  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "12           9    0.642448  0.6373  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "13           9    0.585921  0.5784  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "14           9    0.577699  0.5294  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "15           9    0.663584  0.6667  CNN-ATTN_[vit-fer]_[0.0005_32]   \n",
       "\n",
       "            trait  \n",
       "0      aggressive  \n",
       "1        arrogant  \n",
       "2        dominant  \n",
       "3        assertiv  \n",
       "4            risk  \n",
       "5     independent  \n",
       "6     leader_like  \n",
       "7       confident  \n",
       "8    enthusiastic  \n",
       "9        friendly  \n",
       "10  collaborative  \n",
       "11        sincere  \n",
       "12       likeable  \n",
       "13           kind  \n",
       "14           warm  \n",
       "15   good_natured  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df = get_best_models(df_trait)\n",
    "result_df.to_csv(f'test_sum_devel_{D1:.2f}_{D2:.2f}_{W_INTER:.2f}.csv', index=False)\n",
    "result_df.head(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "328d6163-5534-43a9-9f67-88993d9b680e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.49820454332930497,\n",
       " array([0.53447626, 0.58122181, 0.483436  , 0.22132553, 0.46534533,\n",
       "        0.47060199, 0.51041311, 0.43339587, 0.2533439 , 0.60332572,\n",
       "        0.56650368, 0.47097401, 0.6363766 , 0.55132877, 0.57362998,\n",
       "        0.61557412]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result, _data = calculate_score(result_df)\n",
    "result_df.to_csv(f'test_sum_devel_{D1:.2f}_{D2:.2f}_{W_INTER:.2f}_{np.round(result,4)}.csv', index=False)\n",
    "result, _data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "KIELI2",
   "language": "python",
   "name": "kieli2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}