Multi-Task_for_3Emotional_Classes.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## **Classification of Personality Influenced Emotions using subjective modeling approach  **"
      ],
      "metadata": {
        "id": "tSDmOV2Za5ka"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "metadata": {
        "id": "GHjlIbhRWhAY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Install** **dependencies**"
      ],
      "metadata": {
        "id": "aIAEObNlaq6M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow_text\n",
        "!pip install tensorflow-addons"
      ],
      "metadata": {
        "id": "DsBuDhWBfs-B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Install required packages**"
      ],
      "metadata": {
        "id": "PmccZWEra0x9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix,classification_report\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "#import tensorflow_addons as tfa\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "from tensorflow import keras\n",
        "from random import shuffle\n",
        "import random\n",
        "import re\n",
        "import string"
      ],
      "metadata": {
        "id": "zmbJkULABb64"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Ulitity functions**"
      ],
      "metadata": {
        "id": "_t84WajxcE_s"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# utility functions\n",
        "def data_loader(PATH, EMO_OR_SENTI):\n",
        "    characters = ['Chandler','Joey','Monica','Phoebe','Rachel','Ross']\n",
        "    df = pd.read_csv(PATH, sep='\\t').fillna('Nan')\n",
        "    for actor in characters:\n",
        "        # get rows from the selected character\n",
        "        single_df = df[df['Speaker_1'].apply(lambda x: actor in x)]\n",
        "        # selected emotion to increase the samples of that particulat emotions\n",
        "        if actor == 'Chandler':\n",
        "            increase_emo = ['anger', 'disgust', 'fear']\n",
        "        else:\n",
        "            increase_emo = ['anger', 'disgust', 'fear', 'surprise', 'sadness', 'joy']\n",
        "        for emo in increase_emo:\n",
        "            selected_df = single_df[single_df['Emotion_3'].apply(lambda x: emo in x)]\n",
        "            percent = int(len(selected_df)*25/100)\n",
        "            random_int = random.sample(range(0, len(selected_df)), percent)\n",
        "            for idx in random_int:\n",
        "                row = selected_df.iloc[idx]\n",
        "                sp1 = row['Speaker_1']\n",
        "                sp2 = row['Speaker_2']\n",
        "                per = row['Personality']\n",
        "                # utterence\n",
        "                utt_1 = row['Utterance_1']\n",
        "                utt_2 = row['Utterance_2']\n",
        "                utt_3 = row['Utterance_3']\n",
        "                # emotions\n",
        "                emo_1 = row['Emotion_1']\n",
        "                emo_2 = row['Emotion_2']\n",
        "                emo_3 = row['Emotion_3']\n",
        "                # sentiments\n",
        "                sent_1 = row['Sentiment_1']\n",
        "                sent_2 = row['Sentiment_2']\n",
        "                sent_3 = row['Sentiment_3']\n",
        "                # new row\n",
        "                row1 = {}\n",
        "                row1['Speaker_1'] = sp1\n",
        "                row1['Speaker_2'] = sp2\n",
        "                row1['Personality'] = per\n",
        "                row1['Utterance_1'] = utt_3\n",
        "                row1['Utterance_2'] = utt_2\n",
        "                row1['Utterance_3'] = utt_1\n",
        "                row1['Emotion_1'] = emo_3\n",
        "                row1['Emotion_2'] = emo_2\n",
        "                row1['Emotion_3'] = emo_1\n",
        "                row1['Sentiment_1'] = sent_3\n",
        "                row1['Sentiment_2'] = sent_2\n",
        "                row1['Sentiment_3'] = sent_1\n",
        "                single_df = single_df.append(row1,ignore_index=True)\n",
        "        emo_vad = []\n",
        "        if EMO_OR_SENTI == 'Emotions':\n",
        "            ini_labels = single_df['Emotion_1'].values\n",
        "            for y in ini_labels:\n",
        "                if y == 'anger':\n",
        "                    emo_vad.append([-0.51, 0.59, 0.25])\n",
        "                elif y == 'disgust':\n",
        "                    emo_vad.append([-0.60, 0.35, 0.11])\n",
        "                elif y == 'fear':\n",
        "                    emo_vad.append([-0.62, 0.82, -0.43])\n",
        "                elif y == 'joy':\n",
        "                    emo_vad.append([0.81, 0.51, 0.46])\n",
        "                elif y == 'neutral':\n",
        "                    emo_vad.append([0.00, 0.00, 0.00])\n",
        "                elif y == 'sadness':\n",
        "                    emo_vad.append([-0.63, -0.27, -0.33])\n",
        "                elif y == 'surprise':\n",
        "                    emo_vad.append([0.40, 0.67, -0.13])\n",
        "            final_labels = single_df['Emotion_3'].values\n",
        "            one_hot = pd.get_dummies(final_labels)\n",
        "        elif EMO_OR_SENTI == 'Sentiments':\n",
        "            ini_labels = single_df['Emotion_1'].values\n",
        "            for y in ini_labels:\n",
        "                if y == 'anger':\n",
        "                    emo_vad.append([-0.51, 0.59, 0.25])\n",
        "                elif y == 'disgust':\n",
        "                    emo_vad.append([-0.60, 0.35, 0.11])\n",
        "                elif y == 'fear':\n",
        "                    emo_vad.append([-0.62, 0.82, -0.43])\n",
        "                elif y == 'joy':\n",
        "                    emo_vad.append([0.81, 0.51, 0.46])\n",
        "                elif y == 'neutral':\n",
        "                    emo_vad.append([0.00, 0.00, 0.00])\n",
        "                elif y == 'sadness':\n",
        "                    emo_vad.append([-0.63, -0.27, -0.33])\n",
        "                elif y == 'surprise':\n",
        "                    emo_vad.append([0.40, 0.67, -0.13])\n",
        "            final_labels = single_df['Sentiment_1'].values\n",
        "            one_hot = pd.get_dummies(final_labels)\n",
        "        Utterance_1   = single_df['Utterance_1'].values\n",
        "        Utterance_2   = single_df['Utterance_2'].values\n",
        "        Utterance_3   = single_df['Utterance_3'].values\n",
        "        personality_traits = single_df['Personality'].values\n",
        "        personality = personality_traits[0]\n",
        "        personality = personality.replace('[','').replace(']','')\n",
        "        personality = personality.split(',')\n",
        "        VAD = []\n",
        "        if actor == 'Chandler':\n",
        "            O = float(personality[0])\n",
        "            C = float(personality[1])\n",
        "            E = float(personality[2])\n",
        "            A = float(personality[3])\n",
        "            N = float(personality[4])\n",
        "            P_v = (0.21*E) + (0.59*A) + (0.19*N)\n",
        "            P_a = (0.15*O) + (0.30*A) - (0.57*N)\n",
        "            P_d = (0.25*O) + (0.17*C) + (0.60*E) - (0.32*A)\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_v)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_a)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_d)))\n",
        "        elif actor == 'Joey':\n",
        "            O = float(personality[0])\n",
        "            C = float(personality[1])\n",
        "            E = float(personality[2])\n",
        "            A = float(personality[3])\n",
        "            N = float(personality[4])\n",
        "            P_v = (0.21*E) + (0.59*A) + (0.19*N)\n",
        "            P_a = (0.15*O) + (0.30*A) - (0.57*N)\n",
        "            P_d = (0.25*O) + (0.17*C) + (0.60*E) - (0.32*A)\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_v)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_a)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_d)))\n",
        "        elif actor == 'Monica':\n",
        "            O = float(personality[0])\n",
        "            C = float(personality[1])\n",
        "            E = float(personality[2])\n",
        "            A = float(personality[3])\n",
        "            N = float(personality[4])\n",
        "            P_v = (0.21*E) + (0.59*A) + (0.19*N)\n",
        "            P_a = (0.15*O) + (0.30*A) - (0.57*N)\n",
        "            P_d = (0.25*O) + (0.17*C) + (0.60*E) - (0.32*A)\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_v)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_a)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_d)))\n",
        "        elif actor == 'Phoebe':\n",
        "            O = float(personality[0])\n",
        "            C = float(personality[1])\n",
        "            E = float(personality[2])\n",
        "            A = float(personality[3])\n",
        "            N = float(personality[4])\n",
        "            P_v = (0.21*E) + (0.59*A) + (0.19*N)\n",
        "            P_a = (0.15*O) + (0.30*A) - (0.57*N)\n",
        "            P_d = (0.25*O) + (0.17*C) + (0.60*E) - (0.32*A)\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_v)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_a)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_d)))\n",
        "        elif actor == 'Rachel':\n",
        "            O = float(personality[0])\n",
        "            C = float(personality[1])\n",
        "            E = float(personality[2])\n",
        "            A = float(personality[3])\n",
        "            N = float(personality[4])\n",
        "            P_v = (0.21*E) + (0.59*A) + (0.19*N)\n",
        "            P_a = (0.15*O) + (0.30*A) - (0.57*N)\n",
        "            P_d = (0.25*O) + (0.17*C) + (0.60*E) - (0.32*A)\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_v)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_a)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_d)))\n",
        "        elif actor == 'Ross':\n",
        "            O = float(personality[0])\n",
        "            C = float(personality[1])\n",
        "            E = float(personality[2])\n",
        "            A = float(personality[3])\n",
        "            N = float(personality[4])\n",
        "            P_v = (0.21*E) + (0.59*A) + (0.19*N)\n",
        "            P_a = (0.15*O) + (0.30*A) - (0.57*N)\n",
        "            P_d = (0.25*O) + (0.17*C) + (0.60*E) - (0.32*A)\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_v)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_a)))\n",
        "            VAD.append(float(\"{0:.4f}\".format(P_d)))\n",
        "\n",
        "        Utterance_1 = np.reshape(Utterance_1,[len(Utterance_1),1])\n",
        "        Utterance_2 = np.reshape(Utterance_2,[len(Utterance_2),1])\n",
        "        Utterance_3 = np.reshape(Utterance_3,[len(Utterance_3),1])\n",
        "        VAD = np.reshape(VAD,[1, len(VAD)])\n",
        "        vad_personalities = np.repeat(VAD,len(personality_traits), axis=0)\n",
        "        if actor == 'Chandler':\n",
        "            Chandler = np.concatenate((Utterance_1,Utterance_2,Utterance_3,vad_personalities,emo_vad,one_hot), axis=1)\n",
        "        elif actor == 'Joey':\n",
        "            Joey = np.concatenate((Utterance_1,Utterance_2,Utterance_3,vad_personalities,emo_vad,one_hot), axis=1)\n",
        "        elif actor == 'Monica':\n",
        "            Monica = np.concatenate((Utterance_1,Utterance_2,Utterance_3,vad_personalities,emo_vad,one_hot), axis=1)\n",
        "        elif actor == 'Phoebe':\n",
        "            Phoebe = np.concatenate((Utterance_1,Utterance_2,Utterance_3,vad_personalities,emo_vad,one_hot), axis=1)\n",
        "        elif actor == 'Rachel':\n",
        "            Rachel = np.concatenate((Utterance_1,Utterance_2,Utterance_3,vad_personalities,emo_vad,one_hot), axis=1)\n",
        "        elif actor == 'Ross':\n",
        "            Ross = np.concatenate((Utterance_1,Utterance_2,Utterance_3,vad_personalities,emo_vad,one_hot), axis=1)\n",
        "\n",
        "    return Chandler,Joey,Monica,Phoebe,Rachel,Ross\n",
        "\n",
        "# get minimum samples\n",
        "def min_samples(data):\n",
        "  length = []\n",
        "  for i in range(len(data)):\n",
        "    samples = len(data[i])\n",
        "    length.append(samples)\n",
        "  return min(length)\n",
        "\n",
        "# get 1D vector\n",
        "def reduce_dimension(labels,logits):\n",
        "    y_true = []\n",
        "    y_pred = []\n",
        "    for i in range(len(labels)):\n",
        "        batch_labels = labels[i]\n",
        "        batch_logits = logits[i]\n",
        "        for j in range(len(batch_labels)):\n",
        "            list_labels = batch_labels[j]\n",
        "            list_logits = batch_logits[j]\n",
        "            y_true.append(np.argmax(list_labels))\n",
        "            y_pred.append(np.argmax(list_logits))\n",
        "    return y_true,y_pred\n",
        "\n",
        "# get normalize text\n",
        "def normalize_text(array):\n",
        "    norm_text = []\n",
        "    for text in array:\n",
        "      text = text.lower()\n",
        "      # Remove punctuations\n",
        "      exclude = set(string.punctuation)\n",
        "      text = \"\".join(ch for ch in text if ch not in exclude)\n",
        "      # Remove articles\n",
        "      regex = re.compile(r\"\\b(a|an|the)\\b\", re.UNICODE)\n",
        "      text = re.sub(regex, \" \", text)\n",
        "      # Remove extra white space\n",
        "      text = \" \".join(text.split())\n",
        "      norm_text.append(text)\n",
        "    return norm_text"
      ],
      "metadata": {
        "id": "690VoRH2OosX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Download and split data into train, val , ans test set**"
      ],
      "metadata": {
        "id": "EUL_Dv11cLyq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# data file path\n",
        "PATH = '/gdrive/My Drive/emotion_personality/New_PELD_Exp/Dyadic_PELD.tsv'\n",
        "\n",
        "# select between 0:sentiments and 1:emotions\n",
        "select_list = ['Emotions', 'Sentiments']\n",
        "EMO_OR_SENTI = select_list[1]\n",
        "Chandler,Joey,Monica,Phoebe,Rachel,Ross = data_loader(PATH,EMO_OR_SENTI)\n",
        " \n",
        "# split each character samples into train and test sets\n",
        "chandler_train, chandler_test = train_test_split(Chandler, random_state=42, test_size=0.1)\n",
        "joey_train, joey_test = train_test_split(Joey, random_state=42, test_size=0.1)\n",
        "monica_train, monica_test = train_test_split(Monica, random_state=42, test_size=0.1)\n",
        "phoebe_train, phoebe_test = train_test_split(Phoebe, random_state=42, test_size=0.1)\n",
        "rachel_train, rachel_test = train_test_split(Rachel, random_state=42, test_size=0.1)\n",
        "ross_train, ross_test = train_test_split(Ross, random_state=42, test_size=0.1)\n",
        "\n",
        "# split each character train samples into train and val\n",
        "chandler_train, chandler_val = train_test_split(chandler_train, random_state=42, test_size=0.1)\n",
        "joey_train, joey_val = train_test_split(joey_train, random_state=42, test_size=0.1)\n",
        "monica_train, monica_val = train_test_split(monica_train, random_state=42, test_size=0.1)\n",
        "phoebe_train, phoebe_val = train_test_split(phoebe_train, random_state=42, test_size=0.1)\n",
        "rachel_train, rachel_val = train_test_split(rachel_train, random_state=42, test_size=0.1)\n",
        "ross_train, ross_val = train_test_split(ross_train, random_state=42, test_size=0.1)\n"
      ],
      "metadata": {
        "id": "kM0YttT3CMzF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Create BERT model**"
      ],
      "metadata": {
        "id": "_CiNH_o6cUVm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "preprocessing_path = '/gdrive/My Drive/emotion_personality/New_PELD_Exp/bert_en_uncased_preprocess_3'\n",
        "model_path = '/gdrive/My Drive/emotion_personality/New_PELD_Exp/bert_en_uncased_L-12_H-768_A-12_4'\n",
        "# inputs shapes\n",
        "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
        "preprocessing_layer = hub.KerasLayer(hub.load(preprocessing_path))\n",
        "encoder_inputs = preprocessing_layer(text_input)\n",
        "encoder = hub.KerasLayer(hub.load(model_path),trainable=True, name='BERT_encoder' )\n",
        "outputs = encoder(encoder_inputs)\n",
        "net = outputs['pooled_output']\n",
        "bert_encoder = tf.keras.Model(text_input, net)"
      ],
      "metadata": {
        "id": "CKkLIi3uCRl9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Subjective models**"
      ],
      "metadata": {
        "id": "G41NhDCNcg6G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# subjective models\n",
        "class SubjectiveModel(tf.keras.Model):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "    initializer = tf.keras.initializers.GlorotUniform(seed=42)\n",
        "    self.linear1 = tf.keras.layers.Dense(3, activation=tf.nn.softmax, kernel_initializer=initializer)\n",
        "    self.dropout = tf.keras.layers.Dropout(0.5)\n",
        "  def call(self, inputs, training=False):\n",
        "    out = self.dropout(inputs)\n",
        "    out = self.linear1(out, training=training)\n",
        "    return out\n",
        "\n",
        "# personality influence emotions\n",
        "class PersonalityEmotions(tf.keras.Model):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "    initializer = tf.keras.initializers.GlorotUniform(seed=42)\n",
        "    self.linear1 = tf.keras.layers.Dense(3, activation=None, kernel_initializer=initializer)\n",
        "    self.linear2 = tf.keras.layers.Dense(3, activation=None, kernel_initializer=initializer)\n",
        "    self.linear3 = tf.keras.layers.Dense(3, activation=None, kernel_initializer=initializer)\n",
        "  def call(self, ini_emo, personality_vad, text_embd, training=False):\n",
        "    # linear layer for the personality influenced VAD\n",
        "    personality_influence = self.linear1(personality_vad,training=training)\n",
        "    # linear layer for all the previous text encoding\n",
        "    utter_emotions = self.linear2(text_embd,training=training)\n",
        "    # get the target emotions\n",
        "    target_emotions = ini_emo + utter_emotions * personality_influence\n",
        "    # Tanh activation function\n",
        "    target_emotions = tf.keras.activations.tanh(target_emotions)\n",
        "    # linear layer for the target emotions\n",
        "    target_emotions = self.linear3(target_emotions)\n",
        "    # concatenate layer\n",
        "    concatenate_matrix = tf.keras.layers.concatenate([personality_influence,target_emotions,text_embd])\n",
        "    return concatenate_matrix"
      ],
      "metadata": {
        "id": "B0pFNAiFCbEA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Initialize all the subjective models, optimizer, loss function, and learning rate schedular** "
      ],
      "metadata": {
        "id": "cDn5d-accn1I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# character 1\n",
        "model_1 = SubjectiveModel()\n",
        "personality_vad_1 = PersonalityEmotions()\n",
        "# character 2\n",
        "model_2 = SubjectiveModel()\n",
        "personality_vad_2 = PersonalityEmotions()\n",
        "# character 3\n",
        "model_3 = SubjectiveModel()\n",
        "personality_vad_3 = PersonalityEmotions()\n",
        "# character 4\n",
        "model_4 = SubjectiveModel()\n",
        "personality_vad_4 = PersonalityEmotions()\n",
        "# character 5\n",
        "model_5 = SubjectiveModel()\n",
        "personality_vad_5 = PersonalityEmotions()\n",
        "# character 6\n",
        "model_6 = SubjectiveModel()\n",
        "personality_vad_6 = PersonalityEmotions()\n",
        "\n",
        "# schedular\n",
        "lr_schedule = tf.keras.optimizers.schedules.InverseTimeDecay(\n",
        "  1e-4,\n",
        "  decay_steps=10000,\n",
        "  decay_rate=0.01,\n",
        "  staircase=False)\n",
        "\n",
        "# optimizer\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)\n",
        "# loss function\n",
        "loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=False)"
      ],
      "metadata": {
        "id": "PouNfO9FCqMP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model hyperparameters**"
      ],
      "metadata": {
        "id": "fCx4Zkpgc2A3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#The BERT authors recommend fine-tuning for 4 epochs over the following hyperparameter options: \n",
        "# batch sizes: 8, 16, 32, 64, 128. learning rates: 3e-4, 1e-4, 5e-5, 3e-5\n",
        "epochs = 10\n",
        "batch_size = 8"
      ],
      "metadata": {
        "id": "H7gEFOf7CrD5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Model training, validation, and testing**"
      ],
      "metadata": {
        "id": "oSdUVU9Ic9Cg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(epochs):\n",
        "    print('*' * 50)\n",
        "    print(\"\\nStart of epoch %d\" % (epoch,))\n",
        "    print('*' * 50)\n",
        "    # shuffle all the training samples for each character\n",
        "    shuffle(chandler_train)\n",
        "    shuffle(joey_train)\n",
        "    shuffle(monica_train)\n",
        "    shuffle(phoebe_train)\n",
        "    shuffle(rachel_train)\n",
        "    shuffle(ross_train)\n",
        "    # concatenate all training samples\n",
        "    train_data = []\n",
        "    train_data.append(chandler_train)\n",
        "    train_data.append(joey_train)\n",
        "    train_data.append(monica_train)\n",
        "    train_data.append(phoebe_train)\n",
        "    train_data.append(rachel_train)\n",
        "    train_data.append(ross_train)\n",
        "    minimum_samples = min_samples(train_data)\n",
        "    max_range = np.multiply(int(minimum_samples/batch_size),batch_size)\n",
        "    chandler_loss = []\n",
        "    joey_loss = []\n",
        "    monica_loss = []\n",
        "    phoebe_loss = []\n",
        "    rachel_loss = []\n",
        "    ross_loss = []\n",
        "    for i in range(0,max_range,batch_size):\n",
        "        for j in range(len(train_data)):\n",
        "            X_train = train_data[j]\n",
        "            X_utter1 = X_train[:,0]\n",
        "            X_utter2 = X_train[:,1]\n",
        "            X_p_vad = X_train[:,3:6]\n",
        "            X_emo_vad = X_train[:,6:9]\n",
        "            X_emo_labels = X_train[:,-3:]\n",
        "            # select batch\n",
        "            utter1 = X_utter1[i:i+batch_size]\n",
        "            utter2 = X_utter2[i:i+batch_size]\n",
        "            utter1 = normalize_text(utter1)\n",
        "            utter2 = normalize_text(utter2)\n",
        "            p_vad = X_p_vad[i:i+batch_size].astype('float32')\n",
        "            cat_labels = X_emo_labels[i:i+batch_size].astype('float32')\n",
        "            vad_labels = X_emo_vad[i:i+batch_size].astype('float32')\n",
        "            # convert into tensors\n",
        "            utter1 = tf.convert_to_tensor(utter1)\n",
        "            utter2 = tf.convert_to_tensor(utter2)\n",
        "            p_vad = tf.convert_to_tensor(p_vad)\n",
        "            cat_labels = tf.convert_to_tensor(cat_labels)\n",
        "            vad_labels = tf.convert_to_tensor(vad_labels)\n",
        "            with tf.GradientTape(persistent=True) as tape:\n",
        "                utter1_sementics = bert_encoder(utter1)\n",
        "                utter2_sementics = bert_encoder(utter2)\n",
        "                dialogue_sementics = tf.concat([utter1_sementics, utter2_sementics],1)\n",
        "                if j == 0:\n",
        "                    # personality vad\n",
        "                    per_vad = personality_vad_1(vad_labels,p_vad, dialogue_sementics,training=True)\n",
        "                    # logits\n",
        "                    logits = model_1(per_vad,training=True)\n",
        "                    # loss function  \n",
        "                    loss_value = loss_fn(cat_labels, logits)\n",
        "                    chandler_loss.append(loss_value)\n",
        "                    trainables = model_1.trainable_weights  + personality_vad_1.trainable_weights + bert_encoder.trainable_weights    \n",
        "                elif j == 1:\n",
        "                    # personality vad\n",
        "                    per_vad = personality_vad_2(vad_labels,p_vad, dialogue_sementics,training=True)\n",
        "                    # logits\n",
        "                    logits = model_2(per_vad,training=True)\n",
        "                    # loss function  \n",
        "                    loss_value = loss_fn(cat_labels, logits)\n",
        "                    joey_loss.append(loss_value)\n",
        "                    trainables = model_2.trainable_weights  + personality_vad_2.trainable_weights+ bert_encoder.trainable_weights\n",
        "                elif j == 2:\n",
        "                    # personality vad\n",
        "                    per_vad = personality_vad_3(vad_labels,p_vad, dialogue_sementics,training=True)\n",
        "                    # logits\n",
        "                    logits = model_3(per_vad,training=True)\n",
        "                    # loss function  \n",
        "                    loss_value = loss_fn(cat_labels, logits)\n",
        "                    monica_loss.append(loss_value)\n",
        "                    trainables = model_3.trainable_weights  + personality_vad_3.trainable_weights+ bert_encoder.trainable_weights\n",
        "                elif j == 3:\n",
        "                    # personality vad\n",
        "                    per_vad = personality_vad_4(vad_labels,p_vad,dialogue_sementics,training=True)\n",
        "                    # logits\n",
        "                    logits = model_4(per_vad,training=True)\n",
        "                    # loss function \n",
        "                    loss_value = loss_fn(cat_labels, logits)\n",
        "                    phoebe_loss.append(loss_value)\n",
        "                    trainables = model_4.trainable_weights  + personality_vad_4.trainable_weights+ bert_encoder.trainable_weights\n",
        "                elif j == 4:\n",
        "                    # personality vad\n",
        "                    per_vad = personality_vad_5(vad_labels,p_vad,dialogue_sementics,training=True)\n",
        "                    # logits\n",
        "                    logits = model_5(per_vad,training=True)\n",
        "                    # loss function  \n",
        "                    loss_value = loss_fn(cat_labels, logits)\n",
        "                    rachel_loss.append(loss_value)\n",
        "                    trainables = model_5.trainable_weights  + personality_vad_5.trainable_weights  + bert_encoder.trainable_weights\n",
        "                elif j == 5:\n",
        "                    # personality vad\n",
        "                    per_vad = personality_vad_6(vad_labels,p_vad,dialogue_sementics,training=True)\n",
        "                    # logits\n",
        "                    logits = model_6(per_vad,training=True)\n",
        "                    # loss function  \n",
        "                    loss_value = loss_fn(cat_labels, logits)\n",
        "                    ross_loss.append(loss_value)\n",
        "                    trainables = model_6.trainable_weights  + personality_vad_6.trainable_weights+ bert_encoder.trainable_weights\n",
        "            # Use the gradient tape to automatically retrieve\n",
        "            # the gradients of the trainable variables with respect to the loss.\n",
        "            grads = tape.gradient(loss_value, trainables)\n",
        "            # Run one step of gradient descent by updating\n",
        "            # the value of the variables to minimize the loss.\n",
        "            optimizer.apply_gradients(zip(grads, trainables))\n",
        "    print('-' * 20)\n",
        "    print('Training Loss')\n",
        "    print('-' * 20)\n",
        "    print('Chandler Loss:', np.mean(chandler_loss))\n",
        "    print('Joey Loss:', np.mean(joey_loss))\n",
        "    print('Monica Loss:', np.mean(monica_loss))\n",
        "    print('Phoebe Loss:', np.mean(phoebe_loss))\n",
        "    print('Rachel Loss:', np.mean(rachel_loss))\n",
        "    print('Ross Loss:', np.mean(ross_loss))\n",
        "    # validation\n",
        "    val_data = []\n",
        "    val_data.append(chandler_val)\n",
        "    val_data.append(joey_val)\n",
        "    val_data.append(monica_val)\n",
        "    val_data.append(phoebe_val)\n",
        "    val_data.append(rachel_val)\n",
        "    val_data.append(ross_val)\n",
        "    chandler_loss_val = []\n",
        "    joey_loss_val = []\n",
        "    monica_loss_val = []\n",
        "    phoebe_loss_val = []\n",
        "    rachel_loss_val = []\n",
        "    ross_loss_val = []\n",
        "    minimum_samples = min_samples(val_data)\n",
        "    max_range = np.multiply(int(minimum_samples/batch_size),batch_size)\n",
        "    for i in range(0,max_range,batch_size):\n",
        "        for j in range(len(val_data)):\n",
        "            X_val = val_data[j]\n",
        "            X_utter1 = X_val[:,0]\n",
        "            X_utter2 = X_val[:,1]\n",
        "            X_p_vad = X_val[:,3:6]\n",
        "            X_emo_vad = X_train[:,6:9]\n",
        "            X_emo_labels = X_val[:,-3:]\n",
        "            # select batch\n",
        "            utter1 = X_utter1[i:i+batch_size]\n",
        "            utter2 = X_utter2[i:i+batch_size]\n",
        "            utter1 = normalize_text(utter1)\n",
        "            utter2 = normalize_text(utter2)\n",
        "            p_vad = X_p_vad[i:i+batch_size].astype('float32')\n",
        "            cat_labels = X_emo_labels[i:i+batch_size].astype('float32')\n",
        "            vad_labels = X_emo_vad[i:i+batch_size].astype('float32')\n",
        "            # convert into tensors\n",
        "            utter1 = tf.convert_to_tensor(utter1)\n",
        "            utter2 = tf.convert_to_tensor(utter2)\n",
        "            p_vad = tf.convert_to_tensor(p_vad)\n",
        "            cat_labels = tf.convert_to_tensor(cat_labels)\n",
        "            vad_labels = tf.convert_to_tensor(vad_labels)\n",
        "            # bert encoder\n",
        "            utter1_sementics = bert_encoder(utter1)\n",
        "            utter2_sementics = bert_encoder(utter2)\n",
        "            dialogue_sementics = tf.concat([utter1_sementics, utter2_sementics],1)\n",
        "            if j == 0:\n",
        "                # personality vad\n",
        "                per_vad = personality_vad_1(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "                # logits\n",
        "                logits = model_1(per_vad,training=False)\n",
        "                # loss function  \n",
        "                loss_value = loss_fn(cat_labels, logits)\n",
        "                chandler_loss_val.append(loss_value)\n",
        "            elif j == 1:\n",
        "                # subjective output\n",
        "                per_vad = personality_vad_2(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "                # logits\n",
        "                logits = model_2(per_vad,training=False)\n",
        "                # loss function  \n",
        "                loss_value = loss_fn(cat_labels, logits)\n",
        "                joey_loss_val.append(loss_value)\n",
        "            elif j == 2:\n",
        "                # subjective output\n",
        "                per_vad = personality_vad_3(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "                # logits\n",
        "                logits = model_3(per_vad,training=False)\n",
        "                # loss function  \n",
        "                loss_value = loss_fn(cat_labels, logits)\n",
        "                monica_loss_val.append(loss_value)\n",
        "            elif j == 3:\n",
        "                # subjective output\n",
        "                per_vad = personality_vad_4(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "                # logits\n",
        "                logits = model_4(per_vad,training=False)\n",
        "                # loss function  \n",
        "                loss_value = loss_fn(cat_labels, logits)\n",
        "                phoebe_loss_val.append(loss_value)\n",
        "            elif j == 4:\n",
        "                # subjective output\n",
        "                per_vad = personality_vad_5(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "                # logits\n",
        "                logits = model_5(per_vad,training=False)\n",
        "                # loss function  \n",
        "                loss_value = loss_fn(cat_labels, logits)\n",
        "                rachel_loss_val.append(loss_value)\n",
        "            elif j == 5:\n",
        "                # subjective output\n",
        "                per_vad = personality_vad_6(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "                # logits\n",
        "                logits = model_6(per_vad,training=False)\n",
        "                # loss function  \n",
        "                loss_value = loss_fn(cat_labels, logits)\n",
        "                ross_loss_val.append(loss_value)\n",
        "    print('-' * 20)\n",
        "    print('Validation Loss:')\n",
        "    print('-' * 20)\n",
        "    print('Chandler Loss:', np.mean(chandler_loss_val))\n",
        "    print('Joey Loss:', np.mean(joey_loss_val))\n",
        "    print('Monica Loss:', np.mean(monica_loss_val))\n",
        "    print('Phoebe Loss:', np.mean(phoebe_loss_val))\n",
        "    print('Rachel Loss:', np.mean(rachel_loss_val))\n",
        "    print('Ross Loss:', np.mean(ross_loss_val))\n",
        "    #if epoch == 0:\n",
        "    #if (epoch+1) % 5 == 0:      \n",
        "    # test the model\n",
        "    # concatenate all test samples\n",
        "    chandler_logits = []\n",
        "    chandler_labels = []\n",
        "    joey_logits = []\n",
        "    joey_labels = []\n",
        "    monica_logits = []\n",
        "    monica_labels = []\n",
        "    phoebe_logits = []\n",
        "    phoebe_labels = []\n",
        "    rachel_logits = []\n",
        "    rachel_labels = []\n",
        "    ross_logits = []\n",
        "    ross_labels = []\n",
        "    test_data = []\n",
        "    test_data.append(chandler_test)\n",
        "    test_data.append(joey_test)\n",
        "    test_data.append(monica_test)\n",
        "    test_data.append(phoebe_test)\n",
        "    test_data.append(rachel_test)\n",
        "    test_data.append(ross_test)\n",
        "    minimum_samples = min_samples(test_data)\n",
        "    max_range = np.multiply(int(minimum_samples/batch_size),batch_size)\n",
        "    for i in range(0,max_range,batch_size):\n",
        "      for j in range(len(test_data)):\n",
        "        X_test = test_data[j]\n",
        "        X_utter1 = X_test[:,0]\n",
        "        X_utter2 = X_test[:,1]\n",
        "        X_p_vad = X_test[:,3:6]\n",
        "        X_emo_vad = X_train[:,6:9]\n",
        "        X_emo_labels = X_test[:,-3:]\n",
        "        # select batch\n",
        "        utter1 = X_utter1[i:i+batch_size]\n",
        "        utter2 = X_utter2[i:i+batch_size]\n",
        "        p_vad = X_p_vad[i:i+batch_size].astype('float32')\n",
        "        cat_labels = X_emo_labels[i:i+batch_size].astype('float32')\n",
        "        vad_labels = X_emo_vad[i:i+batch_size].astype('float32')\n",
        "        # convert into tensors\n",
        "        utter1 = tf.convert_to_tensor(utter1)\n",
        "        utter2 = tf.convert_to_tensor(utter2)\n",
        "        p_vad = tf.convert_to_tensor(p_vad)\n",
        "        cat_labels = tf.convert_to_tensor(cat_labels)\n",
        "        vad_labels = tf.convert_to_tensor(vad_labels)\n",
        "        # bert encoder\n",
        "        utter1_sementics = bert_encoder(utter1)\n",
        "        utter2_sementics = bert_encoder(utter2)\n",
        "        dialogue_sementics = tf.concat([utter1_sementics, utter2_sementics],1)\n",
        "        if j == 0:\n",
        "            # subjective output\n",
        "            per_vad = personality_vad_1(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "            # logits\n",
        "            logits = model_1(per_vad,training=False)\n",
        "            chandler_logits.append(logits)\n",
        "            chandler_labels.append(cat_labels)\n",
        "        elif j == 1:\n",
        "            # subjective output\n",
        "            per_vad = personality_vad_2(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "            # logits\n",
        "            logits = model_2(per_vad,training=False)\n",
        "            joey_logits.append(logits)\n",
        "            joey_labels.append(cat_labels)\n",
        "        elif j == 2:\n",
        "            # subjective output\n",
        "            per_vad = personality_vad_3(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "            # logits\n",
        "            logits = model_3(per_vad,training=False)\n",
        "            monica_logits.append(logits)\n",
        "            monica_labels.append(cat_labels)\n",
        "        elif j == 3:\n",
        "            # subjective output\n",
        "            per_vad = personality_vad_4(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "            # logits\n",
        "            logits = model_4(per_vad,training=False)\n",
        "            phoebe_logits.append(logits)\n",
        "            phoebe_labels.append(cat_labels)\n",
        "        elif j == 4:\n",
        "            # subjective output\n",
        "            per_vad = personality_vad_5(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "            # logits\n",
        "            logits = model_5(per_vad,training=False)\n",
        "            rachel_logits.append(logits)\n",
        "            rachel_labels.append(cat_labels)\n",
        "        elif j == 5:\n",
        "            # subjective output\n",
        "            per_vad = personality_vad_6(vad_labels,p_vad,dialogue_sementics,training=False)\n",
        "            # logits\n",
        "            logits = model_6(per_vad,training=False)\n",
        "            ross_logits.append(logits)\n",
        "            ross_labels.append(cat_labels)\n",
        "\n",
        "    target_names = ['anger', 'disgust', 'fear', 'joy', 'neutral', 'sadness', 'surprise']\n",
        "\n",
        "    chandler_true,chandler_pred = reduce_dimension(chandler_labels,chandler_logits)\n",
        "    chandler_result = classification_report(chandler_true,chandler_pred,  digits=4, output_dict=True,zero_division=1)\n",
        "\n",
        "    joey_true,joey_pred = reduce_dimension(joey_labels,joey_logits)\n",
        "    joey_result = classification_report(joey_true,joey_pred,  digits=4, output_dict=True,zero_division=1)\n",
        "\n",
        "    monica_true,monica_pred = reduce_dimension(monica_labels,monica_logits)\n",
        "    monica_result = classification_report(monica_true,monica_pred,  digits=4, output_dict=True,zero_division=1)\n",
        "\n",
        "    phoebe_true,phoebe_pred = reduce_dimension(phoebe_labels,phoebe_logits)\n",
        "    phoebe_result = classification_report(phoebe_true,phoebe_pred,  digits=4, output_dict=True,zero_division=1)\n",
        "\n",
        "    rachel_true,rachel_pred = reduce_dimension(rachel_labels,rachel_logits)\n",
        "    rachel_result = classification_report(rachel_true,rachel_pred,  digits=4, output_dict=True,zero_division=1)\n",
        "\n",
        "    ross_true,ross_pred = reduce_dimension(ross_labels,ross_logits)\n",
        "    ross_result = classification_report(ross_true,ross_pred,  digits=4, output_dict=True,zero_division=1)\n",
        "    # chandler\n",
        "    chandler_results_dict = {}\n",
        "    for key in chandler_result.keys():\n",
        "      if key != 'accuracy':\n",
        "        f1_score = chandler_result.get(key, {}).get('f1-score')\n",
        "        chandler_results_dict[key] = f1_score\n",
        "    print('Chandler:')\n",
        "    print(chandler_results_dict)\n",
        "    # joey\n",
        "    joey_results_dict = {}\n",
        "    for key in joey_result.keys():\n",
        "      if key != 'accuracy':\n",
        "        f1_score = joey_result.get(key, {}).get('f1-score')\n",
        "        joey_results_dict[key] = f1_score\n",
        "    print('Joey:')\n",
        "    print(joey_results_dict)\n",
        "    # monica\n",
        "    monica_results_dict = {}\n",
        "    for key in monica_result.keys():\n",
        "      if key != 'accuracy':\n",
        "        f1_score = monica_result.get(key, {}).get('f1-score')\n",
        "        monica_results_dict[key] = f1_score\n",
        "    print('Monica:')\n",
        "    print(monica_results_dict)\n",
        "    # phoebe\n",
        "    phoebe_results_dict = {}\n",
        "    for key in phoebe_result.keys():\n",
        "      if key != 'accuracy':\n",
        "        f1_score = phoebe_result.get(key, {}).get('f1-score')\n",
        "        phoebe_results_dict[key] = f1_score\n",
        "    print('Phoebe:')\n",
        "    print(phoebe_results_dict)\n",
        "    # rachel\n",
        "    rachel_results_dict = {}\n",
        "    for key in rachel_result.keys():\n",
        "      if key != 'accuracy':\n",
        "        f1_score = rachel_result.get(key, {}).get('f1-score')\n",
        "        rachel_results_dict[key] = f1_score\n",
        "    print('Rachel:')\n",
        "    print(rachel_results_dict)\n",
        "    # ross\n",
        "    ross_results_dict = {}\n",
        "    for key in ross_result.keys():\n",
        "      if key != 'accuracy':\n",
        "        f1_score = ross_result.get(key, {}).get('f1-score')\n",
        "        ross_results_dict[key] = f1_score\n",
        "    print('Ross:')\n",
        "    print(ross_results_dict)\n",
        "    average_dict = {}\n",
        "    for Key1,values1 in chandler_results_dict.items():\n",
        "      for Key2,values2 in joey_results_dict.items():\n",
        "        for Key3,values3 in monica_results_dict.items():\n",
        "          for Key4,values4 in phoebe_results_dict.items():\n",
        "              for Key5,values5 in rachel_results_dict.items():\n",
        "                  for Key6,values6 in ross_results_dict.items():\n",
        "                      if Key1 == Key2 == Key3 == Key4 == Key5 == Key6:\n",
        "                          add_val = values1+values2+values3+values4+values5+values6\n",
        "                          average_dict[Key1] = (add_val/6)\n",
        "\n",
        "    print('-' * 20)\n",
        "    print('Average F1-score of all the emotion categories w.r.t each character:')\n",
        "    print('-' * 20)\n",
        "    print(average_dict)\n",
        "\n"
      ],
      "metadata": {
        "id": "87T5nFGpC7Dn"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}