LSTM_Without_Gating_Sentence_Level.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM_Without_Gating_Sentence_Level.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/notmanan/Depression-Detection-Through-Multi-Modal-Data/blob/master/LSTM_Without_Gating_Sentence_Level.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p4i5eg6FCkT_",
        "colab_type": "code",
        "outputId": "022d21c1-fe60-4aa9-dc59-ba33743afeef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-u5tZ8uC0TLK",
        "colab_type": "code",
        "outputId": "3d94ed5f-e8a7-4002-f611-1c4d34c5e93e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "import gc\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "import math\n",
        "\n",
        "from smart_open import open\n",
        "from nltk.corpus import stopwords\n",
        "import sklearn\n",
        "from sklearn.metrics import classification_report\n",
        "from keras.layers import Dropout\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from gensim.models.keyedvectors import KeyedVectors\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "\n",
        "from tensorflow.keras.layers import LSTM, Dense, Bidirectional\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Input\n",
        "from tensorflow.keras.layers import concatenate\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Th9L4TBzGTHJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True) \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "\n",
        "\n",
        "#LSTM FOR TEXT \n",
        "# first input model\n",
        "class text:\n",
        "  input3 = Input(shape = (250,5100))\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  dense6 = Dense(250)(dense5)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(dense6)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=input3, outputs=output)\n",
        "  # summarize layers\n",
        "  # print(model.summary())\n",
        "  # # plot graph\n",
        "  # print(plot_model(model, to_file='multiple_inputs.png'))\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_9W8SEKzGqMd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Audio\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "\n",
        "#LSTM FOR AUDIO\n",
        "# first input model\n",
        "\n",
        "class audio:\n",
        "  input3 = Input(shape = (250,74))\n",
        "  # dense4 = Dense(1000)(input3)\n",
        "  # dense5 = Dense(500)(dense4)\n",
        "  # dense6 = Dense(250)(dense5)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(60, dropout = 0.2, recurrent_dropout = 0.2)(input3)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=input3, outputs=output)\n",
        "  # summarize layers\n",
        "  # print(model.summary())\n",
        "  # plot graph\n",
        "  # print(plot_model(model, to_file='multiple_inputs.png'))\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B2nD6KtJGria",
        "colab_type": "code",
        "outputId": "95a272aa-2dd8-4be1-b9fd-42de78bfe3e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        }
      },
      "source": [
        "#Video\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "\n",
        "#LSTM FOR VIDEO\n",
        "# first input model\n",
        "class video:\n",
        "  input3 = Input(shape = (250,388))\n",
        "  # dense4 = Dense(1000)(input3)\n",
        "  # dense5 = Dense(500)(dense4)\n",
        "  dense6 = Dense(250)(input3)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(dense6)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=input3, outputs=output)\n",
        "  # summarize layers\n",
        "  print(model.summary())\n",
        "  # plot graph\n",
        "  print(plot_model(model, to_file='multiple_inputs.png'))\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         [(None, 250, 388)]        0         \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 250, 250)          97250     \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 128)               194048    \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 1)                 129       \n",
            "=================================================================\n",
            "Total params: 291,427\n",
            "Trainable params: 291,427\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "<IPython.core.display.Image object>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1MjfyOh8EBGO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text + Audio \n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "class text_audio:\n",
        "# first input model\n",
        "  input1 = Input(shape=(250,74), name = 'Audio_input')\n",
        "  # highway6 = Highway()(highway5)\n",
        "  dense1 = Dense(74)(input1)\n",
        "\n",
        "  input3 = Input(shape = (250,5100), name = 'Text_input')\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  dense6 = Dense(250)(dense5)\n",
        "  dense3 = Dense(74)(dense6)\n",
        "  # merge input models\n",
        "  merge = concatenate([dense1,dense3], axis = 1)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=[input1, input3], outputs=output)\n",
        "  # summarize layers\n",
        "  # print(model.summary())\n",
        "  # # plot graph\n",
        "  # plot_model(model)\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "# inp = np.array([audio_train,video_train,text_train], dtype = object)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hiJ8aIFLF1jG",
        "colab_type": "code",
        "outputId": "7a5c474b-6fc1-40be-b171-c100605792ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        }
      },
      "source": [
        "#Text+Video\n",
        "\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "class text_video:\n",
        "  input2 = Input(shape=(250,388), name = 'Video_input')\n",
        "  # highway2 = Highway()(input2)\n",
        "  # highway3 = Highway()(highway2)\n",
        "  # highway4 = Highway()(highway3)\n",
        "  # dense7 = Dense(200)(highway4)\n",
        "  dense2 = Dense(250)(input2)\n",
        "\n",
        "  input3 = Input(shape = (250,5100), name = 'Text_input')\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  # dense6 = Dense(250)(dense5)\n",
        "  dense3 = Dense(250)(dense5)\n",
        "  # merge input models\n",
        "  merge = concatenate([dense2,dense3], axis = 1)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=[input2, input3], outputs=output)\n",
        "  # summarize layers\n",
        "  print(model.summary())\n",
        "  # plot graph\n",
        "  plot_model(model)\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer= self.optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "# inp = np.array([audio_train,video_train,text_train], dtype = object)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_4\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Text_input (InputLayer)         [(None, 250, 5100)]  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "dense_14 (Dense)                (None, 250, 1000)    5101000     Text_input[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "Video_input (InputLayer)        [(None, 250, 388)]   0                                            \n",
            "__________________________________________________________________________________________________\n",
            "dense_15 (Dense)                (None, 250, 500)     500500      dense_14[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense_13 (Dense)                (None, 250, 250)     97250       Video_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_16 (Dense)                (None, 250, 250)     125250      dense_15[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 500, 250)     0           dense_13[0][0]                   \n",
            "                                                                 dense_16[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lstm_4 (LSTM)                   (None, 128)          194048      concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_17 (Dense)                (None, 1)            129         lstm_4[0][0]                     \n",
            "==================================================================================================\n",
            "Total params: 6,018,177\n",
            "Trainable params: 6,018,177\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QOeZhMQDGF6s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text+Audio+Video\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "class text_audio_video:\n",
        "# first input model\n",
        "  input1 = Input(shape=(250,74), name = 'Audio_input')\n",
        "  # highway1 = Highway()(input1)\n",
        "  # highway5 = Highway()(highway1)\n",
        "  # highway6 = Highway()(highway5)\n",
        "  dense1 = Dense(74)(input1)\n",
        "\n",
        "  # second input model\n",
        "  input2 = Input(shape=(250,388), name = 'Video_input')\n",
        "  # highway2 = Highway()(input2)\n",
        "  # highway3 = Highway()(highway2)\n",
        "  # highway4 = Highway()(highway3)\n",
        "  dense7 = Dense(200)(input2)\n",
        "  dense2 = Dense(74)(dense7)\n",
        "\n",
        "  input3 = Input(shape = (250,5100), name = 'Text_input')\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  dense6 = Dense(250)(dense5)\n",
        "  dense3 = Dense(74)(dense6)\n",
        "  # merge input models\n",
        "  merge = concatenate([dense1,dense2,dense3], axis = 1)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=[input1, input2, input3], outputs=output)\n",
        "  # # summarize layers\n",
        "  # print(model.summary())\n",
        "  # # plot graph\n",
        "  # plot_model(model)\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "# inp = np.array([audio_train,video_train,text_train], dtype = object)\n",
        "\n",
        "\n",
        "\n",
        "  \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "giLhCB-uzv1q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#MAKE DATASET \n",
        "\n",
        "\n",
        "dev_location = \"dev_data\"\n",
        "test_location = \"test_data\"\n",
        "train_location = \"train_data\"\n",
        "\n",
        "devData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/dev_split_Depression_AVEC2017.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "testData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/full_test_split.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "trainData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/train_split_Depression_AVEC2017.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "\n",
        "\n",
        "dataset = np.concatenate((devData, np.concatenate((testData, trainData))))\n",
        "\n",
        "gc.collect()      \n",
        "max_num_words = 17\n",
        "model = KeyedVectors.load_word2vec_format('/content/drive/My Drive/MCA Dataset/Copy of GoogleNews-vectors-negative300.bin', binary=True)\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "\n",
        "\n",
        "def checkDataPointExistence(patientID, split):\n",
        "  for i in split:\n",
        "    if(patientID == i[0]):\n",
        "      return True\n",
        "  return False\n",
        "\n",
        "def getData(patientID, location):\n",
        "  # print(\"PatientID: \" + str(int(patientID)))\n",
        "  retData = [int(patientID)]\n",
        "  textD = getTextData(patientID, location)\n",
        "  audioD = getAudioData(patientID, location, textD)\n",
        "  videoD = getVideoData(str(int(patientID)), location, textD)\n",
        "  # patientD = np.concatenate((textD, audioD, videoD), axis = 1)\n",
        "  # print(\"Final Patient Data: \" + str(patientD.shape))\n",
        "  return textD,audioD,videoD\n",
        "\n",
        "def getTextData(patientID, location):\n",
        "  fileName = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" + str(int(patientID)) + \"_TRANSCRIPT.csv\"\n",
        "  file = np.array(pd.read_csv(fileName,delimiter='\\t',encoding='utf-8', engine='python'))\n",
        "\n",
        "  # Remove All Utterences By Ellie:\n",
        "  for i in range(len(file)):\n",
        "    if(file[i][2] != 'Participant'):\n",
        "      np.delete(file, i)\n",
        "      i-=1\n",
        "\n",
        "  # Remove Speaker Columnn\n",
        "  file = np.delete(file, 2, 1)\n",
        "  \n",
        "  # Convert Text Into Word Vectors:\n",
        "  w2vs = np.zeros((1, max_num_words*300))\n",
        "  for i in range(len(file)):\n",
        "    sentence = file[i][2]\n",
        "    w2v = returnWordToVec(sentence)\n",
        "    w2vs = np.concatenate((w2vs, w2v), axis = 0)\n",
        "  w2vs = np.delete(w2vs, 0, 0)  \n",
        "\n",
        "  # Delete Sentences and Replace With W2Vs\n",
        "  file = np.delete(file, 2, 1)\n",
        "  file = np.concatenate((file, w2vs), axis = 1)\n",
        "  return file\n",
        "\n",
        "def remove_StopWords(sentence):\n",
        "    filtered_sentence = [] \n",
        "    for w in sentence: \n",
        "        if w not in stop_words: \n",
        "            filtered_sentence.append(w)\n",
        "    return filtered_sentence\n",
        "\n",
        "def returnWordToVec(sentence):\n",
        "  global max_num_words, stop_words, model\n",
        "  sentence = str(sentence).split(\" \")\n",
        "  sentence = remove_StopWords(sentence)\n",
        "  index_word = 0\n",
        "  wordMatrix = np.zeros(max_num_words*300)\n",
        "  for j in range(min(max_num_words, len(sentence))):\n",
        "    try:\n",
        "      word = sentence[j]\n",
        "      if(word[0] == '<'):\n",
        "        if(word.find('>')!=-1):\n",
        "          word = word[1:-1]\n",
        "        else:\n",
        "          word = word[1:]\n",
        "      else:\n",
        "        if(word.find('>')!=-1):\n",
        "          word = word[0:-1]\n",
        "      ss = np.array(model[word])\n",
        "      wordMatrix[index_word*300:(index_word+1)*300] = ss\n",
        "      index_word+=1\n",
        "    except Exception as e:\n",
        "      continue\n",
        "  wordMatrix = np.array(wordMatrix.reshape(1,-1))\n",
        "  return wordMatrix\n",
        "\n",
        "def audioDataHelper(X):\n",
        "    for i in range(X.shape[0]):\n",
        "        if(X[i,1] == 0):\n",
        "            X[i,0] = 0\n",
        "            for j in range(7):\n",
        "                X[i,j+1] = 0\n",
        "    X = np.array(X)\n",
        "    return X\n",
        "    \n",
        "def getAudioData(patientID, location, textD):\n",
        "  fileName = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" + str(int(patientID)) + \"_COVAREP.csv\"\n",
        "  data = pd.read_csv(fileName,header = None)\n",
        "  data = data.iloc[:,:].values\n",
        "  data = audioDataHelper(data)\n",
        "  # print(\"Audio Raw Data:\" + str(data.shape))\n",
        "  sentenceDatas = []\n",
        "  for sentence in textD:\n",
        "    sentenceStartime = sentence[0]\n",
        "    sentenceEndTime = sentence[1]\n",
        "    startIndex = math.floor(sentenceStartime/0.01)\n",
        "    endIndex = math.ceil(sentenceEndTime/0.01)\n",
        "    # print(\"Start Time: \" + str(startIndex))\n",
        "    # print(\"End Time: \" + str(endIndex))\n",
        "    sentenceData = data[startIndex: endIndex]\n",
        "    sentenceData = np.average(sentenceData, axis = 0)\n",
        "    # This might be a possible error\n",
        "    sentenceData = np.array(sentenceData.reshape(1, -1))\n",
        "    sentenceDatas.append(sentenceData)\n",
        "  \n",
        "  sentenceDatas = np.array(sentenceDatas)\n",
        "  sentenceDatas = np.reshape(sentenceDatas, (textD.shape[0],-1))\n",
        "  # print(\"Audio Final Data:\" + str(sentenceDatas.shape))\n",
        "\n",
        "  return sentenceDatas\n",
        "\n",
        "def getVideoDataHelper(patientID, location):\n",
        "  root = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" \n",
        "  file1 = root + (patientID)+\"_CLNF_AUs.txt\"\n",
        "  file2 = root + (patientID)+\"_CLNF_features.txt\"\n",
        "  file3 = root + (patientID)+\"_CLNF_features3D.txt\"\n",
        "  file4 = root + (patientID)+\"_CLNF_gaze.txt\"\n",
        "  file5 = root + (patientID)+\"_CLNF_hog.txt\"\n",
        "  file6 = root + (patientID)+\"_CLNF_pose.txt\"\n",
        "  data = processVideoData(file1)\n",
        "  data = np.concatenate((data, processVideoData(file2)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file3)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file4)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file6)), 1)\n",
        "  # print(\"Video Raw Data:\" + str(data.shape))\n",
        "  return data\n",
        "\n",
        "def processVideoData(filename):\n",
        "  try:\n",
        "    data = pd.read_csv(filename,delimiter=',', dtype=float)\n",
        "    X = data.iloc[:,:].values\n",
        "    X = np.delete(X, 0, 1)\n",
        "    X = np.delete(X, 1, 1)\n",
        "  except:\n",
        "    # print(\"Video Data corrupt, fixing.\")\n",
        "    data = pd.read_csv(filename,delimiter=',')\n",
        "    X = data.iloc[:,:].values\n",
        "    X = np.delete(X, 0, 1)\n",
        "    X = np.delete(X, 1, 1)\n",
        "    for i in range(len(X)):\n",
        "        if(isinstance(X[i][5],str) or isinstance(X[i][7],str)):\n",
        "            X[i] = np.zeros((1, X.shape[1]))\n",
        "            # print(\"se\" , end = \" \")    \n",
        "  return X\n",
        "\n",
        "def getVideoData(patientID, location, textD):\n",
        "  data = getVideoDataHelper(patientID, location)\n",
        "  sentenceDatas = []\n",
        "  for sentence in textD:\n",
        "    sentenceStartime = sentence[0]\n",
        "    sentenceEndTime = sentence[1]\n",
        "    startIndex = math.floor(sentenceStartime/0.333)\n",
        "    endIndex = math.ceil(sentenceEndTime/0.333)\n",
        "    # print(\"Start Time: \" + str(startIndex))\n",
        "    sentenceData = data[startIndex: endIndex]\n",
        "    sentenceData = np.average(sentenceData, axis = 0)\n",
        "    # This might be a possible error\n",
        "    sentenceData = np.array(sentenceData.reshape(1, -1))\n",
        "    sentenceDatas.append(sentenceData)\n",
        "  \n",
        "  sentenceDatas = np.array(sentenceDatas)\n",
        "  sentenceDatas = np.reshape(sentenceDatas, (textD.shape[0],-1))\n",
        "  # print(\"Video Final Data:\" + str(sentenceDatas.shape))\n",
        "  return sentenceDatas\n",
        "\n",
        "# Xtrain = []\n",
        "Ytrain = []\n",
        "# Xtest = []\n",
        "Ytest = []\n",
        "\n",
        "\n",
        "audio_train = []\n",
        "video_train = []\n",
        "text_train = []\n",
        "\n",
        "audio_test = []\n",
        "video_test = []\n",
        "text_test = []\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "for datapoint in dataset:\n",
        "  # print(datapoint[0])\n",
        "  if(checkDataPointExistence(datapoint[0], devData)):\n",
        "\n",
        "    # Data Point in Dev Set\n",
        "    text,audio,video = getData(datapoint[0], dev_location)\n",
        "    audio_train.append(audio)\n",
        "    video_train.append(video)\n",
        "    text_train.append(text)\n",
        "    # Xtest.append(data)\n",
        "    Ytrain.append(datapoint[1])\n",
        "    # print(data)\n",
        "  elif(checkDataPointExistence(datapoint[0], testData)):\n",
        "    # Data Point in Test Set\n",
        "    text,audio,video = getData(datapoint[0], test_location)\n",
        "    audio_test.append(audio)\n",
        "    video_test.append(video)\n",
        "    text_test.append(text)\n",
        "    # Xtest.append(data)\n",
        "    Ytest.append(datapoint[1])\n",
        "  elif(checkDataPointExistence(datapoint[0], trainData)):\n",
        "    # Data Point in Train Set\n",
        "    text,audio,video = getData(datapoint[0], train_location)\n",
        "    audio_train.append(audio)\n",
        "    video_train.append(video)\n",
        "    text_train.append(text)\n",
        "    # Xtest.append(data)\n",
        "    Ytrain.append(datapoint[1])\n",
        "\n",
        "def refactor(arr, size):\n",
        "  arrsize = arr.shape[0]\n",
        "  temp = np.zeros((size, arr.shape[1]))\n",
        "  for i in range(min(len(arr), size)):\n",
        "    temp[i] = arr[i]\n",
        "  return temp\n",
        "\n",
        "numberOfSentences = 250\n",
        "\n",
        "devData = []\n",
        "trainData = []\n",
        "testData = []\n",
        "gc.collect()\n",
        "\n",
        "for i in range(len(audio_train)):\n",
        "  audio_train[i] = refactor(audio_train[i], numberOfSentences)\n",
        "  video_train[i] = refactor(video_train[i], numberOfSentences)\n",
        "  text_train[i] = refactor(text_train[i], numberOfSentences)\n",
        "  # print(Xtrain[i].shape)\n",
        "\n",
        "# for i in range(len(audio_dev)):\n",
        "#   audio_dev[i] = refactor(audio_dev[i], numberOfSentences)\n",
        "#   video_dev[i] = refactor(video_dev[i], numberOfSentences)\n",
        "#   text_dev[i] = refactor(text_dev[i], numberOfSentences)\n",
        "#   # print(Xtrain[i].shape)\n",
        "\n",
        "for i in range(len(audio_test)):\n",
        "  audio_test[i] = refactor(audio_train[i], numberOfSentences)\n",
        "  video_test[i] = refactor(video_train[i], numberOfSentences)\n",
        "  text_test[i] = refactor(text_train[i], numberOfSentences)\n",
        "  # print(Xtest[i].shape)\n",
        "audio_test = np.array(audio_test)\n",
        "video_test = np.array(video_test)\n",
        "text_test = np.array(text_test)\n",
        "text_test = text_test[:,:,2:]\n",
        "\n",
        "audio_train = np.array(audio_train)\n",
        "video_train = np.array(video_train)\n",
        "text_train = np.array(text_train)\n",
        "text_train = text_train[:,:,2:]\n",
        "\n",
        "# audio_dev = np.array(audio_dev)\n",
        "# video_dev = np.array(video_dev)\n",
        "# text_dev = np.array(text_dev)\n",
        "# text_dev = text_dev[:,:,2:]\n",
        "\n",
        "dataset = []\n",
        "gc.collect()\n",
        "\n",
        "print(audio_test.shape,video_test.shape,text_test.shape)\n",
        "print(audio_train.shape,video_train.shape,text_train.shape)\n",
        "# print(audio_dev.shape,video_dev.shape,text_dev.shape)\n",
        "\n",
        "Ytrain = np.array(Ytrain)\n",
        "Ytest = np.array(Ytest)\n",
        "\n",
        "\n",
        "import sklearn\n",
        "from sklearn import preprocessing\n",
        "\n",
        "\n",
        "def upsample(X_train,Y_train):\n",
        "  X_train_0 = X_train[Y_train==0]\n",
        "  X_train_1 = X_train[Y_train==1]\n",
        "\n",
        "  Y_train_1 = Y_train[Y_train==1]\n",
        "  # print(Y_train_1.shape)\n",
        "  # print(X_train_1.shape)\n",
        "  size = X_train_0.shape[0] - X_train_1.shape[0]\n",
        "  X = []\n",
        "  Y = []\n",
        "  X_train = list(X_train)\n",
        "  Y_train = list(Y_train)\n",
        "  while(size>0):\n",
        "    size -= 1\n",
        "    index = np.random.randint(0,X_train_1.shape[0]-1)\n",
        "    leave_index = np.random.randint(0,len(X_train)-1)\n",
        "    X_add = X_train_1[index]\n",
        "    X_leave = X_train[leave_index]\n",
        "\n",
        "    Y_add = Y_train_1[index]\n",
        "    Y_leave = Y_train[leave_index]\n",
        "\n",
        "    X_train[leave_index] = X_add\n",
        "    X_train.append(X_leave)\n",
        "\n",
        "    Y_train[leave_index] = Y_add\n",
        "    Y_train.append(Y_leave)\n",
        "\n",
        "\n",
        "  X_train = np.array(X_train)\n",
        "  Y_train = np.array(Y_train)\n",
        "  return X_train,Y_train\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "audio_train = np.nan_to_num(audio_train)\n",
        "video_train = np.nan_to_num(video_train)\n",
        "text_train = np.nan_to_num(text_train)\n",
        "\n",
        "audio_train, _ = upsample(audio_train,Ytrain)\n",
        "video_train, _ = upsample(video_train,Ytrain)\n",
        "text_train, Ytrain = upsample(text_train,Ytrain)\n",
        "\n",
        "print(audio_train.shape)\n",
        "print(video_train.shape)\n",
        "print(text_train.shape)\n",
        "print(Ytrain.shape)\n",
        "\n",
        "for i in range(audio_train.shape[0]):\n",
        "  audio_train[i] = sklearn.preprocessing.normalize(audio_train[i])\n",
        "  video_train[i] = sklearn.preprocessing.normalize(video_train[i])\n",
        "  text_train[i] = sklearn.preprocessing.normalize(text_train[i])\n",
        "\n",
        "\n",
        "print(Ytest.shape)\n",
        "\n",
        "\n",
        "\n",
        "audio_test = np.nan_to_num(audio_test)\n",
        "video_test = np.nan_to_num(video_test)\n",
        "text_test = np.nan_to_num(text_test)\n",
        "\n",
        "\n",
        "for i in range(audio_test.shape[0]):\n",
        "  audio_test[i] = sklearn.preprocessing.normalize(audio_test[i])\n",
        "  video_test[i] = sklearn.preprocessing.normalize(video_test[i])\n",
        "  text_test[i] = sklearn.preprocessing.normalize(text_test[i])\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dgsQxZC_0PWg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def Thresholding(Y_pred, threshold):\n",
        "  Y_pred2 = []\n",
        "  print(\"Y_pred: \", Y_pred.shape)\n",
        "  for i in range(len(Y_pred)):\n",
        "    if(Y_pred[i] < threshold):\n",
        "      Y_pred2.append(0)\n",
        "    else:\n",
        "      Y_pred2.append(1)\n",
        "\n",
        "  return np.array(Y_pred2)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fsRwscil0ujr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"-------------------------------------- TEXT (W/o GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = text()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit(text_train,Ytrain, epochs=50,validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=20, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True), batch_size = 137)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict(text_test)\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.7)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FDCf_4mv0vX3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"--------------------------------------AUDIO (W/o GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = audio()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit(audio_train,Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=30, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True), epochs=50, batch_size = 137)\n",
        "\n",
        "pred = model.predict(audio_test)\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.7)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e81UcOmj1Kwv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"--------------------------------------VIDEO (W/o GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = video()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit(video_train,Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=30, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True), epochs=50, batch_size = 137)\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict(video_test)\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.7)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mnTrAaeP2wLB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"--------------------------------------AUDIO + TEXT (W/o GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = text_audio()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit([audio_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True),epochs=50, batch_size = 137)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([audio_test,text_test])\n",
        "pred2 = model.predict([audio_train,text_train])\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.7)))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QLMs7UOI3PVT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"--------------------------------------VIDEO + TEXT (W/O GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = text_video()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit([video_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True),epochs=50, batch_size = 137)\n",
        "\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([video_test,text_test])\n",
        "pred2 = model.predict([video_train,text_train])\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.7)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Zi-zJ8S3jVW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"--------------------------------------AUDIO + TEXT + VIDEO (W/O GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = text_audio_video()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit([audio_train,video_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True),epochs=50, batch_size = 137)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([audio_test,video_test,text_test])\n",
        "pred2 = model.predict([audio_train,video_train,text_train])\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.7)))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}