3_Evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "colab": {
      "name": "3_Evaluation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.1"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": true,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "varInspector": {
      "cols": {
        "lenName": 16,
        "lenType": 16,
        "lenVar": 40
      },
      "kernels_config": {
        "python": {
          "delete_cmd_postfix": "",
          "delete_cmd_prefix": "del ",
          "library": "var_list.py",
          "varRefreshCmd": "print(var_dic_list())"
        },
        "r": {
          "delete_cmd_postfix": ") ",
          "delete_cmd_prefix": "rm(",
          "library": "var_list.r",
          "varRefreshCmd": "cat(var_dic_list()) "
        }
      },
      "types_to_exclude": [
        "module",
        "function",
        "builtin_function_or_method",
        "instance",
        "_Feature"
      ],
      "window_display": false
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2915c65638da4a35ac7eefb707aa32a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ee6906d66547458a9945b3206a60889a",
              "IPY_MODEL_28319b934f2349e99a1e9e1fbe28be9a"
            ],
            "layout": "IPY_MODEL_452c6b982bfc45dfa7d081e5896142c9"
          }
        },
        "452c6b982bfc45dfa7d081e5896142c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee6906d66547458a9945b3206a60889a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_02717415435a499d92fa15313667ac88",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6922d6d90541497687b2f319211a99cf",
            "value": 570
          }
        },
        "28319b934f2349e99a1e9e1fbe28be9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d2d3c038e14476998178b3837a90d3c",
            "placeholder": "​",
            "style": "IPY_MODEL_bb7061509e034830903feacfca6e8e7b",
            "value": " 570/570 [00:12&lt;00:00, 44.9B/s]"
          }
        },
        "6922d6d90541497687b2f319211a99cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "02717415435a499d92fa15313667ac88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb7061509e034830903feacfca6e8e7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7d2d3c038e14476998178b3837a90d3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a18cc64aa7804df3b3ec6ba1d9f1423e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_145680c85fed456a94bb90eab9134c09",
              "IPY_MODEL_be411fd74d8a4f2ca53442693a5e98fe"
            ],
            "layout": "IPY_MODEL_9d8da39be354420da03f0a6ec0317964"
          }
        },
        "9d8da39be354420da03f0a6ec0317964": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "145680c85fed456a94bb90eab9134c09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8c2d21fa94b47e3bb754240f3094063",
            "max": 536063208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a09561b99b3043cab26d3e7640c01120",
            "value": 536063208
          }
        },
        "be411fd74d8a4f2ca53442693a5e98fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0c415ac9e2f48e0a87d53e1c0b4ff70",
            "placeholder": "​",
            "style": "IPY_MODEL_0dd135b68e8348ef92faa636b815d4af",
            "value": " 536M/536M [00:12&lt;00:00, 43.1MB/s]"
          }
        },
        "a09561b99b3043cab26d3e7640c01120": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "b8c2d21fa94b47e3bb754240f3094063": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0dd135b68e8348ef92faa636b815d4af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0c415ac9e2f48e0a87d53e1c0b4ff70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/julrods/aggressive-tweet-analyzer/blob/main/3_Evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "handed-commission",
        "toc": true
      },
      "source": [
        "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
        "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Environment\" data-toc-modified-id=\"Environment-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Environment</a></span><ul class=\"toc-item\"><li><span><a href=\"#Libraries\" data-toc-modified-id=\"Libraries-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Libraries</a></span></li><li><span><a href=\"#Functions\" data-toc-modified-id=\"Functions-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Functions</a></span></li></ul></li><li><span><a href=\"#Evaluation\" data-toc-modified-id=\"Evaluation-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Evaluation</a></span><ul class=\"toc-item\"><li><span><a href=\"#BERT-setup\" data-toc-modified-id=\"BERT-setup-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>BERT setup</a></span></li><li><span><a href=\"#Making-predictions-for-the-evaluation-data\" data-toc-modified-id=\"Making-predictions-for-the-evaluation-data-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Making predictions for the evaluation data</a></span></li><li><span><a href=\"#Assessing-the-results\" data-toc-modified-id=\"Assessing-the-results-2.3\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Assessing the results</a></span></li></ul></li><li><span><a href=\"#Conclusion\" data-toc-modified-id=\"Conclusion-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Conclusion</a></span></li></ul></div>"
      ],
      "id": "handed-commission"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "peripheral-disease"
      },
      "source": [
        "# Evaluation"
      ],
      "id": "peripheral-disease"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TQ3pKy92AGho"
      },
      "source": [
        "In this notebook I have evaluated the BERT binary classifier that I fine-tuned with the Aggression dataset. The evaluation is done with 17k+ comments that I scraped myself from Kevin Spacey's last 3 Instagram posts. \n",
        "\n",
        "Results: \n",
        "- 83% of the comments were labeled as \"not aggressive\" and 17% as \"aggressive\".\n",
        "- The model achieved a precision of 85% for class \"agressive\" (class 1)."
      ],
      "id": "TQ3pKy92AGho"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f_JoRO6wNp81"
      },
      "source": [
        "## Environment"
      ],
      "id": "f_JoRO6wNp81"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QVP_whkDNxlJ"
      },
      "source": [
        "### Libraries"
      ],
      "id": "QVP_whkDNxlJ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9PfLHAWaNTDx"
      },
      "source": [
        "!pip install transformers"
      ],
      "id": "9PfLHAWaNTDx",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kvCuXkLNVy3"
      },
      "source": [
        "# Base libraries\n",
        "import os\n",
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import pickle\n",
        "\n",
        "# ML and DL libraries\n",
        "from sklearn.metrics import confusion_matrix, f1_score, classification_report\n",
        "from transformers import TFBertModel, TFBertForSequenceClassification"
      ],
      "id": "2kvCuXkLNVy3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EpC84RHfNmMN",
        "outputId": "b95df9f2-2e44-48ed-cf30-f14ce2380701"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "id": "EpC84RHfNmMN",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eH6tAPkON003"
      },
      "source": [
        "### Functions"
      ],
      "id": "eH6tAPkON003"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-dXbcWy9N3dT"
      },
      "source": [
        "def load_vectors(dataset_name):\n",
        "  \"\"\"\n",
        "  Input: name of the dataset\n",
        "  Output: dataset tokenized into 3 vectors (input_ids, attention_masks and labels) ready to be inputed into the model \n",
        "  \"\"\"\n",
        "  \n",
        "  # Create the paths\n",
        "  pickle_inp_path = f'/content/gdrive/MyDrive/Cyber-bullying-project/data/3_tokenized_data/bert_inp_{dataset_name}.pkl'\n",
        "  pickle_mask_path = f'/content/gdrive/MyDrive/Cyber-bullying-project/data/3_tokenized_data/bert_mask_{dataset_name}.pkl'\n",
        "\n",
        "  # Load the files\n",
        "  input_ids = pickle.load(open(pickle_inp_path, 'rb'))\n",
        "  attention_masks = pickle.load(open(pickle_mask_path, 'rb'))\n",
        "\n",
        "  return input_ids, attention_masks"
      ],
      "id": "-dXbcWy9N3dT",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zwVFxI63N9i_"
      },
      "source": [
        "def bert_setup():\n",
        "  \"\"\" \n",
        "  Loads BERT for sequence classification; sets the loss, metric and optimizer; compiles the model\n",
        "  \"\"\"\n",
        "  \n",
        "  base_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "  \n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "  metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5,\n",
        "                                       epsilon=1e-08)\n",
        "  \n",
        "  base_model.compile(loss = loss, optimizer = optimizer, metrics = [metric])\n",
        "  \n",
        "  return base_model"
      ],
      "id": "zwVFxI63N9i_",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_gEpknSCOIfZ"
      },
      "source": [
        "def evaluate_model(model_name, inputs, mask, base_model):\n",
        "  \"\"\" \n",
        "  Input: name of the model we want to load; \n",
        "         evaluation data split into 2 vectors (inputs and mask); \n",
        "         base model to load the weights into\n",
        "  Output: predicted labels\n",
        "  \"\"\"\n",
        " \n",
        "  # Load the model weights\n",
        "  model_save_path = f'/content/gdrive/MyDrive/Cyber-bullying-project/models/{model_name}.h5'\n",
        "  base_model.load_weights(model_save_path)\n",
        "  trained_model = base_model\n",
        "  \n",
        "  # Make predictions\n",
        "  preds = trained_model.predict([inputs, mask],\n",
        "                                batch_size=32)\n",
        "  \n",
        "  # Find the predicted labels\n",
        "  pred_labels = [np.argmax(pred) for pred in preds[0]]\n",
        "\n",
        "  return pred_labels"
      ],
      "id": "_gEpknSCOIfZ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GYCcZKO7O2yP"
      },
      "source": [
        "## Evaluation"
      ],
      "id": "GYCcZKO7O2yP"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oEx3PoJiHnt2"
      },
      "source": [
        "### BERT setup"
      ],
      "id": "oEx3PoJiHnt2"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206,
          "referenced_widgets": [
            "2915c65638da4a35ac7eefb707aa32a4",
            "452c6b982bfc45dfa7d081e5896142c9",
            "ee6906d66547458a9945b3206a60889a",
            "28319b934f2349e99a1e9e1fbe28be9a",
            "6922d6d90541497687b2f319211a99cf",
            "02717415435a499d92fa15313667ac88",
            "bb7061509e034830903feacfca6e8e7b",
            "7d2d3c038e14476998178b3837a90d3c",
            "a18cc64aa7804df3b3ec6ba1d9f1423e",
            "9d8da39be354420da03f0a6ec0317964",
            "145680c85fed456a94bb90eab9134c09",
            "be411fd74d8a4f2ca53442693a5e98fe",
            "a09561b99b3043cab26d3e7640c01120",
            "b8c2d21fa94b47e3bb754240f3094063",
            "0dd135b68e8348ef92faa636b815d4af",
            "d0c415ac9e2f48e0a87d53e1c0b4ff70"
          ]
        },
        "id": "mkO-TJc0NSZi",
        "outputId": "1524e713-2cfd-48cb-fb3e-3d4a864216fa"
      },
      "source": [
        "# BERT setup\n",
        "base_model = bert_setup()"
      ],
      "id": "mkO-TJc0NSZi",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2915c65638da4a35ac7eefb707aa32a4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a18cc64aa7804df3b3ec6ba1d9f1423e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=536063208.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uGtX6UhDHwr-"
      },
      "source": [
        "### Making predictions for the evaluation data"
      ],
      "id": "uGtX6UhDHwr-"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tamil-david"
      },
      "source": [
        "# Load the vectorized evaluation data\n",
        "input_ids, attention_masks = load_vectors('eval_data')"
      ],
      "id": "tamil-david",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dLfebvTuNKG2"
      },
      "source": [
        "# Make predictions\n",
        "pred_labels = evaluate_model('aggression_model_1epoch', input_ids, attention_masks, base_model)"
      ],
      "id": "dLfebvTuNKG2",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZ5p9UIXPQgX"
      },
      "source": [
        "# Load the clean evaluation dataframe and store the labels in a column\n",
        "eval_data_path = '/content/gdrive/MyDrive/Cyber-bullying-project/data/4_evaluation_data/clean_evaluation_data.csv'\n",
        "eval_data = pd.read_csv(eval_data_path)\n",
        "eval_data['label'] = pred_labels"
      ],
      "id": "dZ5p9UIXPQgX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nt2KxTOSPoQj"
      },
      "source": [
        "# Keep the original comment and the label only and save in a new file\n",
        "labeled_eval_data = eval_data[['text', 'label']]\n",
        "labeled_eval_data_path = '/content/gdrive/MyDrive/Cyber-bullying-project/data/4_evaluation_data/labeled_evaluation_data.csv'\n",
        "labeled_eval_data.to_csv(labeled_eval_data_path, index = False)"
      ],
      "id": "nt2KxTOSPoQj",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t22pvrx6H_uH"
      },
      "source": [
        "### Assessing the results"
      ],
      "id": "t22pvrx6H_uH"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 362
        },
        "id": "Hk0ndWKyGcWZ",
        "outputId": "5d796505-9075-4319-fe59-5aa9bd2511d2"
      },
      "source": [
        "labeled_eval_data.sample(10)"
      ],
      "id": "Hk0ndWKyGcWZ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5254</th>\n",
              "      <td>Please com bake kevin</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10151</th>\n",
              "      <td>Thanks Kevin. You’ll be back.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12334</th>\n",
              "      <td>Merry christmas mr president 🌹💙</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2932</th>\n",
              "      <td>Sex offender</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13906</th>\n",
              "      <td>Super weird</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>842</th>\n",
              "      <td>Pedo</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14643</th>\n",
              "      <td>Miss you frank Underwood @kevinspacey 😭😭😭</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7855</th>\n",
              "      <td>We miss you!!!</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10847</th>\n",
              "      <td>You are the best</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5135</th>\n",
              "      <td>Need you back in HOC man</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            text  label\n",
              "5254                       Please com bake kevin      0\n",
              "10151              Thanks Kevin. You’ll be back.      0\n",
              "12334            Merry christmas mr president 🌹💙      0\n",
              "2932                                Sex offender      1\n",
              "13906                                Super weird      0\n",
              "842                                         Pedo      1\n",
              "14643  Miss you frank Underwood @kevinspacey 😭😭😭      0\n",
              "7855                              We miss you!!!      0\n",
              "10847                           You are the best      0\n",
              "5135                    Need you back in HOC man      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wENey90wGjZf"
      },
      "source": [
        "From the above sample we can see that the model predicted the positive comments correctly as not aggressive, and the aggressive comments were correctly predicted as well. The \"Super weird\" comment is not positive, but not aggressive either, so it was classified as not-aggressive. "
      ],
      "id": "wENey90wGjZf"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R7NZk5w2GcWZ",
        "outputId": "468294cd-bd5c-4f2c-e3c2-e3807f396fb2"
      },
      "source": [
        "# Print the number\n",
        "number_total_comments = len(labeled_eval_data)\n",
        "number_class0_comments = len(labeled_eval_data[labeled_eval_data['label'] == 0])\n",
        "number_class1_comments = len(labeled_eval_data[labeled_eval_data['label'] == 1])\n",
        "print(f'Out of {number_total_comments} comments, {number_class0_comments} ({round(number_class0_comments/number_total_comments*100)}%) were labeled as \"aggressive\" and {number_class1_comments} ({round(number_class1_comments/number_total_comments*100)}%) as \"not aggressive\".')"
      ],
      "id": "R7NZk5w2GcWZ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Out of 17749 comments, 14790 (83%) were labeled as \"aggressive\" and 2959 (17%) as \"not aggressive\".\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IrhG8VYnQiyv"
      },
      "source": [
        "The next step was to go through the comments that were labeled as \"aggressive\" to check if they were correctly classified. I did it manualy on Google Sheets and created a new column named wrong_label with values 0 or 1 (0 = correct label, the comment is aggressive / 1 = wrong label, the comment is not aggressive). Due to time constraints, I did not do it for the comments that were labeled by the model as \"not aggressive\"."
      ],
      "id": "IrhG8VYnQiyv"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205
        },
        "id": "9ZWv6ZlIf0eR",
        "outputId": "04676f63-5dc8-4be3-93bc-c636c9c2471e"
      },
      "source": [
        "# Loading the dataset that contains the comments labeled as aggressive by the model, checked by me\n",
        "labeled_eval_data_checked_path = '/content/gdrive/MyDrive/Cyber-bullying-project/data/4_evaluation_data/labeled_evaluation_data_checked.csv'\n",
        "checked = pd.read_csv(labeled_eval_data_checked_path) \n",
        "checked.head()"
      ],
      "id": "9ZWv6ZlIf0eR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>wrong_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Rapist</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Racist pedo</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I so fucking love you Kevin! Happy to see you'...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fuck you asshole</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>You are My perfec daddy ❤️😍😍</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label  wrong_label\n",
              "0                                             Rapist      1          NaN\n",
              "1                                        Racist pedo      1          NaN\n",
              "2  I so fucking love you Kevin! Happy to see you'...      1          1.0\n",
              "3                                   fuck you asshole      1          NaN\n",
              "4                       You are My perfec daddy ❤️😍😍      1          1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205
        },
        "id": "RDiT-p4Jg7Xv",
        "outputId": "e295b62d-6edf-4893-91aa-a3a6451e1333"
      },
      "source": [
        "# I only wrote 1 for the mislabeled instances, so we have to fill the null values with 0 for the ones that are correctly labeled\n",
        "checked['wrong_label'] = checked['wrong_label'].fillna(0).astype(int)\n",
        "checked.head()"
      ],
      "id": "RDiT-p4Jg7Xv",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>wrong_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Rapist</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Racist pedo</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I so fucking love you Kevin! Happy to see you'...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fuck you asshole</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>You are My perfec daddy ❤️😍😍</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label  wrong_label\n",
              "0                                             Rapist      1            0\n",
              "1                                        Racist pedo      1            0\n",
              "2  I so fucking love you Kevin! Happy to see you'...      1            1\n",
              "3                                   fuck you asshole      1            0\n",
              "4                       You are My perfec daddy ❤️😍😍      1            1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVvYDCXwrf8F"
      },
      "source": [
        "# Save the file with the filled values\n",
        "checked.to_csv(labeled_eval_data_checked_path, index = False)"
      ],
      "id": "hVvYDCXwrf8F",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qno_CZfrgPri",
        "outputId": "dceb1a9c-a30c-488b-c4cf-e7901d99b346"
      },
      "source": [
        "# Evaluating the precision for class 1 (classified as \"aggressive\" by the model)\n",
        "precision = 1 - checked['wrong_label'].sum() / len(checked)\n",
        "precision_percent = (1 - checked['wrong_label'].sum() / len(checked)) * 100\n",
        "print(f'Out of all the instances labelled as class 1 (aggressive comments), {precision_percent:.2f}% were correct. The precision of the model for class 1 is {precision:.2f}')"
      ],
      "id": "Qno_CZfrgPri",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Out of all the instances labelled as class 1 (aggressive comments), 85.64% were correct. The precision of the model for class 1 is 0.86\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205
        },
        "id": "Npppknlig_Rh",
        "outputId": "e4c82a12-19b5-4604-a7d8-9ca88bf5de36"
      },
      "source": [
        "# Sample of false positives: \n",
        "checked[checked['wrong_label']==1].sample(5)"
      ],
      "id": "Npppknlig_Rh",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>wrong_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2460</th>\n",
              "      <td>Welcomback boss🔥</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1120</th>\n",
              "      <td>You are fucking AWESOME!</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>Awww yissss ima watch this shit</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2312</th>\n",
              "      <td>Y’all crazy, accuser was anonymous and died. I...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2271</th>\n",
              "      <td>Shit men. Come back to house of cards.</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  label  wrong_label\n",
              "2460                                   Welcomback boss🔥      1            1\n",
              "1120                           You are fucking AWESOME!      1            1\n",
              "47                      Awww yissss ima watch this shit      1            1\n",
              "2312  Y’all crazy, accuser was anonymous and died. I...      1            1\n",
              "2271             Shit men. Come back to house of cards.      1            1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5UXhElKILZb"
      },
      "source": [
        "## Conclusion"
      ],
      "id": "m5UXhElKILZb"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9A5NCOrSsYmG"
      },
      "source": [
        "The precision for class one was satisfactory at 85.64%. \n",
        "\n",
        "Most of the false positives contain words that are negative but used in a \"friendly\" manner:  \n",
        "- beast\n",
        "- boss\n",
        "- bullshit\n",
        "- crack\n",
        "- crap\n",
        "- fuck/fucking\n",
        "- goat\n",
        "- motherfucker\n",
        "- savage\n",
        "- shit\n",
        "- son of a bitch\n",
        "- stupid\n",
        "- sucks\n",
        "\n",
        "Some recurrent examples: \n",
        "- I fucking love you Kevin\n",
        "- You're a fucking God you son of a bitch\n",
        "- Fuck yeah\n",
        "- Hell yes\n",
        "- House of Cards sucks without you\n",
        "\n",
        "To improve the model we could train it again with sentences that have swear words, some that are truly aggressive and some that are just friendly banter. "
      ],
      "id": "9A5NCOrSsYmG"
    }
  ]
}