Pipeline_NLP_new_data.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wk7DzKkWGZV4"
      },
      "source": [
        "## <font color='orange'>Setup</font>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OjzojvvE4prc"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd '/content/drive/My Drive/Steps NLP framework/Folder_steps_material'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-6DI_GhaS-C0"
      },
      "source": [
        "#### Pip installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ihaWpY3S1wAO"
      },
      "outputs": [],
      "source": [
        "# Installation for IOC defanging - https://github.com/ioc-fang/ioc-fanger\n",
        "!sudo pip install -q ioc-fanger\n",
        "\n",
        "# Installation for Pronouns and ellipsis resolution\n",
        "import os\n",
        "if not os.path.exists('./neuralcoref'):\n",
        "  !git clone https://github.com/huggingface/neuralcoref.git\n",
        "%cd neuralcoref\n",
        "!pip install -q -r requirements.txt\n",
        "!pip install -q neuralcoref --no-binary neuralcoref\n",
        "!python setup.py build_ext -q --inplace\n",
        "!pip install -q -e .\n",
        "!python -q -m spacy download en_core_web_lg\n",
        "\n",
        "import en_core_web_lg\n",
        "pass2act_nlp = en_core_web_lg.load()\n",
        "ellipsis_nlp = en_core_web_lg.load()\n",
        "\n",
        "import neuralcoref\n",
        "neuralcoref.add_to_pipe(ellipsis_nlp)\n",
        "%cd ./..\n",
        "\n",
        "# Installation for Synonym homogenization\n",
        "!pip install -q pattern\n",
        "#!pip install -q -U spacy\n",
        "!pip install -q -U torch torchvision torchaudio\n",
        "!pip install -q allennlp  # https://github.com/allenai/allennlp\n",
        "!pip install -q allennlp_models  # https://github.com/allenai/allennlp-models\n",
        "!python -q -m spacy download en_core_web_sm\n",
        "\n",
        "# Installation for Passive\\active conversion - https://github.com/DanManN/pass2act\n",
        "#!python -m spacy download en_core_web_lg\n",
        "\n",
        "# Installation for Misspelling correction - https://github.com/bakwc/JamSpell\n",
        "!!sudo apt-get install -q swig3.0\n",
        "!sudo pip install -q jamspell\n",
        "\n",
        "# Installation for Unrelated content removal\n",
        "!pip install -q transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TFzsVnOMGeQM"
      },
      "source": [
        "## <font color='orange'>Import libraries and data</font>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-8l9kDK-BEBL"
      },
      "outputs": [],
      "source": [
        "%cd '/content/drive/My Drive/Steps NLP framework/Folder_steps_material'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ouhLEnOOZPxa"
      },
      "source": [
        "### Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "188_mR3q5vBT"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "import csv\n",
        "import copy\n",
        "import json\n",
        "import spacy\n",
        "import string\n",
        "from tqdm import tqdm\n",
        "from copy import deepcopy\n",
        "from collections import Counter\n",
        "import pattern.text.en as en\n",
        "from pattern.text.en import conjugate,INFINITIVE,SG\n",
        "\n",
        "import ioc_fanger  # https://github.com/ioc-fang/ioc-fanger\n",
        "\n",
        "from allennlp.predictors.predictor import Predictor  # https://github.com/allenai/allennlp\n",
        "import allennlp_models.tagging  # https://github.com/allenai/allennlp-models\n",
        "\n",
        "import jamspell  # https://github.com/bakwc/JamSpell\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "import torch\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gg25ltsEZS86"
      },
      "source": [
        "### Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d7lj6s3fLeHT"
      },
      "outputs": [],
      "source": [
        "# JamSpell material\n",
        "if not os.path.exists('en.tar.gz'):\n",
        "  !wget https://github.com/bakwc/JamSpell-models/raw/master/en.tar.gz\n",
        "!tar -xvf en.tar.gz\n",
        "\n",
        "jsp = jamspell.TSpellCorrector()\n",
        "assert jsp.LoadLangModel('en.bin')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BY2IfFLnETbN"
      },
      "outputs": [],
      "source": [
        "# Check if GPU is available\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "    cuda_device = 0  # You can specify the GPU index (0, 1, etc.) you want to use\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "    cuda_device = -1  # If no GPU is available, use CPU"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kxELXzzYnkGU"
      },
      "outputs": [],
      "source": [
        "#Unrelated content removal material\n",
        "model_input_dir = 'Sentence_classifier_BERT'\n",
        "# Load the tokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained(model_input_dir)\n",
        "# Load the fine-tuned model\n",
        "model = BertForSequenceClassification.from_pretrained(model_input_dir)\n",
        "model.to(device)\n",
        "\n",
        "window_length = 128"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5lSmaUlPOUBC"
      },
      "outputs": [],
      "source": [
        "# AllenNLP material\n",
        "predictor = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/structured-prediction-srl-bert.2020.12.15.tar.gz\",\n",
        "                                cuda_device=cuda_device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "udHwJmRZ3BcE"
      },
      "outputs": [],
      "source": [
        "# Stopwords material\n",
        "stop_words = set(stopwords.words('english'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oMHxVavM25RQ"
      },
      "outputs": [],
      "source": [
        "# Internet slang words material\n",
        "internet_slang_words = []\n",
        "\n",
        "with open('dictionary_slang_socialmedia_slangnet.txt', \"r\", encoding='utf-8') as file:\n",
        "    for line in file:\n",
        "        internet_slang_words.append(line.strip().lower())\n",
        "with open('dictionary_slang_webforums_slangnet.txt', \"r\", encoding='utf-8') as file:\n",
        "    for line in file:\n",
        "        internet_slang_words.append(line.strip().lower())\n",
        "internet_slang_words = sorted(set(internet_slang_words))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tPQGxq0Wg3NV"
      },
      "outputs": [],
      "source": [
        "# Aliases material\n",
        "def read_alias_from_csv(filename):\n",
        "  data = []\n",
        "\n",
        "  # Use the 'r' mode to open the CSV file for reading\n",
        "  with open(filename, \"r\", newline=\"\") as file:\n",
        "      reader = csv.reader(file)\n",
        "\n",
        "      # Iterate through each row in the CSV file and append it to the data list\n",
        "      for row in reader:\n",
        "          data.append(row)\n",
        "\n",
        "  for line in data:\n",
        "    line[0] = line[0].upper()\n",
        "  return data\n",
        "\n",
        "actor_groups_aliases = read_alias_from_csv('threat_actors_aliases.csv')\n",
        "malware_families_aliases = read_alias_from_csv('threat_names_aliases.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GFnampqaZ4IY"
      },
      "source": [
        "### Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "APaY7LJ1Lmp1"
      },
      "outputs": [],
      "source": [
        "%cd '/content/drive/My Drive/Steps NLP framework/Pipeline_new_data/Folder_datasets'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "084HwYOXZcZB"
      },
      "source": [
        "## <font color='orange'>Define lists and functions</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wQEX6Qe_aGL0"
      },
      "source": [
        "### Lists"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G21L6Up3YmWr"
      },
      "source": [
        "#### Passive\\Active Conversion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CueheetvY1Nn"
      },
      "outputs": [],
      "source": [
        "pass2act_noundict = {'i':'me', 'we':'us', 'you':'you', 'he':'him', 'she':'her', 'they':'them', 'them':'they', 'her':'she', 'him':'he', 'us':'we', 'me':'i'}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y4tzWWbEcCHD"
      },
      "source": [
        "#### Synonym Homogenization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JBwy4rX5byqu"
      },
      "source": [
        "##### Noun lists"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wSWSDi-lbsqq"
      },
      "outputs": [],
      "source": [
        "appdata_list = ['APPDATA' , 'APPDATA_', \"%APPDATA%\", \"%AppData%\", \"%appdata%\", \"% APPDATA%\", \"% AppData%\", \"% appdata%\",\n",
        "                \"<APPDATA>\", \"<AppData>\", \"<appdata>\", \"% APPDATA %\", \"% AppData %\", \"% appdata %\", \"% APPDATA %\", \"% AppData %\", \"% appdata %\",\n",
        "                \"< APPDATA >\", \"%Appdata%\", \"< AppData >\", \"< appdata >\"]\n",
        "\n",
        "common_appdata_list = ['COMMON_APPDATA' ,\"%COMMON_APPDATA%\", \"%common_appdata%\", \"%Common_Appdata%\", \"%COMMON_APPDATA%\",\n",
        "                \"%common_appdata%\", \"%Common_Appdata%\" \"<COMMON_APPDATA>\", \"<common_appdata>\", \"<Common_Appdata>\",\n",
        "                \"% COMMON_APPDATA %\", \"% common_appdata %\", \"% Common_Appdata %\", \"% COMMON_APPDATA %\",\n",
        "                \"% common_appdata %\", \"% Common_Appdata %\" \"< COMMON_APPDATA >\", \"< common_appdata >\",\n",
        "                \"< Common_Appdata >\"]\n",
        "\n",
        "profilename_list = ['PROFILENAME' ,\"PROFILENAME_\",\"%ProfileName%\", \"%PROFILENAME%\", \"%Profilename%\", \"% ProfileName%\", \"% PROFILENAME%\",\n",
        "                \"% Profilename%\", \"<ProfileName>\", \"<PROFILENAME>\", \"<Profilename>\",\n",
        "                \"% ProfileName %\", \"% PROFILENAME %\", \"% Profilename %\", \"% ProfileName %\", \"% PROFILENAME %\",\n",
        "                \"% Profilename %\", \"< ProfileName >\", \"< PROFILENAME >\", \"< Profilename >\",\n",
        "                \"%profilename%\", \"% profilename%\",\"% profilename %\", \"<profilename>\", \"< profilename>\",\"< profilename >\"]\n",
        "\n",
        "username_list = ['USERNAME' ,\"USERNAME_\",\"%UserName%\", \"%USERNAME%\", \"%username%\", \"% UserName%\", \"% USERNAME%\", \"% username%\",\n",
        "                \"<UserName>\", \"<USERNAME>\", \"<username>\", \"<Username>\",\n",
        "                \"% UserName %\", \"% USERNAME %\", \"% username %\", \"% UserName %\", \"% USERNAME %\", \"% username %\",\n",
        "                \"< UserName >\", \"< USERNAME >\", \"< username >\", \"< Username >\"]\n",
        "\n",
        "temp_list = ['TEMP' ,\"TEMP_\", \"%temp%\", \"%TEMP%\", \"%Temp%\", \"% temp%\", \"% TEMP%\",\n",
        "                \"% Temp%\", \"<temp>\", \"<TEMP>\", \"<Temp>\",\n",
        "                \"% temp %\", \"% TEMP %\", \"% Temp %\", \"% temp %\", \"% TEMP %\",\n",
        "                \"% Temp %\", \"< temp >\", \"< TEMP >\", \"< Temp >\", \"<Windows temporary folder>\",\"<temporary folder>\" ,\"%temporary folder%\" ]\n",
        "\n",
        "userprofile_list = ['USERPROFILE' , \"USERPROFILE_\", \"%UserProfile%\", \"%UserProfile%\", \"%UserProfile%\", \"%USERPROFILE%\",\"%Userprofile%\",\n",
        "                \"%userprofile%\", \"<UserProfile>\", \"<USERPROFILE>\", \"<userprofile>\",\"<Userprofile>\",\n",
        "                \"% UserProfile%\", \"% UserProfile%\", \"% UserProfile%\", \"% USERPROFILE%\",\"< Userprofile>\",\n",
        "                \"% userprofile%\", \"< UserProfile>\", \"< USERPROFILE>\", \"< userprofile>\",\"< Userprofile >\",\n",
        "                \"% UserProfile %\", \"% UserProfile %\", \"% UserProfile %\", \"% USERPROFILE %\",\"% Userprofile%\",\n",
        "                \"% userprofile %\", \"< UserProfile >\", \"< USERPROFILE >\", \"< userprofile >\",\"% Userprofile %\",\n",
        "                \"% UserProfile %\"]\n",
        "\n",
        "systemroot_list = ['SYSTEMROOT' ,\"SYSTEMROOT_\", \"%SystemRoot%\", \"% SystemRoot %\", \"% SystemRoot%\",\"%SYSTEMROOT%\", \"% SYSTEMROOT %\", \"% SYSTEMROOT%\",\n",
        "                \"%systemroot%\", \"% systemroot %\", \"% systemroot%\", \"%Systemroot%\", \"% Systemroot %\", \"% Systemroot%\",\n",
        "                \"<SystemRoot>\", \"< SystemRoot >\", \"< SystemRoot>\", \"<SYSTEMROOT>\", \"< SYSTEMROOT >\",\"< SYSTEMROOT>\",\n",
        "                \"<systemroot>\", \"< systemroot >\", \"< systemroot>\", \"<Systemroot>\", \"< Systemroot >\",\"< Systemroot>\"]\n",
        "\n",
        "windows_list = ['WINDOWS' , \"WINDOWS_\",\"%WINDOWS%\", \"% WINDOWS %\", \"% WINDOWS%\", \"%Windows%\", \"% Windows %\", \"% Windows%\",\"%windows%\", \"% windows %\", \"% windows%\",\n",
        "                \"<WINDOWS>\", \"< WINDOWS >\", \"< WINDOWS>\", \"<Windows>\", \"< Windows >\", \"< Windows>\",\"<windows>\", \"< windows >\", \"< windows>\"]\n",
        "\n",
        "windir_list = ['WINDIR' ,\"WINDIR_\",\"%WINDIR%\", \"% WINDIR %\", \"% WINDIR%\", \"%Windir%\", \"% Windir %\", \"% Windir%\", \"%windir%\",\"% windir %\", \"% windir%\",\n",
        "                \"<WINDIR>\", \"< WINDIR >\", \"< WINDIR>\", \"<Windir>\", \"< Windir >\", \"< Windir>\", \"<windir>\",\"< windir >\", \"< windir>\" ]\n",
        "\n",
        "defaultuserprofile =['DEFAULTUSERPROFILE' ,\"DEFAULTUSERPROFILE_\", \"%DEFAULTUSERPROFILE%\", \"% DEFAULTUSERPROFILE %\", \"% DEFAULTUSERPROFILE%\",\n",
        "                \"%defaultuserprofile%\", \"% defaultuserprofile %\", \"% defaultuserprofile%\",\n",
        "                \"%Defaultuserprofile%\", \"% Defaultuserprofile %\", \"% Defaultuserprofile%\",\n",
        "                \"%DefaultUserProfile%\", \"% DefaultUserProfile %\", \"% DefaultUserProfile%\",\n",
        "                \"<DEFAULTUSERPROFILE>\", \"< DEFAULTUSERPROFILE >\", \"< DEFAULTUSERPROFILE>\",\n",
        "                \"<defaultuserprofile>\", \"< defaultuserprofile >\", \"< defaultuserprofile>\",\n",
        "                \"<Defaultuserprofile>\", \"< Defaultuserprofile >\", \"< Defaultuserprofile>\",\n",
        "                \"<DefaultUserProfile>\", \"< DefaultUserProfile >\", \"< DefaultUserProfile>\"]\n",
        "\n",
        "homepath_list = ['HOMEPATH' ,\"HOMEPATH_\",\"%HOMEPATH%\", \"% HOMEPATH %\", \"% HOMEPATH%\",\n",
        "                \"%HomePath%\", \"% HomePath %\", \"% HomePath%\",\n",
        "                \"%homepath%\", \"% homepath %\", \"% homepath%\",\n",
        "                \"%Homepath%\", \"% Homepath %\", \"% Homepath%\",\n",
        "                \"<HOMEPATH>\", \"< HOMEPATH >\", \"< HOMEPATH>\",\n",
        "                \"<HomePath>\", \"< HomePath >\", \"< HomePath>\",\n",
        "                \"<homepath>\", \"< homepath >\", \"< homepath>\",\n",
        "                \"<Homepath>\", \"< Homepath >\", \"< Homepath>\"]\n",
        "\n",
        "homefolder_list = ['HOMEFOLDER' ,\"HOMEFOLDER_\" ,\"HOMEFOLDER\", \"%HOMEFOLDER%\", \"% HOMEFOLDER %\", \"% HOMEFOLDER%\",\n",
        "                \"%HomeFolder%\", \"% HomeFolder %\", \"% HomeFolder%\",\n",
        "                \"%homefolder%\", \"% homefolder %\", \"% homefolder%\",\n",
        "                \"%Homefolder%\", \"% Homefolder %\", \"% Homefolder%\",\n",
        "                \"<HOMEFOLDER>\", \"< HOMEFOLDER >\", \"< HOMEFOLDER>\",\n",
        "                \"<HomeFolder>\", \"< HomeFolder >\", \"< HomeFolder>\",\n",
        "                \"<homefolder>\", \"< homefolder >\", \"< homefolder>\",\n",
        "                \"<Homefolder>\", \"< Homefolder >\", \"< Homefolder>\"]\n",
        "\n",
        "programfiles_list = ['PROGRAMFILES' ,\"PROGRAMFILES_\", \"%PROGRAMFILES%\", \"% PROGRAMFILES %\", \"% PROGRAMFILES%\",\n",
        "                \"%ProgramFiles%\", \"% ProgramFiles %\", \"% ProgramFiles%\",\n",
        "                \"%programfiles%\", \"% programfiles %\", \"% programfiles%\",\n",
        "                \"%Programfiles%\", \"% Programfiles %\", \"% Programfiles%\",\n",
        "                \"<PROGRAMFILES>\", \"< PROGRAMFILES >\", \"< PROGRAMFILES>\",\n",
        "                \"<ProgramFiles>\", \"< ProgramFiles >\", \"< ProgramFiles>\",\n",
        "                \"<programfiles>\", \"< programfiles >\", \"< programfiles>\",\n",
        "                \"<Programfiles>\", \"< Programfiles >\", \"< Programfiles>\"]\n",
        "\n",
        "programfile_list = ['PROGRAMFILE' ,\"PROGRAMFILE_\", \"%PROGRAMFILE%\", \"% PROGRAMFILE %\", \"% PROGRAMFILE%\",\n",
        "                \"%ProgramFile%\", \"% ProgramFile %\", \"% ProgramFile%\",\n",
        "                \"%programfile%\", \"% programfile %\", \"% programfile%\",\n",
        "                \"%Programfile%\", \"% Programfile %\", \"% Programfile%\",\n",
        "                \"<PROGRAMFILE>\", \"< PROGRAMFILE >\", \"< PROGRAMFILE>\",\n",
        "                \"<ProgramFile>\", \"< ProgramFile >\", \"< ProgramFile>\",\n",
        "                \"<programfile>\", \"< programfile >\", \"< programfile>\",\n",
        "                \"<Programfile>\", \"< Programfile >\", \"< Programfile>\"]\n",
        "\n",
        "systemfolder_list = ['SYSTEMFOLDER' ,\"SYSTEMFOLDER_\",\"%SYSTEMFOLDER%\", \"% SYSTEMFOLDER %\", \"% SYSTEMFOLDER%\",\n",
        "                \"%SystemFolder%\", \"% SystemFolder %\", \"% SystemFolder%\",\n",
        "                \"%systemfolder%\", \"% systemfolder %\", \"% systemfolder%\",\n",
        "                \"%Systemfolder%\", \"% Systemfolder %\", \"% Systemfolder%\",\n",
        "                \"<SYSTEMFOLDER>\", \"< SYSTEMFOLDER >\", \"< SYSTEMFOLDER>\",\n",
        "                \"<SystemFolder>\", \"< SystemFolder >\", \"< SystemFolder>\",\n",
        "                \"<systemfolder>\", \"< systemfolder >\", \"< systemfolder>\",\n",
        "                \"<Systemfolder>\", \"< Systemfolder >\", \"< Systemfolder>\",\n",
        "                \"%SYSTEM FOLDER%\", \"% SYSTEM FOLDER %\", \"% SYSTEM FOLDER%\",\n",
        "                \"%System Folder%\", \"% System Folder %\", \"% System Folder%\",\n",
        "                \"%system folder%\", \"% system folder %\", \"% system folder%\",\n",
        "                \"%System folder%\", \"% System folder %\", \"% System folder%\",\n",
        "                \"<SYSTEM FOLDER>\", \"< SYSTEM FOLDER >\", \"< SYSTEM FOLDER>\",\n",
        "                \"<System Folder>\", \"< System Folder >\", \"< System Folder>\",\n",
        "                \"<system folder>\", \"< system folder >\", \"< system folder>\",\n",
        "                \"<System folder>\", \"< System folder >\", \"< System folder>\"]\n",
        "\n",
        "systemdrives_list = ['SYSTEMDRIVES' ,\"SYSTEMDRIVEs_\", \"%SYSTEMDRIVEs%\", \"% SYSTEMDRIVEs %\", \"% SYSTEMDRIVEs%\",\n",
        "                \"%SystemDrives%\", \"% SystemDrives %\", \"% SystemDrives%\",\n",
        "                \"%systemdrives%\", \"% systemdrives %\", \"% systemdrives%\",\n",
        "                \"%Systemdrives%\", \"% Systemdrives %\", \"% Systemdrives%\",\n",
        "                \"<SYSTEMDRIVEs>\", \"< SYSTEMDRIVEs >\", \"< SYSTEMDRIVEs>\",\n",
        "                \"<SystemDrives>\", \"< SystemDrives >\", \"< SystemDrives>\",\n",
        "                \"<systemdrives>\", \"< systemdrives >\", \"< systemdrives>\",\n",
        "                \"<Systemdrives>\", \"< Systemdrives >\", \"< Systemdrives>\",\n",
        "                \"%SYSTEM DRIVEs%\", \"% SYSTEM DRIVEs %\", \"% SYSTEM DRIVEs%\",\n",
        "                \"%System Drives%\", \"% System Drives %\", \"% System Drives%\",\n",
        "                \"%system drives%\", \"% system drives %\", \"% system drives%\",\n",
        "                \"%System drives%\", \"% System drives %\", \"% System drives%\",\n",
        "                \"<SYSTEM DRIVEs>\", \"< SYSTEM DRIVEs >\", \"< SYSTEM DRIVEs>\",\n",
        "                \"<System Drives>\", \"< System Drives >\", \"< System Drives>\",\n",
        "                \"<system drives>\", \"< system drives >\", \"< system drives>\",\n",
        "                \"<System drives>\", \"< System drives >\", \"< System drives>\"\n",
        "                \"%SYSTEMDRIVE%\", \"% SYSTEMDRIVE %\", \"% SYSTEMDRIVE%\",\n",
        "                \"%SystemDRIVE%\", \"% SystemDRIVE %\", \"% SystemDRIVE%\",\n",
        "                \"%systemDRIVE%\", \"% systemDRIVE %\", \"% systemDRIVE%\",\n",
        "                \"%SystemDRIVE%\", \"% SystemDRIVE %\", \"% SystemDRIVE%\",\n",
        "                \"<SYSTEMDRIVE>\", \"< SYSTEMDRIVE >\", \"< SYSTEMDRIVE>\",\n",
        "                \"<SystemDRIVE>\", \"< SystemDRIVE >\", \"< SystemDRIVE>\",\n",
        "                \"<systemDRIVE>\", \"< systemDRIVE >\", \"< systemDRIVE>\",\n",
        "                \"<SystemDRIVE>\", \"< SystemDRIVE >\", \"< SystemDRIVE>\",\n",
        "                \"%SYSTEM DRIVE%\", \"% SYSTEM DRIVE %\", \"% SYSTEM DRIVE%\",\n",
        "                \"%System DRIVE%\", \"% System DRIVE %\", \"% System DRIVE%\",\n",
        "                \"%system DRIVE%\", \"% system DRIVE %\", \"% system DRIVE%\",\n",
        "                \"%System DRIVE%\", \"% System DRIVE %\", \"% System DRIVE%\",\n",
        "                \"<SYSTEM DRIVE>\", \"< SYSTEM DRIVE >\", \"< SYSTEM DRIVE>\",\n",
        "                \"<System DRIVE>\", \"< System DRIVE >\", \"< System DRIVE>\",\n",
        "                \"<system DRIVE>\", \"< system DRIVE >\", \"< system DRIVE>\"]\n",
        "\n",
        "\n",
        "system_list = [ 'SYSTEM' ,\"SYSTEM_\",\"%SYSTEM%\", \"% SYSTEM %\", \"% SYSTEM%\",\n",
        "                \"%System%\", \"% System %\", \"% System%\",\n",
        "                \"%system%\", \"% system %\", \"% system%\",\n",
        "                \"<SYSTEM>\", \"< SYSTEM >\", \"< SYSTEM>\",\n",
        "                \"<System>\", \"< System >\", \"< System>\",\n",
        "                \"<system>\", \"< system >\", \"< system>\"]\n",
        "\n",
        "system32_list = ['SYSTEM32' ,\"SYSTEM32_\", \"%SYSTEM32%\", \"% SYSTEM32 %\", \"% SYSTEM32%\",\n",
        "                \"%System32%\", \"% System32 %\", \"% System32%\",\n",
        "                \"%system32%\", \"% system32 %\", \"% system32%\",\n",
        "                \"<SYSTEM32>\", \"< SYSTEM32 >\", \"< SYSTEM32>\",\n",
        "                \"<System32>\", \"< System32 >\", \"< System32>\",\n",
        "                \"<system32>\", \"< system32 >\", \"< system32>\"]\n",
        "\n",
        "empty_list = ['EMPTY' ,\"EMPTY_\", \"%EMPTY%\", \"% EMPTY %\", \"% EMPTY%\",\n",
        "                \"%Empty%\", \"% Empty %\", \"% Empty%\",\n",
        "                \"%empty%\", \"% empty %\", \"% empty%\",\n",
        "                \"<EMPTY>\", \"< EMPTY >\", \"< EMPTY>\",\n",
        "                \"<Empty>\", \"< Empty >\", \"< Empty>\",\n",
        "                \"<empty>\", \"< empty >\", \"< empty>\"]\n",
        "\n",
        "random_letters_list = ['RANDOM_LETTERS' ,'RANDOM_LETTER', \"%RANDOM LETTERS%\", \"% RANDOM LETTERS %\", \"% RANDOM LETTERS%\",\n",
        "                \"%Random Letters%\", \"% Random Letters %\", \"% Random Letters%\",\n",
        "                \"%random letters%\", \"% random letters %\", \"% random letters%\",\n",
        "                \"<RANDOM LETTERS>\", \"< RANDOM LETTERS >\", \"< RANDOM LETTERS>\",\n",
        "                \"<Random Letters>\", \"< Random Letters >\", \"< Random Letters>\",\n",
        "                \"<random letters>\", \"< random letters >\", \"< random letters>\",\n",
        "                \"%RANDOM LETTER%\", \"% RANDOM LETTER %\", \"% RANDOM LETTER%\",\n",
        "                \"%Random Letter%\", \"% Random Letter %\", \"% Random Letter%\",\n",
        "                \"%random letter%\", \"% random letter %\", \"% random letter%\",\n",
        "                \"<RANDOM LETTER>\", \"< RANDOM LETTER >\", \"< RANDOM LETTER>\",\n",
        "                \"<Random Letter>\", \"< Random Letter >\", \"< Random Letter>\",\n",
        "                \"<random letter>\", \"< random letter >\", \"< random letter>\",\n",
        "                \"%RANDOM_LETTERS%\", \"% RANDOM_LETTERS %\", \"% RANDOM_LETTERS%\",\n",
        "                \"%Random_Letters%\", \"% Random_Letters %\", \"% Random_Letters%\",\n",
        "                \"%random_letters%\", \"% random_letters %\", \"% random_letters%\",\n",
        "                \"<RANDOM_LETTERS>\", \"< RANDOM_LETTERS >\", \"< RANDOM_LETTERS>\",\n",
        "                \"<Random_Letters>\", \"< Random_Letters >\", \"< Random_Letters>\",\n",
        "                \"<random_letters>\", \"< random_letters >\", \"< random_letters>\",\n",
        "                \"%RANDOM_LETTER%\", \"% RANDOM_LETTER %\", \"% RANDOM_LETTER%\",\n",
        "                \"%Random_Letter%\", \"% Random_Letter %\", \"% Random_Letter%\",\n",
        "                \"%random_letter%\", \"% random_letter %\", \"% random_letter%\",\n",
        "                \"<RANDOM_LETTER>\", \"< RANDOM_LETTER >\", \"< RANDOM_LETTER>\",\n",
        "                \"<Random_Letter>\", \"< Random_Letter >\", \"< Random_Letter>\",\n",
        "                \"<random_letter>\", \"< random_letter >\", \"< random_letter>\"]\n",
        "\n",
        "random_numbers_list = ['RANDOM_NUMBERS' ,'RANDOM_NUMBER', \"%RANDOM NUMBERS%\", \"% RANDOM NUMBERS %\", \"% RANDOM NUMBERS%\",\n",
        "                \"%Random Numbers%\", \"% Random Numbers %\", \"% Random Numbers%\",\n",
        "                \"%random numbers%\", \"% random numbers %\", \"% random numbers%\",\n",
        "                \"<RANDOM NUMBERS>\", \"< RANDOM NUMBERS >\", \"< RANDOM NUMBERS>\",\n",
        "                \"<Random Numbers>\", \"< Random Numbers >\", \"< Random Numbers>\",\n",
        "                \"<random numbers>\", \"< random numbers >\", \"< random numbers>\",\n",
        "                \"%RANDOM NUMBER%\", \"% RANDOM NUMBER %\", \"% RANDOM NUMBER%\",\n",
        "                \"%Random Number%\", \"% Random Number %\", \"% Random Number%\",\n",
        "                \"%random number%\", \"% random number %\", \"% random number%\",\n",
        "                \"<RANDOM NUMBER>\", \"< RANDOM NUMBER >\", \"< RANDOM NUMBER>\",\n",
        "                \"<Random Number>\", \"< Random Number >\", \"< Random Number>\",\n",
        "                \"<random number>\", \"< random number >\", \"< random number>\",\n",
        "                \"%RANDOM_NUMBERS%\", \"% RANDOM_NUMBERS %\", \"% RANDOM_NUMBERS%\",\n",
        "                \"%Random_Numbers%\", \"% Random_Numbers %\", \"% Random_Numbers%\",\n",
        "                \"%random_numbers%\", \"% random_numbers %\", \"% random_numbers%\",\n",
        "                \"<RANDOM_NUMBERS>\", \"< RANDOM_NUMBERS >\", \"< RANDOM_NUMBERS>\",\n",
        "                \"<Random_Numbers>\", \"< Random_Numbers >\", \"< Random_Numbers>\",\n",
        "                \"<random_numbers>\", \"< random_numbers >\", \"< random_numbers>\",\n",
        "                \"%RANDOM_NUMBER%\", \"% RANDOM_NUMBER %\", \"% RANDOM_NUMBER%\",\n",
        "                \"%Random_Number%\", \"% Random_Number %\", \"% Random_Number%\",\n",
        "                \"%random_number%\", \"% random_number %\", \"% random_number%\",\n",
        "                \"<RANDOM_NUMBER>\", \"< RANDOM_NUMBER >\", \"< RANDOM_NUMBER>\",\n",
        "                \"<Random_Number>\", \"< Random_Number >\", \"< Random_Number>\",\n",
        "                \"<random_number>\", \"< random_number >\", \"< random_number>\"]\n",
        "\n",
        "command_and_control_list = [\"COMMAND_AND_CONTROL\", \"c&c server\", \"c&c\",\n",
        "                \"command and control sever\", \"command and control\",\n",
        "                \"c2 server\", \"C2 server\", \"c2\", \"C2\",\n",
        "                \"candc server\", \"candc\", \"cc server\", \"CC server\",\n",
        "                \"command & control sever\", \"command & control\",\n",
        "                \"command & controle sever\", \"Command & Controle sever\", \"Command & Controle\",\n",
        "                \"CandC server\", \"CandC\", \"CnC server\", \"CnC\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dq0GuG8sb3_H"
      },
      "source": [
        "##### Phrasal Verb lists"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kCddrav9bsqs"
      },
      "outputs": [],
      "source": [
        "try_going_to_list = ['tries to', 'try to', 'tried to', 'trying to',\n",
        "               'attempts to' , 'attempt to', 'attempted to', 'attempting to',\n",
        "               'am going to', 'is going to', 'are going to', 'was going to', 'were going to']\n",
        "\n",
        "capable_of_list = ['is capable of', 'are capable of', 'was capable of', 'were capable of', 'being capable of']\n",
        "\n",
        "make_modification_list = ['makes the following registry modifications', 'make the following registry modifications',\n",
        "                          'made the following registry modifications', 'making the following registry modifications',\n",
        "                          'makes the following registry modification', 'make the following registry modification',\n",
        "                          'made the following registry modification', 'making the following registry modification',\n",
        "                          'makes the following additional registry modifications', 'make the following additional registry modifications',\n",
        "                          'made the following additional registry modifications', 'making the following additional registry modifications',\n",
        "                          'makes the following additional registry modification', 'make the following additional registry modification',\n",
        "                          'made the following additional registry modification', 'making the following additional registry modification',\n",
        "                          'does the following registry modifications', 'do the following registry modifications',\n",
        "                          'did the following registry modifications', 'doing the following registry modifications',\n",
        "                          'does the following registry modification', 'do the following registry modification',\n",
        "                          'did the following registry modification', 'doing the following registry modification',\n",
        "                          'by making the following registry modifications', 'by making the following registry modification',\n",
        "                          'makes a further registry modification', 'make a further registry modification',\n",
        "                          'made a further registry modification', 'making a further registry modification',\n",
        "                          'performs a further registry modification', 'perform a further registry modification',\n",
        "                          'performed a further registry modification', 'performing a further registry modification',\n",
        "                          'does a further registry modification', 'do a further registry modification',\n",
        "                          'did a further registry modification', 'doing a further registry modification',\n",
        "                          'makes the following modifications to', 'make the following modifications to',\n",
        "                          'made the following modifications', 'making the following modifications to',\n",
        "                          'makes the following modification to', 'make the following modification to',\n",
        "                          'made the following modification to', 'making the following modification to',\n",
        "                          'makes following modification to', 'make following modification to',\n",
        "                          'made following modification to', 'making following modification to',\n",
        "                          'makes following modifications to', 'make following modifications to',\n",
        "                          'made following modifications to', 'making following modifications to', ]\n",
        "\n",
        "the_following_list = ['the following possibly malicious websites', 'The following possibly malicious websites',\n",
        "                      'the following possibly malicious sites', 'The following possibly malicious sites', 'the following folders and subfolders',\n",
        "                      'The following folders and subfolders', 'the following possibly malicious URL', 'The following possibly malicious URL',\n",
        "                      'the following folders and file(s)', 'The following folders and file(s)', 'the following malicious websites',\n",
        "                      'The following malicious websites', 'the following registry locations', 'The following registry locations',\n",
        "                      'the following registry location', 'The following registry location', 'the following folders and files',\n",
        "                      'The following folders and files', 'the following registry subkeys', 'The following registry subkeys',\n",
        "                      'the following copies of itself', 'The following copies of itself', 'the following registry entries',\n",
        "                      'The following registry entries', 'the following folders and file', 'The following folders and file',\n",
        "                      'the following registry subkey', 'The following registry subkey', 'the following registry values', 'The following registry values',\n",
        "                      'the following malicious sites', 'The following malicious sites', 'the following subdirectories', 'The following subdirectories',\n",
        "                      'the following registry entry', 'The following registry entry', 'the following copy of itself', 'The following copy of itself',\n",
        "                      'the following registry value', 'The following registry value', 'the following registry paths', 'The following registry paths',\n",
        "                      'the following registry keys', 'The following registry keys', 'the following malicious URL', 'The following malicious URL',\n",
        "                      'the following registry path', 'The following registry path', 'the following hash file(s)', 'The following hash file(s)',\n",
        "                      'the following IP addresses', 'The following IP addresses', 'the following registry key', 'The following registry key',\n",
        "                      'the following directories', 'The following directories', 'the following subfolders:', 'The following subfolders:',\n",
        "                      'the following web servers', 'The following web servers', 'the following web server', 'The following web server',\n",
        "                      'the following hash files', 'The following hash files', 'the following IP address', 'The following IP address',\n",
        "                      'the following copies of', 'The following copies of', 'the following webservers', 'The following webservers', 'the following directory',\n",
        "                      'The following directory', 'the following web sites', 'The following web sites', 'the following webserver', 'The following webserver',\n",
        "                      'the following locations', 'The following locations', 'the following hash file', 'The following hash file', 'the following addresses',\n",
        "                      'The following addresses', 'the following copy of', 'The following copy of', 'the following web site', 'The following web site',\n",
        "                      'the following websites', 'The following websites', 'the following location', 'The following location', 'the following registry',\n",
        "                      'The following registry', 'the below directories', 'The below directories', 'the following servers', 'The following servers',\n",
        "                      'the following folders', 'The following folders', 'the following file(s)', 'The following file(s)', 'the following website',\n",
        "                      'The following website', 'the following address', 'The following address', 'the following folder', 'The following folder',\n",
        "                      'the following hashes', 'The following hashes', 'the following server', 'The following server', 'the following paths', 'The following paths',\n",
        "                      'the following sites', 'The following sites', 'the following files', 'The following files', 'the following lines', 'The following lines',\n",
        "                      'the following items', 'The following items', 'the below locations', 'The below locations', 'the below directory', 'The below directory',\n",
        "                      'the following URLs', 'The following URLs', 'the below websites', 'The below websites', 'the below registry', 'The below registry',\n",
        "                      'the following file', 'The following file', 'the following path', 'The following path', 'the below location', 'The below location',\n",
        "                      'the following line', 'The following line', 'the registry value', 'The registry value', 'the following site', 'The following site',\n",
        "                      'the following hash', 'The following hash', 'the following IPs', 'The following IPs', 'the below file(s)', 'The below file(s)',\n",
        "                      'the following URL', 'The following URL', 'the following IP', 'The following IP', 'the below files', 'The below files', 'the below paths',\n",
        "                      'The below paths', 'the below path', 'The below path', 'the below file', 'The below file', 'the following', 'The following', 'the below IPs',\n",
        "                      'The below IPs', 'the registry', 'The registry', 'following folders and subfolders', 'Following folders and subfolders',\n",
        "                      'following folders and file(s)', 'Following folders and file(s)', 'following registry locations', 'Following registry locations',\n",
        "                      'following registry location', 'Following registry location', 'following folders and files', 'Following folders and files',\n",
        "                      'following registry subkeys', 'Following registry subkeys', 'following folders and file', 'Following folders and file',\n",
        "                      'following registry entries', 'Following registry entries', 'following registry subkey', 'Following registry subkey',\n",
        "                      'following registry entry', 'Following registry entry', 'following directories', 'Following directories', 'following subfolders',\n",
        "                      'Following subfolders', 'following directory', 'Following directory', 'following locations', 'Following locations', 'following location',\n",
        "                      'Following location', 'following file(s)', 'Following file(s)', 'below directories', 'Below directories', 'following folders',\n",
        "                      'Following folders', 'following folder', 'Following folder', 'below directory', 'Below directory', 'below locations', 'Below locations',\n",
        "                      'following files', 'Following files', 'following paths', 'Following paths', 'registry value', 'Registry value', 'following URLs',\n",
        "                      'Following URLs', 'following file', 'Following file', 'following path', 'Following path', 'below location', 'Below location', 'below websites',\n",
        "                      'Below websites', 'following URL', 'Following URL', 'below file(s)', 'Below file(s)', 'below paths', 'Below paths', 'below files',\n",
        "                      'Below files', 'below file', 'Below file', 'below path', 'Below path', 'below IPs', 'Below IPs']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LkcYxKv6b7GX"
      },
      "source": [
        "##### Verb lists"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fbk1dakQbsqt"
      },
      "outputs": [],
      "source": [
        "verbs_lists_homogenization = [\n",
        "    ['unlink', 'delete', 'clear', 'remove', 'erase', 'wipe', 'purge', 'expunge'],\n",
        "    ['write', 'entrench', 'entrenches', 'entrenched', 'exfiltrate', 'exfiltrates', 'exfiltrated', 'exfil', 'exfils', 'exfiled', 'store', 'drop', 'install', 'place', 'deploy', 'implant', 'putfile', 'compose', 'create', 'copy', 'save', 'add', 'append', 'form'],\n",
        "    ['read', 'survey', 'download', 'navigate', 'locate', 'gather', 'extract', 'obtain', 'acquire', 'check', 'detect', 'record'],\n",
        "    ['exec', 'use', 'execute', 'run', 'launch', 'call', 'perform', 'list', 'invoke', 'inject', 'open', 'target', 'resume'],\n",
        "    ['mmap', 'mmaps', 'mmap\\'d', 'mmaped', 'allocate', 'assign'],\n",
        "    ['fork', 'clone', 'spawn', 'issue', 'set'],\n",
        "    ['setuid', 'elevate', 'impersonate'],\n",
        "    ['send', 'transfer', 'post', 'postsinformation', 'postsinformations', 'move', 'transmit', 'deliver', 'push', 'redirect', 'beacon', 'beacons', 'beaconed'],\n",
        "    ['receive', 'accept', 'take', 'get', 'collect'],\n",
        "    ['connect', 'click', 'browse', 'portscan', 'alert', 'communicate'],\n",
        "    ['change', 'modify'],\n",
        "    ['exit', 'terminate', 'stop', 'end', 'finish', 'abort', 'conclude']\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-sgvVvsz7AWK"
      },
      "source": [
        "#### Pronouns and Subject ellipsis resolution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ibg_yTP3aT3p"
      },
      "outputs": [],
      "source": [
        "ellipsis_verbs = ['makes', 'make' 'tries', 'Try' ,'adds', 'add', 'runs', 'run', 'deletes',\n",
        "                  'delete', 'removes', 'remove', 'registers', 'register', 'replicates',\n",
        "                  'replicate', 'creates', 'create', 'executes', 'execute', 'modifies',\n",
        "                  'modify', 'downloads', 'download', 'spreads', 'spread', 'conducts',\n",
        "                  'conduct', 'copies', 'copy', 'forks', 'fork', 'writes', 'write', 'reads',\n",
        "                  'read', 'retrieves', 'retrieve', 'redirects', 'redirect', 'wipes', 'wipe',\n",
        "                  'exfiltrates', 'exfiltrate', 'installs', 'install', 'deploys', 'deploy',\n",
        "                  'clears', 'clear', 'erases', 'erase', 'drops', 'drop', 'entrenches',\n",
        "                  'entrench', 'runs', 'run', 'collects', 'collect', 'writes', 'write',\n",
        "                  'locates', 'locate', 'allocates', 'allocate', 'clones', 'clone', 'uses',\n",
        "                  'use', 'performs', 'perform', 'spawns', 'spawn', 'issues', 'issue', 'sets',\n",
        "                  'set', 'clones', 'clone', 'executes', 'execute', 'launches', 'launch',\n",
        "                  'saves', 'save', 'adds', 'add', 'extracts', 'extract', 'gets', 'get',\n",
        "                  'injects', 'inject', 'obtains', 'obtain', 'gathers', 'gather', 'downloads',\n",
        "                  'download', 'beacons', 'beacon', 'places', 'place', 'navigates', 'navigate',\n",
        "                  'composes', 'compose', 'acquires', 'acquire', 'browses', 'browse',\n",
        "                  'performs', 'perform', 'opens', 'open', 'sends', 'send', 'targets',\n",
        "                  'target', 'accepts', 'accept', 'receives', 'receive', 'transfers',\n",
        "                  'transfer', 'invokes', 'invoke', 'modify', 'modifies', 'connects',\n",
        "                  'connect', 'communicates', 'communicate', 'posts', 'post', 'propagates',\n",
        "                  'propagate', 'terminates', 'terminate', 'monitors', 'monitor', 'attempts',\n",
        "                  'attempt', 'generates', 'generate', 'searches', 'search', 'contains',\n",
        "                  'contain', 'hides', 'hide', 'infects', 'infect', 'appends', 'append',\n",
        "                  'closes', 'close', 'checks', 'check']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hgtj4XovaIZz"
      },
      "source": [
        "### Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Elfj18hcMe3"
      },
      "source": [
        "#### Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QpL9vtpHZn2n"
      },
      "outputs": [],
      "source": [
        "# Lib conjugate has some issues at first iteration\n",
        "def pattern_stopiteration_workaround():\n",
        "    try:\n",
        "        en.tenses('crying')\n",
        "    except:\n",
        "        pass\n",
        "pattern_stopiteration_workaround()\n",
        "\n",
        "def pattern_stopiteration_workaround():\n",
        "    try:\n",
        "        conjugate(verb='crying',tense=INFINITIVE)\n",
        "    except:\n",
        "        pass\n",
        "pattern_stopiteration_workaround()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UEhiIgL-cxaX"
      },
      "source": [
        "#### <font color='yellow'>Sanitization</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aGSnKhb7n6TR"
      },
      "source": [
        "##### Unrelated content removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qLHcYToVoAqM"
      },
      "outputs": [],
      "source": [
        "def analyze_sentence(sentence, tags='', update_tags=False):\n",
        "  # Apply tokenization as for the training part\n",
        "  inputs = tokenizer.encode_plus(\n",
        "      sentence,\n",
        "      add_special_tokens = True,      # Add '[CLS]' and '[SEP]'\n",
        "      max_length = window_length,     # Pad & truncate all sentences\n",
        "      pad_to_max_length = True,\n",
        "      return_attention_mask = True,   # Construct attn. masks\n",
        "      return_tensors = 'pt',          # Return pytorch tensors\n",
        "  )\n",
        "\n",
        "  inputs = {key: value.to(device) for key, value in inputs.items()}\n",
        "\n",
        "  # Make the prediction\n",
        "  with torch.no_grad():\n",
        "      outputs = model(**inputs)\n",
        "      logits = outputs.logits\n",
        "\n",
        "  probabilities = torch.softmax(logits, dim=1)\n",
        "  predicted_class = torch.argmax(probabilities, dim=1).item()\n",
        "\n",
        "  if predicted_class == 1:\n",
        "    if update_tags:\n",
        "      return sentence, tags\n",
        "    return sentence\n",
        "  elif predicted_class == 0:\n",
        "    if update_tags:\n",
        "      return '', ''\n",
        "    return ''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JiUZpjScaSDZ"
      },
      "source": [
        "##### IOC Defanging"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "joP4lqmmaT8M"
      },
      "outputs": [],
      "source": [
        "def defang_iocs_in_text(sentence):\n",
        "    defanged_text = ioc_fanger.defang(sentence)\n",
        "    return defanged_text\n",
        "\n",
        "def fang_iocs_in_text(sentence):\n",
        "    fanged_text = ioc_fanger.fang(sentence)\n",
        "    return fanged_text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9ICuzEcamgB"
      },
      "source": [
        "##### Misspelling Correction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rYj9adYLaoUp"
      },
      "outputs": [],
      "source": [
        "def spell_check(sentence):\n",
        "  s_split = sentence.split()\n",
        "  text = ' '.join(s_split[:150])\n",
        "  remaining = ' '.join(s_split[150:])\n",
        "  text = jsp.FixFragment(text) + ' ' + remaining\n",
        "  return text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQcmSkr2c1e5"
      },
      "source": [
        "#### <font color='yellow'>Text Normalization</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yEV5sn9CZLDw"
      },
      "source": [
        "##### Passive\\Active Conversion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TNlcNSEKZMmg"
      },
      "outputs": [],
      "source": [
        "def pass2act_nouninv(noun):\n",
        "    n = noun.lower()\n",
        "    if n in pass2act_noundict:\n",
        "        return pass2act_noundict[n]\n",
        "    return noun"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BSFF71q2ZmQ5"
      },
      "outputs": [],
      "source": [
        "def pass2act(doc, rec=False):\n",
        "  parse = pass2act_nlp(doc)\n",
        "  newdoc = ''\n",
        "  is_passive = False\n",
        "  for sent in parse.sents:\n",
        "\n",
        "      # Init parts of sentence to capture:\n",
        "      subjpass = ''\n",
        "      subj = ''\n",
        "      verb = ''\n",
        "      verbtense = ''\n",
        "      adverb = {'bef':'', 'aft':''}\n",
        "      part = ''\n",
        "      prep = ''\n",
        "      agent = ''\n",
        "      aplural = False\n",
        "      advcltree = None\n",
        "      aux = list(list(pass2act_nlp('. .').sents)[0]) # start with 2 'null' elements\n",
        "      xcomp = ''\n",
        "      punc = '.'\n",
        "      # Analyse dependency tree:\n",
        "      for word in sent:\n",
        "          if word.dep_ == 'advcl':\n",
        "              if word.head.dep_ in ('ROOT', 'auxpass'):\n",
        "                  advcltree = word.subtree\n",
        "          if word.dep_ == 'nsubjpass':\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  subjpass = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "          if word.dep_ == 'nsubj':\n",
        "              subj = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "              if word.head.dep_ == 'auxpass':\n",
        "                  if word.head.head.dep_ == 'ROOT':\n",
        "                      subjpass = subj\n",
        "          if word.dep_ in ('advmod','npadvmod','oprd'):\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  if verb == '':\n",
        "                      adverb['bef'] = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "                  else:\n",
        "                      adverb['aft'] = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "          if word.dep_ == 'auxpass':\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  if not subjpass:\n",
        "                      subjpass = subj\n",
        "          if word.dep_ in ('aux','auxpass','neg'):\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  aux += [word]\n",
        "          if word.dep_ == 'ROOT':\n",
        "              verb = word.text\n",
        "              if word.tag_ == 'VB':\n",
        "                  verbtense = en.INFINITIVE\n",
        "              elif word.tag_ == 'VBD':\n",
        "                  verbtense = en.PAST\n",
        "              elif word.tag_ == 'VBG':\n",
        "                  verbtense = en.PRESENT\n",
        "                  verbaspect = en.PROGRESSIVE\n",
        "              elif word.tag_ == 'VBN':\n",
        "                  verbtense = en.PAST\n",
        "              else:\n",
        "                  try:\n",
        "                      verbtense = en.tenses(word.text)[0][0]\n",
        "                  except IndexError:\n",
        "                      pass\n",
        "          if word.dep_ == 'prt':\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  part = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "          if word.dep_ == 'prep':\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  prep = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "          if word.dep_.endswith('obj'):\n",
        "              if word.head.dep_ == 'agent':\n",
        "                  if word.head.head.dep_ == 'ROOT':\n",
        "                      agent = ''.join(w.text + ', ' if w.dep_=='appos' else (w.text_with_ws) for w in word.subtree).strip()\n",
        "                      aplural = word.tag_ in ('NNS','NNPS')\n",
        "          if word.dep_ in ('xcomp','ccomp','conj'):\n",
        "              if word.head.dep_ == 'ROOT':\n",
        "                  xcomp = ''.join(w.text_with_ws for w in word.subtree).strip()\n",
        "                  that = xcomp.startswith('that')\n",
        "                  #xcomp = pass2act(xcomp, True).strip(' .')\n",
        "                  xcomp, check_is_passive = pass2act(xcomp, True)\n",
        "                  xcomp = xcomp.strip(' .')\n",
        "                  if not is_passive:\n",
        "                    is_passive = check_is_passive\n",
        "                  if not xcomp.startswith('that') and that:\n",
        "                      xcomp = 'that '+xcomp\n",
        "          if word.dep_ == 'punct' and not rec:\n",
        "              if word.text != '\"':\n",
        "                  punc = word.text\n",
        "\n",
        "      # exit if not passive:\n",
        "      if subjpass == '':\n",
        "          newdoc += str(sent) + ' '\n",
        "          continue\n",
        "\n",
        "      # if no agent is found:\n",
        "      if agent == '':\n",
        "          # what am I gonna do? BITconEEEEEEECT!!!!\n",
        "          newdoc += str(sent) + ' '\n",
        "          continue\n",
        "\n",
        "      # invert nouns:\n",
        "      is_passive = True\n",
        "      agent = pass2act_nouninv(agent)\n",
        "      subjpass = pass2act_nouninv(subjpass)\n",
        "\n",
        "      # FUCKING CONJUGATION!!!!!!!!!!!!!:\n",
        "      auxstr = ''\n",
        "      num = en.SINGULAR if not aplural or agent in ('he','she') else en.PLURAL\n",
        "      aux.append(aux[0])\n",
        "      verbaspect = None\n",
        "\n",
        "      for (pp, p, a, n) in zip(aux,aux[1:],aux[2:],aux[3:]):\n",
        "          if a.lemma_ == '.':\n",
        "              continue\n",
        "\n",
        "          if a.lemma_ == 'not':\n",
        "              if p.lemma_ == 'be':\n",
        "                  if n.lemma_ == 'be':\n",
        "                      verbtense = en.tenses(a.text)[0][0]\n",
        "                      auxstr += en.conjugate('be',tense=en.tenses(p.text)[0][0],number=num) + ' '\n",
        "                      verbaspect = en.PROGRESSIVE\n",
        "                  else:\n",
        "                    if len(en.tenses(p.text)) > 0 and len(en.tenses(p.text)[0]) > 0:\n",
        "                      auxstr += en.conjugate('do',tense=en.tenses(p.text)[0][0],number=num) + ' '\n",
        "                    else:\n",
        "                      auxstr += p.text + ' '\n",
        "                    verbtense = en.INFINITIVE\n",
        "              auxstr += 'not '\n",
        "          elif a.lemma_ == 'be':\n",
        "              if p.lemma_ == 'be':\n",
        "                  verbtense = en.tenses(a.text)[0][0]\n",
        "                  auxstr += en.conjugate('be',tense=en.tenses(a.text)[0][0],number=num) + ' '\n",
        "                  verbaspect = en.PROGRESSIVE\n",
        "              elif p.tag_ == 'MD':\n",
        "                  verbtense = en.INFINITIVE\n",
        "          elif a.lemma_ == 'have':\n",
        "              num == en.PLURAL if p.tag_ == 'MD' else num\n",
        "              if len(en.tenses(a.text)) > 0:\n",
        "                auxstr += en.conjugate('have',tense=en.tenses(a.text)[0][0],number=num) + ' '\n",
        "              else:\n",
        "                auxstr += a.text + ' '\n",
        "              if n.lemma_ == 'be':\n",
        "                  verbaspect = en.PROGRESSIVE\n",
        "                  verbtense = en.tenses(n.text)[0][0]\n",
        "          else:\n",
        "              auxstr += a.text_with_ws\n",
        "      auxstr = auxstr.lower().strip()\n",
        "\n",
        "      if verbaspect:\n",
        "          verb = en.conjugate(verb,tense=verbtense,aspect=verbaspect)\n",
        "      else:\n",
        "          verb = en.conjugate(verb,tense=verbtense)\n",
        "\n",
        "      advcl = ''\n",
        "      if advcltree:\n",
        "          for w in advcltree:\n",
        "              if w.pos_ == 'VERB' and len(en.tenses(w.text)) > 0 and len(en.tenses(w.text)[0]) > 0 and en.tenses(w.text)[0][4] == en.PROGRESSIVE:\n",
        "                  advcl += 'which ' + en.conjugate(w.text,tense=en.tenses(verb)[0][0]) + ' '\n",
        "              else:\n",
        "                  advcl += w.text_with_ws\n",
        "\n",
        "      newsent = ' '.join(list(filter(None, [agent,auxstr,adverb['bef'],verb,part,subjpass,adverb['aft'],advcl,prep,xcomp])))+punc\n",
        "      if not rec:\n",
        "          newsent = newsent[0].upper() + newsent[1:]\n",
        "      newdoc += newsent + ' '\n",
        "  return newdoc, is_passive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A-vJ0oA_Z4bi"
      },
      "outputs": [],
      "source": [
        "# Define a function to remove punctuation\n",
        "def remove_punctuation(sentence):\n",
        "  return ''.join(char for char in sentence if char not in string.punctuation)\n",
        "\n",
        "def convert_active_to_passive(sentence, tags='', update_tags=False):\n",
        "  stripped_sentence = ' '.join([word for word in sentence.split() if word not in string.punctuation])  # Strip tokens of single punctuation\n",
        "\n",
        "  # Apply conversion\n",
        "  new_sentence, is_passive = pass2act(stripped_sentence)\n",
        "\n",
        "  if not is_passive or new_sentence == stripped_sentence: # Return original sentence if no modification\n",
        "    if update_tags:\n",
        "      return sentence, tags\n",
        "    return sentence\n",
        "\n",
        "  # If there are modifications, use the stripped sentence\n",
        "  if update_tags:\n",
        "    stripped_tags = []\n",
        "    tags_split = tags.split(',')\n",
        "    for idx, word in enumerate(sentence.split()):\n",
        "      if word not in string.punctuation:\n",
        "        stripped_tags.append(tags_split[idx])\n",
        "\n",
        "  sentence = stripped_sentence\n",
        "\n",
        "  if update_tags:\n",
        "    tags_split = stripped_tags\n",
        "    tags = ','.join(stripped_tags)\n",
        "    new_tags_split = []\n",
        "\n",
        "    \"\"\"print('sentence', sentence, '-', len(sentence.split()))\n",
        "    print('tags_split', tags_split, '-', len(tags_split))\n",
        "    print()\"\"\"\n",
        "\n",
        "    new_sentence.split()\n",
        "    new_sentence = ' '.join([word for word in new_sentence.split() if word not in string.punctuation])\n",
        "\n",
        "    # Remove punctuation from both sentences\n",
        "    cleaned_sentence1 = remove_punctuation(sentence).lower()\n",
        "    cleaned_sentence2 = remove_punctuation(new_sentence).lower()\n",
        "\n",
        "    # Tokenize the cleaned sentences\n",
        "    tokens1 = cleaned_sentence1.split()\n",
        "    tokens2 = cleaned_sentence2.split()\n",
        "\n",
        "    # Find the common words between the two sentences\n",
        "    common_words = set(tokens1) & set(tokens2)\n",
        "\n",
        "    # Find words that are in sentence 1 but not in sentence 2 (deleted words)\n",
        "    deleted_words = set(tokens1) - set(tokens2)\n",
        "    deleted_words = [word for word in deleted_words if word not in string.punctuation]\n",
        "\n",
        "    # Check if in deleted words there is any of noundict that was changed in active form\n",
        "    pronouns_changed = False\n",
        "    for d_w in deleted_words:\n",
        "      if d_w in pass2act_noundict:\n",
        "        pronouns_changed = True\n",
        "        d_w_changed = pass2act_noundict[d_w]\n",
        "        original_position = tokens1.index(d_w)\n",
        "        tokens1[original_position] = d_w_changed\n",
        "\n",
        "    if pronouns_changed:  # If there is at least a pronoun, do some steps again\n",
        "      # Find the common words between the two sentences\n",
        "      common_words = set(tokens1) & set(tokens2)\n",
        "\n",
        "      # Find words that are in sentence 1 but not in sentence 2 (deleted words)\n",
        "      deleted_words = set(tokens1) - set(tokens2)\n",
        "      deleted_words = [word for word in deleted_words if word not in string.punctuation]\n",
        "\n",
        "    # Find words that are in sentence 2 but not in sentence 1 (added words)\n",
        "    added_words = set(tokens2) - set(tokens1)\n",
        "    added_words = [word for word in added_words if word not in string.punctuation]\n",
        "\n",
        "    # If there are added words, the only possibility is that is the verb that changed form, so check if it hase the same lemmatized form of a deleted word\n",
        "    verbs_changed = False\n",
        "    add_not_token = False\n",
        "    if added_words:\n",
        "        added_words_lower = [word.lower() for word in added_words]\n",
        "        deleted_words_lower = [word.lower() for word in deleted_words]\n",
        "        tokens1_lower = [word.lower() for word in tokens1]\n",
        "\n",
        "        for word in added_words:\n",
        "            #print(f'\\'{word}\\' is added')\n",
        "\n",
        "            # Check particular modal cases\n",
        "            if word.lower() == 'canot':\n",
        "              d_w = 'cant'\n",
        "              d_w_changed = word\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: '{word}' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              continue\n",
        "            elif word.lower() == 'couldnot':\n",
        "              d_w = 'couldnt'\n",
        "              d_w_changed = word\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: '{word}' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              continue\n",
        "            elif word.lower() == 'mustnot':\n",
        "              d_w = 'mustnt'\n",
        "              d_w_changed = word\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: '{word}' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              continue\n",
        "            elif word.lower() == 'shouldnot':\n",
        "              d_w = 'shouldnt'\n",
        "              d_w_changed = word\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: '{word}' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              continue\n",
        "\n",
        "            # Check lemmatization\n",
        "            lemm_added_word = WordNetLemmatizer().lemmatize(word.lower(), 'v')\n",
        "            for d_w in deleted_words:\n",
        "              lemm_deleted_word = WordNetLemmatizer().lemmatize(d_w.lower(), 'v')\n",
        "              if lemm_added_word == lemm_deleted_word:\n",
        "                #print(f'MATCH! : {word} --> {lemm_added_word} == {lemm_deleted_word} <-- {d_w}')\n",
        "                verbs_changed = True\n",
        "                d_w_changed = word\n",
        "                original_position = tokens1.index(d_w)\n",
        "                tokens1[original_position] = d_w_changed\n",
        "                #print(f\"Words in sentence 2: '{word}' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "\n",
        "        # Check particular cases with be --> do\n",
        "        if 'does' in added_words_lower:\n",
        "          if 'not' in added_words_lower:\n",
        "            if 'arent' in deleted_words_lower:\n",
        "              d_w = 'arent'\n",
        "              d_w_changed = 'does'\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: 'does not' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              tokens1 = tokens1 + ['not']\n",
        "              add_not_token = True\n",
        "            elif 'isnt' in deleted_words_lower:\n",
        "              d_w = 'isnt'\n",
        "              d_w_changed = 'does'\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: 'does not' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              tokens1 = tokens1 + ['not']\n",
        "              add_not_token = True\n",
        "          elif 'are' in deleted_words_lower:\n",
        "            d_w = 'are'\n",
        "            d_w_changed = 'does'\n",
        "            original_position = tokens1_lower.index(d_w)\n",
        "            tokens1[original_position] = d_w_changed\n",
        "            #print(f\"Words in sentence 2: 'does' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "            verbs_changed = True\n",
        "          elif 'is' in deleted_words_lower:\n",
        "            d_w = 'is'\n",
        "            d_w_changed = 'does'\n",
        "            original_position = tokens1_lower.index(d_w)\n",
        "            tokens1[original_position] = d_w_changed\n",
        "            #print(f\"Words in sentence 2: 'does' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "            verbs_changed = True\n",
        "        elif 'do' in added_words_lower:\n",
        "          if 'not' in added_words_lower:\n",
        "            if 'arent' in deleted_words_lower:\n",
        "              d_w = 'arent'\n",
        "              d_w_changed = 'do'\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: 'do not' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              tokens1 = tokens1 + ['not']\n",
        "              add_not_token = True\n",
        "            elif 'isnt' in deleted_words_lower:\n",
        "              d_w = 'isnt'\n",
        "              d_w_changed = 'do'\n",
        "              original_position = tokens1_lower.index(d_w)\n",
        "              tokens1[original_position] = d_w_changed\n",
        "              #print(f\"Words in sentence 2: 'do not' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "              verbs_changed = True\n",
        "              tokens1 = tokens1 + ['not']\n",
        "              add_not_token = True\n",
        "          elif 'are' in deleted_words_lower:\n",
        "            d_w = 'are'\n",
        "            d_w_changed = 'do'\n",
        "            original_position = tokens1_lower.index(d_w)\n",
        "            tokens1[original_position] = d_w_changed\n",
        "            #print(f\"Words in sentence 2: 'do' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "            verbs_changed = True\n",
        "          elif 'is' in deleted_words_lower:\n",
        "            d_w = 'is'\n",
        "            d_w_changed = 'do'\n",
        "            original_position = tokens1_lower.index(d_w)\n",
        "            tokens1[original_position] = d_w_changed\n",
        "            #print(f\"Words in sentence 2: 'do' is modified and it was \\'{d_w}\\' in position '{original_position}'\")\n",
        "            verbs_changed = True\n",
        "\n",
        "\n",
        "    if verbs_changed:  # If there is at least a verb changed, do some steps again\n",
        "      # Find the common words between the two sentences\n",
        "      common_words = set(tokens1) & set(tokens2)\n",
        "\n",
        "      # Find words that are in sentence 1 but not in sentence 2 (deleted words)\n",
        "      deleted_words = set(tokens1) - set(tokens2)\n",
        "      deleted_words = [word for word in deleted_words if word not in string.punctuation]\n",
        "\n",
        "\n",
        "    # Save idx of deleted words\n",
        "    deleted_idx = []\n",
        "    if deleted_words:\n",
        "        for word in deleted_words:\n",
        "            original_position = [i for i, token in enumerate(tokens1) if token == word]\n",
        "            deleted_idx.extend(original_position)\n",
        "            #print(f\"Deleted words in sentence 2: '{word}' that was in position '{original_position}'\")\n",
        "\n",
        "\n",
        "    # Check if there is some words in common_words that has different lenght between old and new sentences\n",
        "    for word in common_words:\n",
        "        position1 = [i for i, token in enumerate(tokens1) if token == word]\n",
        "        position2 = [i for i, token in enumerate(tokens2) if token == word]\n",
        "        if len(position1) > len(position2):\n",
        "          diff = len(position1) - len(position2)\n",
        "          count = 0\n",
        "          for i, token in enumerate(tokens1):\n",
        "            if count >= diff:\n",
        "              break\n",
        "            if token == word:\n",
        "              deleted_idx.extend([i])\n",
        "              #print(f\"Deleted words in sentence 2: '{word}' that was in position '{i}'\")\n",
        "              count += 1\n",
        "\n",
        "    # Remove deleted words from the tokens of the second sentence\n",
        "    deleted_idx.sort()\n",
        "    for i in range(len(tags_split)):\n",
        "      if i not in deleted_idx:\n",
        "        new_tags_split.append(tags_split[i])\n",
        "\n",
        "    if add_not_token:\n",
        "      tags_split = tags_split + ['O']\n",
        "      new_tags_split = new_tags_split + ['O']\n",
        "\n",
        "    # Compare the common words in both sentences\n",
        "    for word in common_words:\n",
        "        position1 = [i for i, token in enumerate(tokens1) if token == word]\n",
        "        position2 = [i for i, token in enumerate(tokens2) if token == word]\n",
        "\n",
        "        if position1 != position2:\n",
        "            # Adjust tokens of the second sentence\n",
        "            try:\n",
        "              for idx1, idx2 in zip(position1, position2):\n",
        "                new_tags_split[idx2] = tags_split[idx1]\n",
        "              #print(f\"Word '{word}' was in position(s) {position1} in sentence 1, but in position(s) {position2} in sentence 2\")\n",
        "            except:\n",
        "              \"\"\"print('\\nERROR during comparison')\n",
        "              print('Original sentence:', sentence)\n",
        "              print('New sentence     :', new_sentence)\n",
        "              print()\"\"\"\n",
        "              return sentence, tags\n",
        "        elif len(position1) == 1 and len(position2) == 1 and position1 == position2 and position2[0] < len(new_tags_split) and position1[0] < len(tags_split) and new_tags_split[position2[0]] != tags_split[position1[0]]:  # Same position but different tags\n",
        "            new_tags_split[position2[0]] = tags_split[position1[0]]\n",
        "\n",
        "    \"\"\"print()\n",
        "    print(new_sentence, '-', len(new_sentence.split()))\n",
        "    print(new_tags_split, '-', len(new_tags_split))\"\"\"\n",
        "\n",
        "    if len(new_sentence.split()) != len(new_tags_split):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags_split), \"w\\\\\", new_sentence.split(), 'and', new_tags_split)\n",
        "      return sentence, tags\n",
        "    return new_sentence, ','.join(new_tags_split)\n",
        "\n",
        "  return new_sentence"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_zBsSmyDa1Ik"
      },
      "source": [
        "##### Synonym Homogenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C-sc6Nyrcn1_"
      },
      "outputs": [],
      "source": [
        "def get_noun_lists():\n",
        "  return appdata_list,temp_list,userprofile_list,systemroot_list,profilename_list,username_list,common_appdata_list,random_letters_list,random_numbers_list,systemdrives_list,system32_list,system_list,windir_list,windows_list, empty_list,programfiles_list,programfile_list,defaultuserprofile,homefolder_list,homepath_list,command_and_control_list\n",
        "\n",
        "noun_lists = get_noun_lists()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kA7gRiYrcn2B"
      },
      "outputs": [],
      "source": [
        "def get_noun_regex_list(idx):\n",
        "\n",
        "  pattern_basic = []\n",
        "  pattern_special = []\n",
        "  for el in noun_lists[idx][1:]:\n",
        "    if '<' in el or '>' in el or '%' in el or '&' in el:  # Set a special pattern for keywords with characters not recognized by '\\b'\n",
        "      pattern_special.append(el)\n",
        "      continue\n",
        "    pattern_basic.append(el)\n",
        "\n",
        "  if len(pattern_basic) > 0 and len(pattern_special) > 0:\n",
        "    pattern_basic = r'\\b(?:' + '|'.join(map(re.escape, pattern_basic)) + r')\\b'\n",
        "    pattern_special = '|'.join(map(re.escape, pattern_special))\n",
        "    pattern = pattern_basic +'|'+ pattern_special\n",
        "\n",
        "  elif len(pattern_basic) > 0:\n",
        "    pattern = r'\\b(?:' + '|'.join(map(re.escape, pattern_basic)) + r')\\b'\n",
        "\n",
        "  elif len(pattern_special) > 0:\n",
        "    pattern = '|'.join(map(re.escape, pattern_special))\n",
        "\n",
        "  return re.compile(pattern, re.IGNORECASE)\n",
        "\n",
        "def get_noun_regex_lists():\n",
        "  return get_noun_regex_list(0),get_noun_regex_list(1),get_noun_regex_list(2),get_noun_regex_list(3),get_noun_regex_list(4),get_noun_regex_list(5),get_noun_regex_list(6),get_noun_regex_list(7),get_noun_regex_list(8),get_noun_regex_list(9),get_noun_regex_list(10),get_noun_regex_list(11),get_noun_regex_list(12),get_noun_regex_list(13),get_noun_regex_list(14),get_noun_regex_list(15),get_noun_regex_list(16),get_noun_regex_list(17),get_noun_regex_list(18),get_noun_regex_list(19),get_noun_regex_list(20)\n",
        "\n",
        "noun_regex_lists = get_noun_regex_lists()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zL25hXslcn2C"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Given a list of tags, return the element with more occurences\n",
        "If there are more tags, it excludes 'O' tags in order to give more importance to entities\n",
        "\"\"\"\n",
        "def get_major_tag(tags):\n",
        "  tag_list = []\n",
        "  for el in tags:\n",
        "    if len(el) > 1:\n",
        "      tag_list.append(el[2:])  # Remove prefix and keep entity\n",
        "    else:\n",
        "      tag_list.append(el)\n",
        "\n",
        "  element_counts = Counter(tag_list)  # Count the occurrences of each element in the list\n",
        "\n",
        "  if len(element_counts) > 1 and 'O' in element_counts:\n",
        "    del element_counts['O']\n",
        "\n",
        "  highest_count = max(element_counts.values())  # Find the highest count in the Counter\n",
        "  most_common_elements = [element for element, count in element_counts.items() if count == highest_count]  # Get all elements with the highest count\n",
        "\n",
        "  most_common_element = most_common_elements[0]\n",
        "\n",
        "  if most_common_element != 'O':\n",
        "    return 'B-' + most_common_element  # Format the tags\n",
        "\n",
        "  return most_common_element  # 'O'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F0dtGWoqcn2D"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def homogenization_noun(sentence, tags='', update_tags=False):\n",
        "  new_sentence = sentence\n",
        "\n",
        "  sentence_split = sentence.split()\n",
        "  len_sentence_split = len(sentence_split)\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "  len_tags_split = len(tags_split)\n",
        "\n",
        "  tags_to_change_list = []\n",
        "\n",
        "  for var, big_regex in zip(noun_lists, noun_regex_lists):\n",
        "    new_sentence = big_regex.sub(var[0], new_sentence)  # Apply regex\n",
        "\n",
        "    if update_tags:  # Training phase - need to adapt tags\n",
        "      matched_words = []\n",
        "      matched_words.extend(re.findall(big_regex, sentence))  # Save all the matches to change entity tags\n",
        "\n",
        "      for matched_word in set(matched_words):\n",
        "        len_matched_word = len(matched_word.split())  # Count number of words in the keyword\n",
        "\n",
        "        for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "          if idx + len_matched_word > len_sentence_split:\n",
        "            break\n",
        "\n",
        "          if '<' in matched_word or '>' in matched_word or '%' in matched_word or '&' in matched_word:\n",
        "            pattern = re.escape(matched_word)\n",
        "          else:\n",
        "            pattern = r'\\b{}\\b'.format(re.escape(matched_word))\n",
        "\n",
        "          if re.search(pattern, ' '.join(sentence_split[idx:idx+len_matched_word]), re.IGNORECASE):\n",
        "            tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "              'word_to_change': matched_word,\n",
        "              'tags-idx': [i for i in range(idx, idx+len_matched_word)],\n",
        "              'final-tag': get_major_tag(tags_split[idx:idx+len_matched_word])\n",
        "            })\n",
        "\n",
        "  if update_tags:\n",
        "    tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "\n",
        "    new_tags = []\n",
        "    idx = 0\n",
        "    while len(tags_to_change_list) > 0:  # Update tags\n",
        "      dict_word_to_change = tags_to_change_list[0]\n",
        "      word_to_change = dict_word_to_change['word_to_change']\n",
        "      tags_idx = dict_word_to_change['tags-idx']\n",
        "      final_tag = dict_word_to_change['final-tag']\n",
        "\n",
        "      while idx not in tags_idx:\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx+=1\n",
        "\n",
        "      new_tags.append(final_tag)\n",
        "      idx = tags_idx[-1] + 1\n",
        "\n",
        "      tags_to_change_list.pop(0)\n",
        "\n",
        "      while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "          tags_to_change_list.pop(0)\n",
        "\n",
        "    while idx < len(tags_split):  # Copy remaining tags\n",
        "      new_tags.append(tags_split[idx])\n",
        "      idx += 1\n",
        "\n",
        "    if len(new_tags) != len(new_sentence.split()):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT NOUN STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "      return sentence, tags\n",
        "\n",
        "    new_tags = ','.join(new_tags)\n",
        "    return new_sentence, new_tags\n",
        "\n",
        "  return new_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hcm3srZlcn2E"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def homogenization_try_going_to(sentence, tags='', update_tags=False):\n",
        "  new_sentence = sentence\n",
        "\n",
        "  sentence_split = sentence.split()\n",
        "  len_sentence_split = len(sentence_split)\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "  len_tags_split = len(tags_split)\n",
        "\n",
        "  tags_to_change_list = []\n",
        "\n",
        "  for element in try_going_to_list:\n",
        "    if element in new_sentence.lower():\n",
        "        match_el = re.search(element+'[ ]+(\\S+)', new_sentence, re.IGNORECASE)\n",
        "\n",
        "        if match_el:\n",
        "          verb = match_el.group(1)\n",
        "          matched_group = match_el.group()\n",
        "          new_sentence = new_sentence.replace(matched_group, verb)\n",
        "\n",
        "          if update_tags:\n",
        "            len_matched_group = len(matched_group.split())  # Count number of words in the keyword\n",
        "\n",
        "            for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "              if idx + len_matched_group > len_sentence_split:\n",
        "                break\n",
        "\n",
        "              if re.search(matched_group, ' '.join(sentence_split[idx:idx+len_matched_group]), re.IGNORECASE):\n",
        "                tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "                  'word_to_change': matched_group,\n",
        "                  'tags-idx': [i for i in range(idx, idx+len_matched_group-1)]\n",
        "                })\n",
        "\n",
        "  if update_tags:\n",
        "    tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "\n",
        "    new_tags = []\n",
        "    idx = 0\n",
        "    while len(tags_to_change_list) > 0:  # Update tags\n",
        "      dict_word_to_change = tags_to_change_list[0]\n",
        "      word_to_change = dict_word_to_change['word_to_change']\n",
        "      tags_idx = dict_word_to_change['tags-idx']\n",
        "\n",
        "      while idx not in tags_idx:\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx+=1\n",
        "\n",
        "      idx = tags_idx[-1] + 1\n",
        "\n",
        "      tags_to_change_list.pop(0)\n",
        "\n",
        "      while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "          tags_to_change_list.pop(0)\n",
        "\n",
        "    while idx < len(tags_split):  # Copy remaining tags\n",
        "      new_tags.append(tags_split[idx])\n",
        "      idx += 1\n",
        "\n",
        "    if len(new_tags) != len(new_sentence.split()):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT TRY_TO STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "      return sentence, tags\n",
        "\n",
        "    new_tags = ','.join(new_tags)\n",
        "    return new_sentence, new_tags\n",
        "\n",
        "  return new_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dwab0X6scn2F"
      },
      "outputs": [],
      "source": [
        "def custom_partition(sentence, element_split):  # To preserve original case\n",
        "    # Find the index of the element in a case-insensitive manner\n",
        "    index = sentence.lower().find(element_split.lower())\n",
        "\n",
        "    # Determine the original casing of the element\n",
        "    element = sentence[index:index + len(element_split)].strip()\n",
        "\n",
        "    # Split the sentence into three parts: text before, element, and text after\n",
        "    before = sentence[:index].strip()\n",
        "    after = sentence[index + len(element_split):].strip()\n",
        "\n",
        "    # Reconstruct the result with the original casing\n",
        "    return before, element, after\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def homogenization_is_capable_of(sentence, tags='', update_tags=False):\n",
        "  new_sentence = sentence\n",
        "\n",
        "  sentence_split = sentence.split()\n",
        "  len_sentence_split = len(sentence_split)\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "  len_tags_split = len(tags_split)\n",
        "\n",
        "  tags_to_change_list = []\n",
        "\n",
        "  for element in capable_of_list:\n",
        "    if element in new_sentence.lower():\n",
        "      before_element, element, after_element = custom_partition(new_sentence, element)  # new_sentence.partition(element)\n",
        "\n",
        "      verb_ing = after_element.split()[0].lower()\n",
        "      remaining_after_element = ' '.join(after_element.split()[1:])\n",
        "\n",
        "      # Reconstruct phrase to preserve original case but lowercase verb to analyze to avoid errors\n",
        "      new_sentence = before_element + ' ' + element.lower() + ' ' +  verb_ing + ' ' + remaining_after_element\n",
        "\n",
        "      token = new_sentence.split()\n",
        "      for i in nltk.pos_tag(token):\n",
        "        if i[0] == verb_ing and i[1] == 'VBG':\n",
        "          verb_present_form = conjugate(verb=verb_ing,tense=INFINITIVE)\n",
        "          new_sentence = before_element + \" \" + verb_present_form + \" \" + remaining_after_element\n",
        "\n",
        "      if update_tags:\n",
        "        matched_group = element + ' ' + verb_ing\n",
        "        len_matched_group = len(matched_group.split())  # Count number of words in the keyword\n",
        "\n",
        "        for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "          if idx + len_matched_group > len_sentence_split:\n",
        "            break\n",
        "\n",
        "          if re.search(matched_group, ' '.join(sentence_split[idx:idx+len_matched_group]), re.IGNORECASE):\n",
        "            tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "              'word_to_change': matched_group,\n",
        "              'tags-idx': [i for i in range(idx, idx+len_matched_group-1)]\n",
        "            })\n",
        "\n",
        "  if update_tags:\n",
        "    tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "\n",
        "    new_tags = []\n",
        "    idx = 0\n",
        "    while len(tags_to_change_list) > 0:  # Update tags\n",
        "      dict_word_to_change = tags_to_change_list[0]\n",
        "      word_to_change = dict_word_to_change['word_to_change']\n",
        "      tags_idx = dict_word_to_change['tags-idx']\n",
        "\n",
        "      while idx not in tags_idx:\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx+=1\n",
        "\n",
        "      idx = tags_idx[-1] + 1\n",
        "\n",
        "      tags_to_change_list.pop(0)\n",
        "\n",
        "      while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "          tags_to_change_list.pop(0)\n",
        "\n",
        "    while idx < len(tags_split):  # Copy remaining tags\n",
        "      new_tags.append(tags_split[idx])\n",
        "      idx += 1\n",
        "\n",
        "    if len(new_tags) != len(new_sentence.split()):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT CAPABLE_OF STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "      return sentence, tags\n",
        "\n",
        "    new_tags = ','.join(new_tags)\n",
        "    return new_sentence, new_tags\n",
        "\n",
        "  return new_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IHVlwzxrcn2G"
      },
      "outputs": [],
      "source": [
        "def custom_split(sentence, element):\n",
        "  # Find the index of the substring 'element' in a case-insensitive manner\n",
        "  index = sentence.lower().find(element.lower())\n",
        "\n",
        "  before = sentence[:index]\n",
        "  after =  sentence[index+len(element):]\n",
        "\n",
        "  return before, after\n",
        "\n",
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def homogenization_modification(sentence, tags='', update_tags=False):\n",
        "  new_sentence = sentence\n",
        "\n",
        "  sentence_split = sentence.split()\n",
        "  len_sentence_split = len(sentence_split)\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "  len_tags_split = len(tags_split)\n",
        "\n",
        "  tags_to_change_list = []\n",
        "\n",
        "  for element in make_modification_list:\n",
        "    if element in new_sentence.lower():\n",
        "      before, after = custom_split(new_sentence, element)\n",
        "      new_sentence = before + 'modify' + after\n",
        "\n",
        "      if update_tags:\n",
        "        len_element = len(element.split())  # Count number of words in the keyword\n",
        "\n",
        "        for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "          if idx + len_element > len_sentence_split:\n",
        "            break\n",
        "\n",
        "          if re.search(element, ' '.join(sentence_split[idx:idx+len_element]), re.IGNORECASE):\n",
        "            tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "              'word_to_change': element,\n",
        "              'tags-idx': [i for i in range(idx, idx+len_element)],\n",
        "              'final-tag': 'O'\n",
        "            })\n",
        "\n",
        "  if update_tags:\n",
        "    tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "\n",
        "    new_tags = []\n",
        "    idx = 0\n",
        "    while len(tags_to_change_list) > 0:  # Update tags\n",
        "      dict_word_to_change = tags_to_change_list[0]\n",
        "      word_to_change = dict_word_to_change['word_to_change']\n",
        "      tags_idx = dict_word_to_change['tags-idx']\n",
        "      final_tag = dict_word_to_change['final-tag']\n",
        "\n",
        "      while idx not in tags_idx:\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx+=1\n",
        "\n",
        "      new_tags.append(final_tag)\n",
        "      idx = tags_idx[-1] + 1\n",
        "\n",
        "      tags_to_change_list.pop(0)\n",
        "\n",
        "      while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "          tags_to_change_list.pop(0)\n",
        "\n",
        "    while idx < len(tags_split):  # Copy remaining tags\n",
        "      new_tags.append(tags_split[idx])\n",
        "      idx += 1\n",
        "\n",
        "    if len(new_tags) != len(new_sentence.split()):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT MODIFICATION STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "      return sentence, tags\n",
        "\n",
        "    new_tags = ','.join(new_tags)\n",
        "    return new_sentence, new_tags\n",
        "\n",
        "  return new_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d82IEUnUb2jO"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def homogenization_the_following(sentence, tags='', update_tags=False):  # RIMUOVI TOKENS DI PAROLE RIMOSSE\n",
        "    tags_to_change_list = []\n",
        "    new_sentence = sentence\n",
        "\n",
        "    for element in the_following_list:\n",
        "      if element in new_sentence:\n",
        "          new_sentence.strip()\n",
        "          s_split = new_sentence.split(element)\n",
        "          if len(s_split) > 1 and len(s_split[1]) > 0 and s_split[1][0] == ':':\n",
        "            s_split[1] = s_split[1][1:]\n",
        "          new_sentence = ''.join(s_split)\n",
        "\n",
        "          if update_tags:\n",
        "            sentence_split = sentence.split()\n",
        "            len_sentence_split = len(sentence_split)\n",
        "            len_matched_element = len(element.split())  # Count number of words in the keyword\n",
        "            re_element = re.escape(element)\n",
        "\n",
        "            for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "              if idx + len_matched_element > len_sentence_split:\n",
        "                break\n",
        "\n",
        "              if re.search(re_element, ' '.join(sentence_split[idx:idx+len_matched_element]), re.IGNORECASE):\n",
        "                tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "                  'word_to_change': element,\n",
        "                  'tags-idx': [i for i in range(idx, idx+len_matched_element)]\n",
        "                })\n",
        "\n",
        "    new_sentence = re.sub(r'\\s+', ' ', new_sentence)\n",
        "\n",
        "    if update_tags:\n",
        "      tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "      tags_split = tags.split(',')\n",
        "\n",
        "      new_tags = []\n",
        "      idx = 0\n",
        "      while len(tags_to_change_list) > 0:  # Update tags\n",
        "        dict_word_to_change = tags_to_change_list[0]\n",
        "        word_to_change = dict_word_to_change['word_to_change']\n",
        "        tags_idx = dict_word_to_change['tags-idx']\n",
        "\n",
        "        while idx not in tags_idx:\n",
        "          new_tags.append(tags_split[idx])\n",
        "          idx+=1\n",
        "\n",
        "        idx = tags_idx[-1] + 1\n",
        "\n",
        "        tags_to_change_list.pop(0)\n",
        "\n",
        "        while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "            tags_to_change_list.pop(0)\n",
        "\n",
        "      while idx < len(tags_split):  # Copy remaining tags\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx += 1\n",
        "\n",
        "      if len(new_tags) != len(new_sentence.split()):\n",
        "        #print(\"\\nERROR DIFFERENT LENGHT AT THE_FOLLOWING STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "        return sentence, tags\n",
        "\n",
        "      new_tags = ','.join(new_tags)\n",
        "      return new_sentence, new_tags\n",
        "\n",
        "    #print(tags_to_change_list)\n",
        "    return new_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dae9eTMUcn2H"
      },
      "outputs": [],
      "source": [
        "def get_verbs_in_sentence(sentence):\n",
        "  #try:\n",
        "  s_split = sentence.split()\n",
        "  s = ' '.join(s_split[:150])\n",
        "  predictions = predictor.predict(s.lower())\n",
        "  \"\"\"except:\n",
        "    #print('errors while predicting verbs in sentence'.upper(), sentence)\n",
        "    return []\"\"\"\n",
        "\n",
        "  verb = []\n",
        "  verbs_list = []\n",
        "  for k in predictions['verbs']:\n",
        "    verb.append(k['verb'])\n",
        "    if k['description'].count('[') > 1:\n",
        "      verbs_list.append(' '.join(verb))\n",
        "      verb = []\n",
        "  return verbs_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hXUUTEjDuZED"
      },
      "outputs": [],
      "source": [
        "def get_verbs_in_sentences(sentences):\n",
        "  sent_dict_list = []\n",
        "  for s in sentences:\n",
        "    s_split = s.split()\n",
        "    s = ' '.join(s_split[:150])\n",
        "    sent_dict_list.append({'sentence': s})\n",
        "\n",
        "  list_emergency = []\n",
        "  try:\n",
        "    predictions = predictor.predict_batch_json(sent_dict_list)\n",
        "  except:\n",
        "    print()\n",
        "    for sentence in sentences:\n",
        "      try:\n",
        "        s_split = sentence.split()\n",
        "        s = ' '.join(s_split[:150])\n",
        "        prediction = predictor.predict(s.lower())\n",
        "        list_emergency.append(prediction)\n",
        "      except:\n",
        "        #print('errors while predicting verbs in sentence'.upper(), sentence)\n",
        "        list_emergency.append({'verbs': []})\n",
        "\n",
        "  if len(list_emergency) > 0:\n",
        "    predictions = list_emergency\n",
        "\n",
        "  total_verbs_list = []\n",
        "  for prediction in predictions:\n",
        "    verb = []\n",
        "    verbs_list = []\n",
        "    for k in prediction['verbs']:\n",
        "      verb.append(k['verb'])\n",
        "      if k['description'].count('[') > 1:\n",
        "        verbs_list.append(' '.join(verb))\n",
        "        verb = []\n",
        "    total_verbs_list.append(verbs_list)\n",
        "\n",
        "  return total_verbs_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WrXS-2qecn2I"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Takes the verb, the sentence, and from which idx starting to search\n",
        "Returns the list of idx of the words composing the verb, if it's a negative form and if there was any error in searching\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "def get_first_index_from(word, sentence, start_from_idx):\n",
        "  sentence_split = sentence.split()\n",
        "  sentence_len = len(sentence_split)\n",
        "\n",
        "  word_split_len = len(word.split())\n",
        "  idx = start_from_idx\n",
        "  negative_form = 0\n",
        "\n",
        "  error = 0\n",
        "\n",
        "  if word_split_len == 1:  # Verb with single word\n",
        "    while idx < sentence_len:\n",
        "      current_split = sentence_split[idx].lower()  # Get word of sentence_split at idx\n",
        "      if word in current_split:  # Verify if it's the verb\n",
        "        if word+'n\\'t' in current_split or word+'nt' in current_split:  # Verify if it's negative\n",
        "          negative_form = 1\n",
        "        return [idx], negative_form, error\n",
        "      idx += 1\n",
        "\n",
        "  else:  # Verb with more words\n",
        "    first_word = word.split()[0]\n",
        "    remaining_words = ' '.join(word.split()[1:])\n",
        "    indexes_list = []\n",
        "\n",
        "    while idx < sentence_len:\n",
        "      current_split = sentence_split[idx].lower()  # Get word of sentence_split at idx\n",
        "      if first_word in current_split:  # Verify if it's the verb\n",
        "        if first_word+'n\\'t' in current_split or first_word+'nt' in current_split:  # Verify if it's negative\n",
        "          negative_form = 1\n",
        "        start_idx_remaining_words = idx+1\n",
        "        while start_idx_remaining_words < sentence_len:  # When the first word of the verb is found, search for the remaining parts (they may be not be tied to the first term)\n",
        "          if remaining_words in ' '.join(sentence_split[start_idx_remaining_words:(start_idx_remaining_words + word_split_len-1)]).lower():\n",
        "            indexes_list.append(idx)\n",
        "            for i in range(start_idx_remaining_words, start_idx_remaining_words + word_split_len-1):\n",
        "              indexes_list.append(i)\n",
        "            return indexes_list, negative_form, error\n",
        "          start_idx_remaining_words += 1\n",
        "      idx += 1\n",
        "\n",
        "  error = 1\n",
        "  return [], -1, error"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cs3OEGJgcn2J"
      },
      "outputs": [],
      "source": [
        "def standardize_verb(verb):\n",
        "  for v_list in verbs_lists_homogenization:\n",
        "    if verb in v_list:\n",
        "      return v_list[0]\n",
        "  return verb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2aKq4xCqcn2K"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Substitute the verb with its lemmatize form, taking the dictionary with the information about the verb and the sentence\n",
        "Returns the new sentence and the index of the last modification\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "def apply_lemmatization(substitutions, sentence):\n",
        "  sentence_split = sentence.split()\n",
        "  new_sentence_split = []\n",
        "  new_sentence_idx_last_modification = 0\n",
        "\n",
        "  verb = substitutions['verb']\n",
        "  lemm = standardize_verb(substitutions['lemm'])\n",
        "  tags_idx = substitutions['tags-idx']\n",
        "  tag_len = substitutions['tag-len']\n",
        "  negative_form = substitutions['negative-form']\n",
        "\n",
        "  last_verb_word = tags_idx[-1]\n",
        "\n",
        "  for idx, word in enumerate(sentence_split):\n",
        "    if idx not in tags_idx:  # Not a verb to remove\n",
        "      new_sentence_split.append(word)  # Keep it in the phrase\n",
        "    elif idx == last_verb_word:  # Last part of the verb\n",
        "      new_sentence_split.append(lemm)  # Keep the lemmatized form\n",
        "      new_sentence_idx_last_modification = len(new_sentence_split)-1  # Update index\n",
        "      if negative_form:  # If negative form, add a not before the verb\n",
        "        new_sentence_split.insert(new_sentence_idx_last_modification, 'not')\n",
        "        new_sentence_idx_last_modification += 1  # Update index\n",
        "\n",
        "  return ' '.join(new_sentence_split), new_sentence_idx_last_modification"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UYYcOEA_cn2M"
      },
      "outputs": [],
      "source": [
        "def adjust_tags(substitution, tags):\n",
        "  tags_idx = substitution['tags-idx']\n",
        "  tag_len = substitution['tag-len']\n",
        "  negative_form = substitution['negative-form']\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "\n",
        "  tags_to_check = []\n",
        "  for tag_idx in tags_idx:\n",
        "    tags_to_check.append(tags_split[tag_idx])\n",
        "  final_tag = get_major_tag(tags_to_check)\n",
        "\n",
        "  new_tags_split = []\n",
        "  last_verb_tag = tags_idx[-1]\n",
        "\n",
        "  for idx, tag in enumerate(tags_split):\n",
        "    if idx not in tags_idx:  # Not a verb to remove\n",
        "      new_tags_split.append(tag)  # Keep it in the phrase\n",
        "    elif idx == last_verb_tag:  # Last part of the verb\n",
        "      new_tags_split.append(final_tag)  # Keep the lemmatized form\n",
        "      new_tag_idx_last_modification = len(new_tags_split)-1  # Update index\n",
        "      if negative_form:  # If negative form, add a not before the verb\n",
        "        new_tags_split.insert(new_tag_idx_last_modification, 'O')\n",
        "  return ','.join(new_tags_split)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XL7cDFlacn2N"
      },
      "outputs": [],
      "source": [
        "def homogenization_verbs(sentence, tags='', update_tags=False):\n",
        "  new_sentence = sentence\n",
        "  new_tags = copy.deepcopy(tags)\n",
        "  verbs_list_in_sentence = get_verbs_in_sentence(new_sentence)\n",
        "\n",
        "  # Search for lemmatizing\n",
        "  start_idx = -1\n",
        "  for verb in verbs_list_in_sentence:\n",
        "    substitutions = []\n",
        "    lemm = WordNetLemmatizer().lemmatize(verb.split()[-1].lower(), 'v')\n",
        "\n",
        "    tags_idx, negative_form, error = get_first_index_from(verb, new_sentence, start_idx+1)\n",
        "    if error:\n",
        "      #print('\\nERR W\\\\ \"', verb, '\" IN THE SENTENCE:', sentence)\n",
        "      continue\n",
        "\n",
        "    substitution = {\n",
        "        'verb': verb,\n",
        "        'lemm': lemm,\n",
        "        'tags-idx': tags_idx,\n",
        "        'tag-len': len(verb.split()),\n",
        "        'negative-form': negative_form\n",
        "    }\n",
        "\n",
        "    new_sentence, start_idx = apply_lemmatization(substitution, new_sentence)\n",
        "\n",
        "    if update_tags:\n",
        "      new_tags = adjust_tags(substitution, new_tags)\n",
        "\n",
        "  if update_tags:\n",
        "    if len(new_sentence.split()) != len(new_tags.split(',')):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT THE_FOLLOWING STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags.split(',')), \"w\\\\\", new_sentence.split(), 'and', new_tags.split(','))\n",
        "      return sentence, tags\n",
        "\n",
        "    return new_sentence, new_tags\n",
        "\n",
        "  return new_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hhJY2gByv-vJ"
      },
      "outputs": [],
      "source": [
        "def homogenization_verbs_post(sentences, tags_l='', update_tags=False):\n",
        "  total_sentences = []\n",
        "  total_tags = []\n",
        "\n",
        "  new_sentences = copy.deepcopy(sentences)\n",
        "  verbs_list_in_sentences = get_verbs_in_sentences(new_sentences)\n",
        "\n",
        "  if not update_tags:\n",
        "    for sentence, verbs_list_in_sentence in zip(new_sentences, verbs_list_in_sentences):\n",
        "      new_sentence = sentence\n",
        "\n",
        "      # Search for lemmatizing\n",
        "      start_idx = -1\n",
        "      for verb in verbs_list_in_sentence:\n",
        "        substitutions = []\n",
        "        lemm = WordNetLemmatizer().lemmatize(verb.split()[-1].lower(), 'v')\n",
        "\n",
        "        tags_idx, negative_form, error = get_first_index_from(verb, new_sentence, start_idx+1)\n",
        "        if error:\n",
        "          #print('\\nERR W\\\\ \"', verb, '\" IN THE SENTENCE:', sentence)\n",
        "          continue\n",
        "\n",
        "        substitution = {\n",
        "            'verb': verb,\n",
        "            'lemm': lemm,\n",
        "            'tags-idx': tags_idx,\n",
        "            'tag-len': len(verb.split()),\n",
        "            'negative-form': negative_form\n",
        "        }\n",
        "\n",
        "        new_sentence, start_idx = apply_lemmatization(substitution, new_sentence)\n",
        "\n",
        "      total_sentences.append(new_sentence)\n",
        "    return total_sentences\n",
        "\n",
        "\n",
        "  new_tags_list = copy.deepcopy(tags_l)\n",
        "\n",
        "  for sentence, verbs_list_in_sentence, tags in zip(new_sentences, verbs_list_in_sentences, new_tags_list):\n",
        "    new_sentence = sentence\n",
        "    new_tags = copy.deepcopy(tags)\n",
        "\n",
        "    # Search for lemmatizing\n",
        "    start_idx = -1\n",
        "    for verb in verbs_list_in_sentence:\n",
        "      substitutions = []\n",
        "      lemm = WordNetLemmatizer().lemmatize(verb.split()[-1].lower(), 'v')\n",
        "\n",
        "      tags_idx, negative_form, error = get_first_index_from(verb, new_sentence, start_idx+1)\n",
        "      if error:\n",
        "        #print('\\nERR W\\\\ \"', verb, '\" IN THE SENTENCE:', sentence)\n",
        "        continue\n",
        "\n",
        "      substitution = {\n",
        "          'verb': verb,\n",
        "          'lemm': lemm,\n",
        "          'tags-idx': tags_idx,\n",
        "          'tag-len': len(verb.split()),\n",
        "          'negative-form': negative_form\n",
        "      }\n",
        "\n",
        "      new_sentence, start_idx = apply_lemmatization(substitution, new_sentence)\n",
        "\n",
        "      new_tags = adjust_tags(substitution, new_tags)\n",
        "\n",
        "    if len(new_sentence.split()) != len(new_tags.split(',')):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT THE_FOLLOWING STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags.split(',')), \"w\\\\\", new_sentence.split(), 'and', new_tags.split(','))\n",
        "      total_sentences.append(sentence)\n",
        "      total_tags.append(tags)\n",
        "      continue\n",
        "\n",
        "    total_sentences.append(new_sentence)\n",
        "    total_tags.append(new_tags)\n",
        "\n",
        "  return total_sentences, total_tags"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PGW0zQ7ncn2N"
      },
      "outputs": [],
      "source": [
        "def homogenization_steps(sentence, tags='', update_tags=False):\n",
        "  if update_tags:\n",
        "    sentence, tags = homogenization_noun(sentence, tags=tags, update_tags=True)\n",
        "    sentence, tags = homogenization_try_going_to(sentence, tags=tags, update_tags=True)\n",
        "    sentence, tags = homogenization_is_capable_of(sentence, tags=tags, update_tags=True)\n",
        "    sentence, tags = homogenization_modification(sentence, tags=tags, update_tags=True)\n",
        "    sentence, tags = homogenization_the_following(sentence, tags=tags, update_tags=True)\n",
        "    sentence, tags = homogenization_verbs(sentence, tags=tags, update_tags=True)\n",
        "    return sentence, tags\n",
        "\n",
        "  sentence = homogenization_noun(sentence)\n",
        "  sentence = homogenization_try_going_to(sentence)\n",
        "  sentence = homogenization_is_capable_of(sentence)\n",
        "  sentence = homogenization_modification(sentence)\n",
        "  sentence = homogenization_the_following(sentence)\n",
        "  sentence = homogenization_verbs(sentence)\n",
        "  return sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tz0Gzosq4Sbl"
      },
      "outputs": [],
      "source": [
        "def homogenization_steps_post(sentences, tags_l='', update_tags=False):\n",
        "  total_sentences = []\n",
        "  total_tags = []\n",
        "\n",
        "  if update_tags:\n",
        "    for sentence, tags in zip(sentences, tags_l):\n",
        "      sentence, tags = homogenization_noun(sentence, tags=tags, update_tags=True)\n",
        "      sentence, tags = homogenization_try_going_to(sentence, tags=tags, update_tags=True)\n",
        "      sentence, tags = homogenization_is_capable_of(sentence, tags=tags, update_tags=True)\n",
        "      sentence, tags = homogenization_modification(sentence, tags=tags, update_tags=True)\n",
        "      sentence, tags = homogenization_the_following(sentence, tags=tags, update_tags=True)\n",
        "      #sentence, tags = homogenization_verbs(sentence, tags=tags, update_tags=True)\n",
        "      total_sentences.append(sentence)\n",
        "      total_tags.append(tags)\n",
        "\n",
        "    return homogenization_verbs_post(total_sentences, tags_l=total_tags, update_tags=True)\n",
        "\n",
        "  for sentence in sentences:\n",
        "    sentence = homogenization_noun(sentence)\n",
        "    sentence = homogenization_try_going_to(sentence)\n",
        "    sentence = homogenization_is_capable_of(sentence)\n",
        "    sentence = homogenization_modification(sentence)\n",
        "    sentence = homogenization_the_following(sentence)\n",
        "    #sentence = homogenization_verbs(sentence)\n",
        "    total_sentences.append(sentence)\n",
        "  return homogenization_verbs_post(total_sentences)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8EBVjWNcc5vh"
      },
      "source": [
        "#### <font color='yellow'>Name Resolution</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xccZryuM7VGw"
      },
      "source": [
        "##### Pronouns and Subject ellipsis resolution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rQ9YIFCcVEes"
      },
      "outputs": [],
      "source": [
        "def remove_punctuation_from_words(sentence):\n",
        "  s_split = sentence.lower().split()\n",
        "\n",
        "  new_sent = []\n",
        "  for w in s_split:\n",
        "    new_w = ''.join(char for char in w if char not in string.punctuation)\n",
        "    if new_w == '':\n",
        "      new_w = w\n",
        "    new_sent.append(new_w)\n",
        "\n",
        "  return new_sent\n",
        "\n",
        "def ellipsis_pronouns(sentence, tags='', update_tags=False):\n",
        "    doc = ellipsis_nlp(sentence)\n",
        "    new_sentence = doc._.coref_resolved.replace(\"\\n\", \"\")\n",
        "    #print('\\nSENTENCE', sentence)\n",
        "    #print('\\nNEW SENTENCE', new_sentence)\n",
        "    #print(tags.split(','))\n",
        "    #print(sentence.split())\n",
        "    #print(new_sentence.split())\n",
        "    #print()\n",
        "\n",
        "    if doc._.has_coref:\n",
        "      list_mentions = []\n",
        "      for cluster in doc._.coref_clusters:\n",
        "          list_mentions.append(remove_punctuation_from_words(str(cluster.mentions[0])))  # Original noun\n",
        "      #print(list_mentions)\n",
        "\n",
        "      if update_tags:\n",
        "        sentence_split = remove_punctuation_from_words(sentence)\n",
        "        new_sentence_split = remove_punctuation_from_words(new_sentence)\n",
        "        tags_split = tags.split(',')\n",
        "\n",
        "        idx_sentence = 0\n",
        "        idx_new_sentence = 0\n",
        "        len_sentence = len(sentence_split)\n",
        "        len_new_sentence = len(new_sentence_split)\n",
        "\n",
        "        changed_words = []\n",
        "        changed_idx = []\n",
        "        changes_to_check = []\n",
        "\n",
        "        while idx_sentence < len_sentence and idx_new_sentence < len_new_sentence:\n",
        "          #print('idx_sentence:', idx_sentence, '- idx_new_sentence:', idx_new_sentence)\n",
        "          #print('sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence] ?', sentence_split[idx_sentence], new_sentence_split[idx_new_sentence], sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence])\n",
        "          if sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence]:\n",
        "            changed_words.append(new_sentence_split[idx_new_sentence])\n",
        "            changed_idx.append(idx_sentence)\n",
        "            idx_sentence += 1\n",
        "            idx_new_sentence += 1\n",
        "            temp_idx_new_sentence = idx_new_sentence\n",
        "\n",
        "            if idx_sentence < len_sentence:\n",
        "              while idx_new_sentence < len_new_sentence:\n",
        "                if sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence]:\n",
        "                  changed_words.append(new_sentence_split[idx_new_sentence])\n",
        "                  idx_new_sentence += 1\n",
        "                else:\n",
        "                  changes_to_check.append({\n",
        "                      'changed_words_sentence': changed_words,\n",
        "                      'changed_idx_sentence': changed_idx,\n",
        "                      'final_tags': []\n",
        "                  })\n",
        "                  changed_words = []\n",
        "                  changed_idx = []\n",
        "                  break\n",
        "\n",
        "              #### new block for special cases\n",
        "              if idx_new_sentence >= len_new_sentence: #problem w\\ 2 pronouns one after the other\n",
        "                for el in list_mentions:\n",
        "                  len_el = len(el)\n",
        "                  if changed_words[:len_el] == el:\n",
        "                    break\n",
        "\n",
        "                changes_to_check.append({\n",
        "                      'changed_words_sentence': changed_words[:len_el],\n",
        "                      'changed_idx_sentence': changed_idx,\n",
        "                      'final_tags': []\n",
        "                  })\n",
        "                idx_new_sentence = temp_idx_new_sentence + len_el-1\n",
        "\n",
        "                if idx_new_sentence >= len_new_sentence:\n",
        "                  #print('ERROR IN SUBSTITUTIONS FOR SENTENCE', sentence)\n",
        "                  return sentence, tags\n",
        "\n",
        "                changed_words = []\n",
        "                changed_words.append(new_sentence_split[idx_new_sentence])\n",
        "                changed_idx = []\n",
        "                changed_idx.append(idx_sentence)\n",
        "                idx_sentence += 1\n",
        "                idx_new_sentence += 1\n",
        "\n",
        "\n",
        "                if idx_sentence < len_sentence:\n",
        "                  while idx_new_sentence < len_new_sentence:\n",
        "                    #print('idx_sentence:', idx_sentence, '- idx_new_sentence:', idx_new_sentence)\n",
        "                    #print('sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence] ?', sentence_split[idx_sentence], new_sentence_split[idx_new_sentence], sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence])\n",
        "                    if sentence_split[idx_sentence] != new_sentence_split[idx_new_sentence]:\n",
        "                      changed_words.append(new_sentence_split[idx_new_sentence])\n",
        "                      idx_new_sentence += 1\n",
        "                    else:\n",
        "                      changes_to_check.append({\n",
        "                          'changed_words_sentence': changed_words,\n",
        "                          'changed_idx_sentence': changed_idx,\n",
        "                          'final_tags': []\n",
        "                      })\n",
        "                      changed_words = []\n",
        "                      changed_idx = []\n",
        "                      break\n",
        "\n",
        "                else:\n",
        "                  while idx_new_sentence < len_new_sentence:\n",
        "                      changed_words.append(new_sentence_split[idx_new_sentence])\n",
        "                      idx_new_sentence += 1\n",
        "                  changes_to_check.append({\n",
        "                      'changed_words_sentence': changed_words,\n",
        "                      'changed_idx_sentence': changed_idx,\n",
        "                      'final_tags': []\n",
        "                  })\n",
        "                  changed_words = []\n",
        "                  changed_idx = []\n",
        "              ####\n",
        "\n",
        "            else:\n",
        "              while idx_new_sentence < len_new_sentence:\n",
        "                  changed_words.append(new_sentence_split[idx_new_sentence])\n",
        "                  idx_new_sentence += 1\n",
        "              changes_to_check.append({\n",
        "                  'changed_words_sentence': changed_words,\n",
        "                  'changed_idx_sentence': changed_idx,\n",
        "                  'final_tags': []\n",
        "              })\n",
        "              changed_words = []\n",
        "              changed_idx = []\n",
        "\n",
        "          idx_sentence += 1\n",
        "          idx_new_sentence += 1\n",
        "\n",
        "        idx_sentence = 0\n",
        "        idx_changes = 0\n",
        "        len_changes_to_check = len(changes_to_check)\n",
        "        while idx_changes < len_changes_to_check:\n",
        "          current_change = changes_to_check[idx_changes]\n",
        "          len_changed_words = len(current_change['changed_words_sentence'])\n",
        "          idx_sentence = current_change['changed_idx_sentence'][0]-len_changed_words\n",
        "          while idx_sentence >= 0:\n",
        "            if sentence_split[idx_sentence:idx_sentence+len_changed_words] == current_change['changed_words_sentence']:\n",
        "              current_change['final_tags'] = tags_split[idx_sentence:idx_sentence+len_changed_words]\n",
        "              idx_changes += 1\n",
        "              break\n",
        "            idx_sentence -= 1\n",
        "          if idx_sentence <= -1:\n",
        "            idx_changes += 1\n",
        "\n",
        "        \"\"\"for el in changes_to_check:\n",
        "          print(el)\"\"\"\n",
        "\n",
        "        new_tags = []\n",
        "        idx = 0\n",
        "        while len(changes_to_check) > 0:  # Update tags\n",
        "          changed_idx_sentence = changes_to_check[0]['changed_idx_sentence']\n",
        "          final_tags = changes_to_check[0]['final_tags']\n",
        "\n",
        "          while idx not in changed_idx_sentence:\n",
        "            new_tags.append(tags_split[idx])\n",
        "            idx+=1\n",
        "\n",
        "          new_tags.extend(final_tags)\n",
        "          idx = changed_idx_sentence[-1] + 1\n",
        "\n",
        "          changes_to_check.pop(0)\n",
        "\n",
        "        while idx < len_sentence:  # Copy remaining tags\n",
        "          new_tags.append(tags_split[idx])\n",
        "          idx += 1\n",
        "\n",
        "\n",
        "        if len(new_tags) != len(new_sentence.split()):\n",
        "          #print(\"\\nERROR DIFFERENT LENGHT AT PRONOUNS STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\\n\", new_sentence.split(), 'and\\n', new_tags)\n",
        "          return sentence, tags\n",
        "\n",
        "\n",
        "        new_tags = ','.join(new_tags)\n",
        "        return new_sentence, new_tags\n",
        "\n",
        "      return new_sentence\n",
        "\n",
        "    else:\n",
        "      if update_tags:\n",
        "        return sentence, tags\n",
        "      return sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QJ0zStgLVSUD"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Search subject of sentence starting from the last one\n",
        "\"\"\"\n",
        "def detect_subj(sentence_list):\n",
        "    subject = ''\n",
        "    for idx, sentence in enumerate(reversed(sentence_list)):\n",
        "        idx_sent = len(sentence_list) - 1 - idx\n",
        "        doc = ellipsis_nlp(sentence)\n",
        "        for token in doc:\n",
        "            if token.dep_ == \"nsubj\":\n",
        "                subject = token.text\n",
        "        if subject:\n",
        "          #print(f'Subject found at sentence # {idx_sent}: {sentence}')\n",
        "          return subject, idx_sent\n",
        "\n",
        "    #print(f'No Subject IT -1')\n",
        "    return 'It', -1 # If no subject is detected\n",
        "\n",
        "def capitalize(line):\n",
        "    return line[0].upper() + line[1:]\n",
        "\n",
        "\"\"\"\n",
        "Add subject to sentences that starts with a verb\n",
        "It can be found in previous sentences or simply add a 'It'\n",
        "\"\"\"\n",
        "def ellipsis_subject(sentence_list, search_subj_in_previous_text=False, tags_list=[], update_tags=False):\n",
        "  new_sentence_list = []\n",
        "  new_tags_list = []\n",
        "\n",
        "  for idx, sentence in enumerate(sentence_list):  # For each sentence\n",
        "    if sentence == '' or sentence == ' ':\n",
        "      new_sentence_list.append('')\n",
        "      if update_tags:\n",
        "        new_tags_list.append('')\n",
        "      continue\n",
        "\n",
        "    # Split it into tokens and examine different cases (I didn't fully understand how the cases are structured but I trust Extractor authors)\n",
        "    token = sentence.split()\n",
        "    doc = ellipsis_nlp(sentence)\n",
        "\n",
        "    # Case 1\n",
        "    if nltk.pos_tag(token)[0][1] == \"VB\" or nltk.pos_tag(token)[0][1] == \"VBZ\" or doc[0].pos_ == \"VERB\" or doc[0].text.lower() in ellipsis_verbs:\n",
        "      if search_subj_in_previous_text:  # Search text in previous sentences\n",
        "        subject, idx_sent = detect_subj(new_sentence_list)  # Retrieve subject and idx of the sentence\n",
        "        new_sentence = capitalize(subject) + \" \" + nltk.pos_tag(token)[0][0].lower() + \" \" + \" \".join(sentence.split(\" \")[1:])  # Add subject to sentence\n",
        "\n",
        "        if update_tags:  # Also update tags\n",
        "          if idx_sent == -1:  # Not subject found\n",
        "            #print('CASE 1 No subject found')\n",
        "            new_tags = 'O,' + tags_list[idx]\n",
        "          else:\n",
        "            #print(f'CASE 1 Subject found at sentence # {idx_sent}: {new_sentence_list[idx_sent]} with tags {new_tags_list[idx_sent]}')\n",
        "            #print(f'The subject is \"{subject}\"')\n",
        "            tags_split = new_tags_list[idx_sent].split(',')\n",
        "            sentence_split = new_sentence_list[idx_sent].split()\n",
        "            pattern = re.compile(rf'\\b(?:{re.escape(subject)})\\b', re.IGNORECASE)\n",
        "            for tok, tag in zip(reversed(sentence_split), reversed(tags_split)):\n",
        "              if pattern.search(tok):  # Search for the word used as subject and add its tag to the new sentence's tags\n",
        "                new_tags = tag + ',' + tags_list[idx]\n",
        "                break\n",
        "      else:  # Do not search text in previous sentences - Simply add 'it'\n",
        "        new_sentence = \" It \" + nltk.pos_tag(token)[0][0].lower() + \" \" + \" \".join(sentence.split(\" \")[1:])\n",
        "        if update_tags:\n",
        "          new_tags = 'O,' + tags_list[idx]\n",
        "\n",
        "    # Case 2\n",
        "    elif doc[0].dep_ == \"ROOT\":\n",
        "        # Subcase 2.1\n",
        "        if doc[0].text.lower() in ellipsis_verbs:  # Didn't get the division of cases but ok\n",
        "          if search_subj_in_previous_text:  # Search text in previous sentences\n",
        "            subject, idx_sent = detect_subj(new_sentence_list)  # Retrieve subject and idx of the sentence\n",
        "            new_sentence = capitalize(subject) + \" \" + doc[0].text.lower() + \" \" + \" \".join(sentence.split(\" \")[1:])  # Add subject to sentence\n",
        "\n",
        "            if update_tags:  # Also update tags\n",
        "              if idx_sent == -1:  # Not subject found\n",
        "                #print('CASE 2 No subject found')\n",
        "                new_tags = 'O,' + tags_list[idx]\n",
        "              else:\n",
        "                #print(f'CASE 2 Subject found at sentence # {idx_sent}: {new_sentence_list[idx_sent]} with tags {new_tags_list[idx_sent]}')\n",
        "                #print(f'The subject is \"{subject}\"')\n",
        "                tags_split = new_tags_list[idx_sent].split(',')\n",
        "                sentence_split = new_sentence_list[idx_sent].split()\n",
        "                pattern = re.compile(rf'\\b(?:{re.escape(subject)})\\b', re.IGNORECASE)\n",
        "                for tok, tag in zip(reversed(sentence_split), reversed(tags_split)):\n",
        "                  if pattern.search(tok):  # Search for the word used as subject and add its tag to the new sentence's tags\n",
        "                    new_tags = tag + ',' + tags_list[idx]\n",
        "                    break\n",
        "\n",
        "          else:  # Do not search text in previous sentences - Simply add 'it'\n",
        "            new_sentence = \" It \" + doc[0].text.lower() + \" \" + \" \".join(sentence.split(\" \")[1:])\n",
        "            if update_tags:\n",
        "              new_tags = 'O,' + tags_list[idx]\n",
        "\n",
        "        # Subcase 2.2\n",
        "        else:\n",
        "          new_sentence = sentence\n",
        "          if update_tags:\n",
        "            new_tags = tags_list[idx]\n",
        "\n",
        "    # Case 3\n",
        "    elif doc[0].text.lower() in ellipsis_verbs and doc[0].dep_ != \"ROOT\":\n",
        "        new_sentence = doc.text\n",
        "        if update_tags:\n",
        "          new_tags = tags_list[idx]\n",
        "\n",
        "    # Case 4\n",
        "    else:\n",
        "        new_sentence = sentence\n",
        "        if update_tags:\n",
        "          new_tags = tags_list[idx]\n",
        "\n",
        "    if update_tags and len(new_sentence.strip().split()) != len(new_tags.split(',')):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT AT SUBJECT STEP --> sentence_len:\", len(new_sentence.strip().split()), \"!= tags_len:\", len(new_tags.split(',')), \"w\\\\ SENTENCE\", sentence, 'and', new_sentence.strip().split(), 'and', new_tags.split(','))\n",
        "      new_sentence_list.append(sentence_list[idx])\n",
        "      new_tags_list.append(tags_list[idx])\n",
        "      continue\n",
        "\n",
        "    new_sentence_list.append(new_sentence.strip())\n",
        "    if update_tags:\n",
        "      new_tags_list.append(new_tags)\n",
        "\n",
        "  if update_tags:\n",
        "    return new_sentence_list, new_tags_list\n",
        "\n",
        "  return new_sentence_list"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GvqyVYvPa8lU"
      },
      "source": [
        "##### Stopwords Removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cKplFk8MbRQF"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def remove_stopwords(sentence, tags='', update_tags=False):\n",
        "  filtered_sentence = []\n",
        "  filtered_tags = []\n",
        "  sentence_tokenized = sentence.split()\n",
        "  tags_tokenized = tags.split(',')\n",
        "\n",
        "  # Train-phase case\n",
        "  if update_tags:\n",
        "    #print('Train-phase')\n",
        "    \"\"\"if len(sentence_tokenized) != len(tags_tokenized):\n",
        "      print('ERROR: different lenght tokens and tags')\n",
        "      print(sentence_tokenized)\n",
        "      print(tags_tokenized)\n",
        "      return None, None\"\"\"\n",
        "\n",
        "    for word, tag in zip(sentence_tokenized, tags_tokenized):\n",
        "      if word.lower() not in stop_words:\n",
        "        filtered_sentence.append(word)\n",
        "        filtered_tags.append(tag)\n",
        "\n",
        "    if len(filtered_sentence) != len(filtered_tags):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT --> sentence_len:\", len(filtered_sentence), \"!= tags_len:\", len(filtered_tags), \"w\\\\\", filtered_sentence, 'and', filtered_tags)\n",
        "      return sentence, tags\n",
        "\n",
        "    return ' '.join(filtered_sentence), ','.join(filtered_tags)\n",
        "\n",
        "  # Test-phase case\n",
        "  #print('Prediction-phase')\n",
        "  for word in sentence_tokenized:\n",
        "    if word.lower() not in stop_words:\n",
        "      filtered_sentence.append(word)\n",
        "\n",
        "  return ' '.join(filtered_sentence)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L4PD6GazbB5s"
      },
      "source": [
        "##### Internet Slang Removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uAiIYmPjbIbW"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def remove_internet_slangs(sentence, tags='', update_tags=False):\n",
        "  filtered_sentence = []\n",
        "  filtered_tags = []\n",
        "  sentence_tokenized = sentence.split()\n",
        "  tags_tokenized = tags.split(',')\n",
        "\n",
        "  # Train-phase case\n",
        "  if update_tags:\n",
        "    #print('Train-phase')\n",
        "    \"\"\"if len(sentence_tokenized) != len(tags_tokenized):\n",
        "      print('ERROR: different lenght tokens and tags')\n",
        "      print(sentence_tokenized)\n",
        "      print(tags_tokenized)\n",
        "      return None, None\"\"\"\n",
        "\n",
        "    for word, tag in zip(sentence_tokenized, tags_tokenized):\n",
        "      if word.lower() not in internet_slang_words:\n",
        "        filtered_sentence.append(word)\n",
        "        filtered_tags.append(tag)\n",
        "\n",
        "    if len(filtered_sentence) != len(filtered_tags):\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT --> sentence_len:\", len(filtered_sentence), \"!= tags_len:\", len(filtered_tags), \"w\\\\\", filtered_sentence, 'and', filtered_tags)\n",
        "      #print(\"\\nERROR DIFFERENT LENGHT w\\\\\", filtered_sentence, 'and', filtered_tags)\n",
        "      return sentence, tags\n",
        "\n",
        "    return ' '.join(filtered_sentence), ','.join(filtered_tags)\n",
        "\n",
        "  # Test-phase case\n",
        "  #print('Prediction-phase')\n",
        "  for word in sentence_tokenized:\n",
        "    if word.lower() not in internet_slang_words:\n",
        "      filtered_sentence.append(word)\n",
        "\n",
        "  return ' '.join(filtered_sentence)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G20m_Ld7h7pz"
      },
      "source": [
        "##### Aliases Handling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7LnD0U_EiDum"
      },
      "outputs": [],
      "source": [
        "def get_actor_groups_regex_lists():\n",
        "  actor_groups_regex_lists = []\n",
        "\n",
        "  for alias_list in actor_groups_aliases:\n",
        "    pattern = r'\\b(?:' + '|'.join(map(re.escape, alias_list)) + r')\\b'\n",
        "    actor_groups_regex_lists.append(re.compile(pattern, re.IGNORECASE))\n",
        "\n",
        "  return actor_groups_regex_lists\n",
        "\n",
        "actor_groups_regex_lists = get_actor_groups_regex_lists()\n",
        "\n",
        "\n",
        "def get_malware_families_regex_lists():\n",
        "  malware_families_regex_lists = []\n",
        "\n",
        "  for alias_list in malware_families_aliases:\n",
        "    pattern = r'\\b(?:' + '|'.join(map(re.escape, alias_list)) + r')\\b'\n",
        "    malware_families_regex_lists.append(re.compile(pattern, re.IGNORECASE))\n",
        "\n",
        "  return malware_families_regex_lists\n",
        "\n",
        "malware_families_regex_lists = get_malware_families_regex_lists()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vEpPx79siFlm"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def handle_actor_groups_aliases(sentence, tags='', update_tags=False):\n",
        "  if sentence == '':\n",
        "    if update_tags:\n",
        "      return sentence, tags, []\n",
        "    return sentence, []\n",
        "\n",
        "  new_sentence = sentence\n",
        "\n",
        "  sentence_split = sentence.split()\n",
        "  len_sentence_split = len(sentence_split)\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "  len_tags_split = len(tags_split)\n",
        "\n",
        "  tags_to_change_list = []\n",
        "  alias_pair_list = []\n",
        "\n",
        "  for var, big_regex in zip(actor_groups_aliases, actor_groups_regex_lists):\n",
        "    new_sentence = big_regex.sub(var[0], new_sentence)  # Apply regex\n",
        "\n",
        "    matched_words = []\n",
        "    matched_words.extend(re.findall(big_regex, sentence))  # Save all the matches to change entity tags\n",
        "\n",
        "    for matched_word in set(matched_words):\n",
        "      len_matched_word = len(matched_word.split())  # Count number of words in the keyword\n",
        "\n",
        "      for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "        if idx + len_matched_word > len_sentence_split:\n",
        "          break\n",
        "\n",
        "        pattern = r'\\b{}\\b'.format(re.escape(matched_word))\n",
        "\n",
        "        if re.search(pattern, ' '.join(sentence_split[idx:idx+len_matched_word]), re.IGNORECASE):\n",
        "          final_tags = ['B-APT']\n",
        "          for i in range(len(var[0].split())-1):\n",
        "            final_tags.append('I-APT')\n",
        "          tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "            'word_to_change': matched_word,\n",
        "            'tags-idx': [i for i in range(idx, idx+len_matched_word)],\n",
        "            'final-tag': final_tags,\n",
        "            'final-word': var[0]\n",
        "          })\n",
        "\n",
        "  tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "\n",
        "  for el in tags_to_change_list:\n",
        "    word_to_change = el['word_to_change']\n",
        "    final_word = el['final-word']\n",
        "    tags_idx = el['tags-idx']\n",
        "    alias_pair_list.append({\n",
        "        'alias_name':final_word,\n",
        "        'original_name':word_to_change\n",
        "    })\n",
        "\n",
        "  if update_tags:\n",
        "    new_tags = []\n",
        "    idx = 0\n",
        "    while len(tags_to_change_list) > 0:  # Update tags\n",
        "      dict_word_to_change = tags_to_change_list[0]\n",
        "      word_to_change = dict_word_to_change['word_to_change']\n",
        "      tags_idx = dict_word_to_change['tags-idx']\n",
        "      final_tag = dict_word_to_change['final-tag']\n",
        "\n",
        "      while idx not in tags_idx:\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx+=1\n",
        "\n",
        "      new_tags.extend(final_tag)\n",
        "      idx = tags_idx[-1] + 1\n",
        "\n",
        "      tags_to_change_list.pop(0)\n",
        "\n",
        "      while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "          tags_to_change_list.pop(0)\n",
        "\n",
        "    while idx < len(tags_split):  # Copy remaining tags\n",
        "      new_tags.append(tags_split[idx])\n",
        "      idx += 1\n",
        "\n",
        "    if len(new_tags) != len(new_sentence.split()):\n",
        "      print(\"\\nERROR DIFFERENT LENGHT AT THREAT ACTORS STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "      return sentence, tags, []\n",
        "\n",
        "    new_tags = ','.join(new_tags)\n",
        "    return new_sentence, new_tags, alias_pair_list\n",
        "\n",
        "  return new_sentence, alias_pair_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V9KCqAdUiLaf"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  tags=[], update_tags=False are optional since the function could be called during the prediction-phase\n",
        "  During the train-phase, just specify the arguments to update the tags associated to the sentence\n",
        "\n",
        "\"\"\"\n",
        "def handle_malware_families_aliases(sentence, tags='', update_tags=False):\n",
        "  if sentence == '':\n",
        "    if update_tags:\n",
        "      return sentence, tags, []\n",
        "    return sentence, []\n",
        "\n",
        "  new_sentence = sentence\n",
        "\n",
        "  sentence_split = sentence.split()\n",
        "  len_sentence_split = len(sentence_split)\n",
        "\n",
        "  tags_split = tags.split(',')\n",
        "  len_tags_split = len(tags_split)\n",
        "\n",
        "  tags_to_change_list = []\n",
        "  alias_pair_list = []\n",
        "\n",
        "  for var, big_regex in zip(malware_families_aliases, malware_families_regex_lists):\n",
        "    new_sentence = big_regex.sub(var[0], new_sentence)  # Apply regex\n",
        "\n",
        "    matched_words = []\n",
        "    matched_words.extend(re.findall(big_regex, sentence))  # Save all the matches to change entity tags\n",
        "\n",
        "    for matched_word in set(matched_words):\n",
        "      len_matched_word = len(matched_word.split())  # Count number of words in the keyword\n",
        "\n",
        "      for idx, _ in enumerate(sentence_split):  # Search the matched keyword in the original sentence and retrieve indexes\n",
        "        if idx + len_matched_word > len_sentence_split:\n",
        "          break\n",
        "\n",
        "        pattern = r'\\b{}\\b'.format(re.escape(matched_word))\n",
        "\n",
        "        if re.search(pattern, ' '.join(sentence_split[idx:idx+len_matched_word]), re.IGNORECASE):\n",
        "          final_tags = ['B-APT']\n",
        "          for i in range(len(var[0].split())-1):\n",
        "            final_tags.append('I-APT')\n",
        "          tags_to_change_list.append({  # Save indexes of the keyword to update tags\n",
        "            'word_to_change': matched_word,\n",
        "            'tags-idx': [i for i in range(idx, idx+len_matched_word)],\n",
        "            'final-tag': final_tags,\n",
        "            'final-word': var[0]\n",
        "          })\n",
        "\n",
        "  tags_to_change_list = sorted(tags_to_change_list, key=lambda x: x['tags-idx'][0])  # Sort indexes from the first appearance to the last in the sentence\n",
        "\n",
        "  for el in tags_to_change_list:\n",
        "    word_to_change = el['word_to_change']\n",
        "    final_word = el['final-word']\n",
        "    tags_idx = el['tags-idx']\n",
        "    alias_pair_list.append({\n",
        "        'alias_name':final_word,\n",
        "        'original_name':word_to_change\n",
        "    })\n",
        "\n",
        "  if update_tags:\n",
        "    new_tags = []\n",
        "    idx = 0\n",
        "    while len(tags_to_change_list) > 0:  # Update tags\n",
        "      dict_word_to_change = tags_to_change_list[0]\n",
        "      word_to_change = dict_word_to_change['word_to_change']\n",
        "      tags_idx = dict_word_to_change['tags-idx']\n",
        "      final_tag = dict_word_to_change['final-tag']\n",
        "\n",
        "      while idx not in tags_idx:\n",
        "        new_tags.append(tags_split[idx])\n",
        "        idx+=1\n",
        "\n",
        "      new_tags.extend(final_tag)\n",
        "      idx = tags_idx[-1] + 1\n",
        "\n",
        "      tags_to_change_list.pop(0)\n",
        "\n",
        "      while len(tags_to_change_list) > 0 and tags_idx[-1] >= tags_to_change_list[0]['tags-idx'][0]:  # Check if there are more matches in the some words and skip them\n",
        "          tags_to_change_list.pop(0)\n",
        "\n",
        "    while idx < len(tags_split):  # Copy remaining tags\n",
        "      new_tags.append(tags_split[idx])\n",
        "      idx += 1\n",
        "\n",
        "    if len(new_tags) != len(new_sentence.split()):\n",
        "      print(\"\\nERROR DIFFERENT LENGHT AT THREAT NAMES STEP --> sentence_len:\", len(new_sentence.split()), \"!= tags_len:\", len(new_tags), \"w\\\\\", new_sentence.split(), 'and', new_tags)\n",
        "      return sentence, tags, []\n",
        "\n",
        "    new_tags = ','.join(new_tags)\n",
        "    return new_sentence, new_tags, alias_pair_list\n",
        "\n",
        "  return new_sentence, alias_pair_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mPelJ6N7iQVx"
      },
      "outputs": [],
      "source": [
        "def handle_aliases(sentence, tags='', update_tags=False):\n",
        "  if update_tags:\n",
        "    sentence, tags, actor_aliases = handle_actor_groups_aliases(sentence, tags=tags, update_tags=True)\n",
        "    sentence, tags, malware_aliases = handle_malware_families_aliases(sentence, tags=tags, update_tags=True)\n",
        "    return sentence, tags, actor_aliases, malware_aliases\n",
        "\n",
        "  sentence, actor_aliases = handle_actor_groups_aliases(sentence)\n",
        "  sentence, malware_aliases = handle_malware_families_aliases(sentence)\n",
        "  return sentence, actor_aliases, malware_aliases"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zYvN0-HBGkMk"
      },
      "source": [
        "## <font color='orange'>Pipeline NLP</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6oOzt5HzGq6W"
      },
      "source": [
        "### <font color='yellow'>Sanitization</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "58ruJz9zpC2G"
      },
      "source": [
        "#### Unrelated content removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2UXZYkiFpBOv"
      },
      "outputs": [],
      "source": [
        "def call_unrelatedContentRemoval(data, final_filename):\n",
        "  if unrelated_content_removal:\n",
        "    final_filename.append('unrelatedContentRemoval')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Unrelated content removal: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='Unrelated Content Removal'):\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = analyze_sentence(sentence)\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X9gfiZ_sGxOt"
      },
      "source": [
        "#### IOC Defanging"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rw8Mhmi6IlKa"
      },
      "outputs": [],
      "source": [
        "def call_iocDefanging(data, final_filename):\n",
        "  if ioc_defanging:\n",
        "    final_filename.append('iocDefanging')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('IOC Defanging: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='IOC Defanging'):\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = defang_iocs_in_text(sentence)\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FrdmPNniG9SE"
      },
      "source": [
        "#### Misspelling Correction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pT0QivICGK9d"
      },
      "outputs": [],
      "source": [
        "def call_misspellingCorrection(data, final_filename):\n",
        "  if misspelling_correction:\n",
        "    final_filename.append('misspellingCorrection')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Misspelling correction: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='Misspelling correction'):\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = spell_check(sentence)\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nf2gNjStPZeI"
      },
      "source": [
        "### <font color='yellow'>Text Normalization</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R3RB2hqaavFD"
      },
      "source": [
        "#### Passive\\Active Conversion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FYsLXf3IavFI"
      },
      "outputs": [],
      "source": [
        "def call_passiveActiveConversion(data, final_filename):\n",
        "  if passive_active_conversion:\n",
        "    final_filename.append('passiveActiveConversion')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Passive\\\\Active Conversion: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='Passive\\\\Active Conversion'):\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = convert_active_to_passive(sentence)\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R4srYbakPCXr"
      },
      "source": [
        "#### Synonym Homogenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IvPGyKJjP-Wo"
      },
      "outputs": [],
      "source": [
        "def call_synonymHomogenization(data, final_filename):\n",
        "  if synonym_homogenization:\n",
        "    final_filename.append('synonymHomogenization')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join('/content/drive/My Drive/Pipeline_new_data/Folder_datasets/Results/temp5', temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Synonym Homogenization: Step already completed --> Load data')\n",
        "    else:\n",
        "      i = 0\n",
        "      for entry in tqdm(data, desc='Synonym Homogenization'):\n",
        "        data[i]['content'] = homogenization_steps_post(entry['content'])\n",
        "        i += 1\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jW9KhVqd3O9G"
      },
      "source": [
        "### <font color='yellow'>Name Resolution</font>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CYJN5qEkWPWK"
      },
      "source": [
        "#### Pronouns and Subject Ellipsis Resolution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F1tGugT8X5kw"
      },
      "outputs": [],
      "source": [
        "def call_pronounsSubjectEllipsisResolution(data, final_filename):\n",
        "  if pronouns_subject_ellipsis_resolution:\n",
        "    final_filename.append('pronounsSubjectEllipsisResolution')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Pronouns and Subject Ellipsis Resolution: Step already completed --> Load data')\n",
        "    else:\n",
        "      i = 0\n",
        "      for entry in tqdm(data, desc='Pronouns and Subject Ellipsis Resolution'):\n",
        "        print(f'Entry ID {entry[\"ID\"]}')\n",
        "        if entry['ID'] == 18370:\n",
        "          i += 1\n",
        "          continue\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = ellipsis_pronouns(sentence)\n",
        "        data[i]['content'] = ellipsis_subject(entry['content'], search_subj_in_previous_text=search_subj_in_previous_text)\n",
        "        i += 1\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQBy2QFf3T4q"
      },
      "source": [
        "#### Stopwords Removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p3UWWnM64R4H"
      },
      "outputs": [],
      "source": [
        "def call_stopwordsRemoval(data, final_filename):\n",
        "  if stopwords_removal:\n",
        "    final_filename.append('stopwordsRemoval')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Stopwords Removal: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='Stopwords Removal'):\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = remove_stopwords(sentence)\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kvn5aFEr5BAq"
      },
      "source": [
        "#### Internet Slang Removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jTNi8QO65Rkt"
      },
      "outputs": [],
      "source": [
        "def call_internetSlangRemoval(data, final_filename):\n",
        "  if internet_slang_removal:\n",
        "    final_filename.append('internetSlangRemoval')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Internet Slang Removal: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='Internet Slang Removal'):\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx] = remove_internet_slangs(sentence)\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nmAb3FxNiejS"
      },
      "source": [
        "#### Aliases Handling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tcnvpTPXjoN5"
      },
      "outputs": [],
      "source": [
        "def call_aliasesHandling(data, final_filename):\n",
        "  if aliases_handling:\n",
        "    final_filename.append('aliasesHandling')\n",
        "\n",
        "    temp_filename = 'temp_' + '_'.join(final_filename) + '.json'\n",
        "    file_path = os.path.join(temp_file_path, temp_filename)\n",
        "\n",
        "    if os.path.exists(file_path):\n",
        "      with open(file_path, 'r', encoding='utf-8') as json_file:\n",
        "        data = json.load(json_file)\n",
        "      print('Aliases Handling: Step already completed --> Load data')\n",
        "    else:\n",
        "      for entry in tqdm(data, desc='Aliases Handling'):\n",
        "        aliases_list = []\n",
        "        for idx, sentence in enumerate(entry['content']):\n",
        "          entry['content'][idx], actor_aliases, malware_aliases = handle_aliases(sentence)\n",
        "          aliases_list.append({'actor_aliases':actor_aliases, 'malware_aliases':malware_aliases})\n",
        "        entry['aliases_list'] = aliases_list\n",
        "\n",
        "      with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "        json.dump(data, json_file, indent=4)\n",
        "\n",
        "      print(f'\\nSaved in {file_path}')\n",
        "  else:\n",
        "    print('Step not executed')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fJ1so8pHtyzG"
      },
      "source": [
        "### <font color='lightblue'>Save result</font>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9OZg5aoaDe0H"
      },
      "outputs": [],
      "source": [
        "def call_save(data, final_filename):\n",
        "  final_filename = '_'.join(final_filename) + '.json'\n",
        "  file_path = os.path.join('/content/drive/My Drive/Steps NLP framework/Pipeline_new_data/Folder_datasets/Results', final_filename)\n",
        "\n",
        "  with open(file_path, 'w', encoding='utf-8') as json_file:\n",
        "      json.dump(data, json_file, indent=4)\n",
        "\n",
        "  print(f'Saved in {file_path}')\n",
        "  return data, final_filename"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ode-V4t8iH7q"
      },
      "source": [
        "## Loop block"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6Q1jCYC2sLrf"
      },
      "outputs": [],
      "source": [
        "# Chunks to process\n",
        "min_len = 1\n",
        "stop_len = min_len + 40"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YNidLTtTqHoa"
      },
      "outputs": [],
      "source": [
        "source = 0  # 0:HF or 1:REP\n",
        "\n",
        "# Choose steps to perform in the pipeline\n",
        "unrelated_content_removal = True\n",
        "ioc_defanging = True\n",
        "misspelling_correction = False\n",
        "\n",
        "passive_active_conversion = True\n",
        "synonym_homogenization = True\n",
        "if source == 0:\n",
        "  synonym_homogenization = False\n",
        "\n",
        "pronouns_subject_ellipsis_resolution = True\n",
        "if pronouns_subject_ellipsis_resolution:\n",
        "  search_subj_in_previous_text = True\n",
        "stopwords_removal = True\n",
        "internet_slang_removal = True\n",
        "aliases_handling = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aayP0hJKjiBD"
      },
      "outputs": [],
      "source": [
        "temp_file_path = '/content/drive/My Drive/Steps NLP framework/Pipeline_new_data/Folder_datasets/Results/temp'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WFLErRzhrX4M"
      },
      "outputs": [],
      "source": [
        "if source == 0:\n",
        "  max_len = 9005\n",
        "elif source == 1:\n",
        "  max_len = 394\n",
        "\n",
        "max_len = min(stop_len, max_len)\n",
        "print(f'from {min_len} to {max_len-1}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RxHxvXtaUGWY"
      },
      "outputs": [],
      "source": [
        "for i_dataset in range (min_len, max_len):\n",
        "  print(f'{i_dataset}/{max_len-1}')\n",
        "  final_filename = []\n",
        "\n",
        "  if source == 0:\n",
        "    path = f'/content/drive/My Drive/Steps NLP framework/Pipeline_new_data/Folder_datasets/HF_dataset_{i_dataset}.json'\n",
        "    final_filename.append(f'HF_{i_dataset}')\n",
        "  elif source == 1:\n",
        "    path = f'/content/drive/My Drive/Steps NLP framework/Pipeline_new_data/Folder_datasets/REP_dataset_{i_dataset}.json'\n",
        "    final_filename.append(f'REP_{i_dataset}')\n",
        "\n",
        "  with open(path, 'r', encoding='utf-8') as json_file:\n",
        "    data = json.load(json_file)\n",
        "\n",
        "  data, final_filename = call_unrelatedContentRemoval(data, final_filename)\n",
        "  data, final_filename = call_iocDefanging(data, final_filename)\n",
        "  data, final_filename = call_misspellingCorrection(data, final_filename)\n",
        "  data, final_filename = call_passiveActiveConversion(data, final_filename)\n",
        "  data, final_filename = call_synonymHomogenization(data, final_filename)\n",
        "  data, final_filename = call_pronounsSubjectEllipsisResolution(data, final_filename)\n",
        "  data, final_filename = call_stopwordsRemoval(data, final_filename)\n",
        "  data, final_filename = call_internetSlangRemoval(data, final_filename)\n",
        "  data, final_filename = call_aliasesHandling(data, final_filename)\n",
        "  data, final_filename = call_save(data, final_filename)\n",
        "\n",
        "  print('\\n-----------------------------\\n')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "Wk7DzKkWGZV4",
        "-6DI_GhaS-C0",
        "TFzsVnOMGeQM",
        "Gg25ltsEZS86",
        "084HwYOXZcZB",
        "wQEX6Qe_aGL0",
        "G21L6Up3YmWr",
        "Y4tzWWbEcCHD",
        "JBwy4rX5byqu",
        "Dq0GuG8sb3_H",
        "LkcYxKv6b7GX",
        "-sgvVvsz7AWK",
        "Hgtj4XovaIZz",
        "9Elfj18hcMe3",
        "UEhiIgL-cxaX",
        "aGSnKhb7n6TR",
        "JiUZpjScaSDZ",
        "L9ICuzEcamgB",
        "8EBVjWNcc5vh",
        "xccZryuM7VGw",
        "GvqyVYvPa8lU",
        "L4PD6GazbB5s",
        "6oOzt5HzGq6W",
        "58ruJz9zpC2G",
        "X9gfiZ_sGxOt",
        "FrdmPNniG9SE",
        "Nf2gNjStPZeI",
        "R3RB2hqaavFD",
        "R4srYbakPCXr",
        "jW9KhVqd3O9G",
        "CYJN5qEkWPWK",
        "MQBy2QFf3T4q",
        "kvn5aFEr5BAq",
        "nmAb3FxNiejS"
      ],
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}