Intrusion_Detection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Intrusion Detection",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python",
      "language": "python",
      "name": "conda-env-python-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "ORfD1EUiZuLK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9b81a0e5-5e11-452d-c8c4-445504ce24e8"
      },
      "source": [
        "!wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/Advanced_ML_anomaly_detection_L3/DataSets.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-11-12 09:55:10--  https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/Advanced_ML_anomaly_detection_L3/DataSets.zip\n",
            "Resolving cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud (cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud)... 169.63.118.104\n",
            "Connecting to cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud (cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud)|169.63.118.104|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 402533304 (384M) [application/zip]\n",
            "Saving to: ‘DataSets.zip’\n",
            "\n",
            "DataSets.zip        100%[===================>] 383.88M  25.4MB/s    in 15s     \n",
            "\n",
            "2021-11-12 09:55:25 (25.9 MB/s) - ‘DataSets.zip’ saved [402533304/402533304]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ILNW-o17MX9Y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd5e1f30-bc91-4e16-8ebc-977606d6143e"
      },
      "source": [
        "!pip install gensim\n",
        "!pip install python-Levenshtein\n",
        "!pip install tensorflow==2.2.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: gensim in /usr/local/lib/python3.7/dist-packages (3.6.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.15.0)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.19.5)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (5.2.1)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.4.1)\n",
            "Requirement already satisfied: python-Levenshtein in /usr/local/lib/python3.7/dist-packages (0.12.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from python-Levenshtein) (57.4.0)\n",
            "Requirement already satisfied: tensorflow==2.2.0 in /usr/local/lib/python3.7/dist-packages (2.2.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (0.12.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.12.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (3.3.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.1.2)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.32.0)\n",
            "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (2.2.2)\n",
            "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (2.2.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (3.17.3)\n",
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.4.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (0.2.0)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.19.5)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.15.0)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.6.3)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (2.10.0)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (0.3.3)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.2.0) (0.37.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (57.4.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (3.3.4)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (1.8.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (1.0.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (1.35.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (4.2.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (4.8.2)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (0.4.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (3.0.4)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (3.1.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.2.0) (3.6.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jdVcijxmTTha",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 376
        },
        "outputId": "d18d7164-1e4e-4b32-d889-2f22558bb043"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "import xml.etree.ElementTree as ElTree\n",
        "import re, h5py, itertools, math, glob, zipfile, os\n",
        "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
        "from sklearn.metrics import log_loss, auc, roc_curve\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.layers.core import Masking\n",
        "from tensorflow.keras.layers import Dense, LSTM, Dropout, Embedding, TimeDistributed\n",
        "from keras.layers.wrappers import Bidirectional\n",
        "from tensorflow.keras.models import Model, Sequential, load_model\n",
        "from keras.utils import np_utils\n",
        "from tensorflow.python.client import device_lib\n",
        "from lxml import etree\n",
        "from gensim.models import Word2Vec\n",
        "\n",
        "# %matplotlib inline\n",
        "plt.rcParams['figure.figsize'] = (15, 5)\n",
        "plt.style.use('ggplot')\n",
        "seed = 42\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(action = \"ignore\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-f0877d2be812>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mlog_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mroc_curve\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMasking\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDense\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLSTM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDropout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mEmbedding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTimeDistributed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrappers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBidirectional\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow.keras.layers.core'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 924
        },
        "id": "qQPEf3ZenBRh",
        "outputId": "58d859c5-4a65-4804-a5b0-0b7857c1718d"
      },
      "source": [
        "\n",
        "!pip install --upgrade tensorflow-gpu==2.2.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tensorflow-gpu==2.2.0\n",
            "  Downloading tensorflow_gpu-2.2.0-cp37-cp37m-manylinux2010_x86_64.whl (516.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 516.2 MB 4.0 kB/s \n",
            "\u001b[?25hRequirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (0.3.3)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (2.10.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.1.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.15.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (0.2.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (0.37.0)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.6.3)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.19.5)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (3.3.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.1.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (3.17.3)\n",
            "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (2.2.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.12.1)\n",
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.4.1)\n",
            "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (2.2.2)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (0.12.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu==2.2.0) (1.32.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (1.0.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (57.4.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (1.8.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (2.23.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (1.35.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (3.3.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (4.2.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (4.8.2)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (2.10)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (3.1.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow-gpu==2.2.0) (3.6.0)\n",
            "Installing collected packages: tensorflow-gpu\n",
            "  Attempting uninstall: tensorflow-gpu\n",
            "    Found existing installation: tensorflow-gpu 2.7.0\n",
            "    Uninstalling tensorflow-gpu-2.7.0:\n",
            "      Successfully uninstalled tensorflow-gpu-2.7.0\n",
            "Successfully installed tensorflow-gpu-2.2.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "tensorflow"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nwMoaJLV13Z3"
      },
      "source": [
        "pd.set_option(\"precision\", 3)\n",
        "pd.options.display.float_format = '{:.3f}'.format"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xlvlUEQ2PaKz"
      },
      "source": [
        "def plot_history(history):\n",
        "    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n",
        "    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n",
        "    acc_list = [s for s in history.history.keys() if 'accuracy' in s and 'val' not in s]\n",
        "    val_acc_list = [s for s in history.history.keys() if 'accuracy' in s and 'val' in s]\n",
        "    \n",
        "    plt.figure(figsize = (12, 5), dpi = 100)\n",
        "    COLOR = 'gray'\n",
        "    \n",
        "    plt.rc('legend', fontsize = 14)   \n",
        "    plt.rc('figure', titlesize = 12)  \n",
        "        \n",
        "    if len(loss_list) == 0:\n",
        "        print('Loss is missing in history')\n",
        "        return \n",
        "  \n",
        "    epochs = range(1, len(history.history[loss_list[0]]) + 1)\n",
        "    \n",
        "   \n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.subplots_adjust(wspace = 2, hspace = 2)\n",
        "    plt.rcParams['text.color'] = 'black'\n",
        "    plt.rcParams['axes.titlecolor'] = 'black'\n",
        "    plt.rcParams['axes.labelcolor'] = COLOR\n",
        "    plt.rcParams['xtick.color'] = COLOR\n",
        "    plt.rcParams['ytick.color'] = COLOR\n",
        "    for l in loss_list:\n",
        "        plt.plot(epochs, history.history[l], 'b-o',\n",
        "                 label = 'Train (' + str(str(format(history.history[l][-1],'.4f'))+')'))\n",
        "    for l in val_loss_list:\n",
        "        plt.plot(epochs, history.history[l], 'g',\n",
        "                 label = 'Valid (' + str(str(format(history.history[l][-1],'.4f'))+')'))\n",
        "    \n",
        "    plt.title('Loss')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.legend(facecolor = 'gray', loc = 'best')\n",
        "    plt.grid(True)\n",
        "    plt.tight_layout()\n",
        "    \n",
        "   \n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.subplots_adjust(wspace = 2, hspace = 2)\n",
        "    plt.rcParams['text.color'] = 'black'\n",
        "    plt.rcParams['axes.titlecolor'] = 'black'\n",
        "    plt.rcParams['axes.labelcolor'] = COLOR\n",
        "    plt.rcParams['xtick.color'] = COLOR\n",
        "    plt.rcParams['ytick.color'] = COLOR\n",
        "    for l in acc_list:\n",
        "        plt.plot(epochs, history.history[l], 'b-o',\n",
        "                 label = 'Train (' + str(format(history.history[l][-1],'.4f'))+')')\n",
        "    for l in val_acc_list:    \n",
        "        plt.plot(epochs, history.history[l], 'g',\n",
        "                 label = 'Valid (' + str(format(history.history[l][-1],'.4f'))+')')\n",
        "\n",
        "    plt.title('Accuracy')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.legend(facecolor = 'gray', loc = 'best')\n",
        "    plt.grid(True)\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "class B_Generator(object):\n",
        "    def __init__(self, BZ, XX, YY, ohe):\n",
        "        self.BZ = BZ\n",
        "        self.n_b = int(math.floor(np.shape(XX)[0] / BZ))\n",
        "        self.b_index = [a * BZ for a in range(0, self.n_b)]\n",
        "        self.XX = XX\n",
        "        self.YY = YY\n",
        "        self.ohe = ohe\n",
        "        \n",
        "    def __iter__(self):\n",
        "        for var_0 in itertools.cycle(self.b_index):\n",
        "            YY = self.YY[var_0 : (var_0 + self.BZ)]\n",
        "            ohe_Y = self.ohe.transform(YY.reshape(len(YY), 1))\n",
        "            yield (self.XX[var_0 : (var_0 + self.BZ),], ohe_Y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-tW2bcLSr86z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "679e7cf3-0010-4b0a-866d-127481081e9b"
      },
      "source": [
        "number = 2\n",
        "\n",
        "if zipfile.is_zipfile('DataSets.zip'):\n",
        "  file_1 = zipfile.ZipFile('DataSets.zip', 'r')\n",
        "else:\n",
        "  print('Type file isn`t ZIP')\n",
        "\n",
        "name_dataset = file_1.namelist()[number]\n",
        "file_1.extract(name_dataset)\n",
        "print(\"File\", name_dataset, \"has been read\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File TestbedSunJun13Flows.xml has been read\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNff39M3D6bj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b661f486-ac2d-4d88-bf9e-283e2930e012"
      },
      "source": [
        "tree_set = ElTree.parse(name_dataset)\n",
        "root_tree_set = tree_set.getroot()\n",
        "\n",
        "result = []\n",
        "var_1 = root_tree_set.items()[0][1][:-4]\n",
        "\n",
        "for item in root_tree_set.findall(var_1):\n",
        "    result.append({node.tag: node.text for node in item.getiterator()})\n",
        "\n",
        "dSET = pd.DataFrame(result)\n",
        "dSET = dSET.drop(dSET.columns[[0]], axis = 1)\n",
        "dSET = dSET.drop_duplicates()\n",
        "\n",
        "dSET = dSET.sort_values('startDateTime')\n",
        "dSET['IPs_sequence'] = dSET['source'] + '_' + dSET['destination'] + '_' + dSET['startDateTime'].str[:13]\n",
        "\n",
        "dSET['res_port'] = np.where(dSET.destinationPort <= dSET.sourcePort,\n",
        "                            dSET['destinationPort'],\n",
        "                            dSET['sourcePort'])\n",
        "\n",
        "\n",
        "dSET = dSET.rename(columns = {'totalSourceBytes': 'totSB',\n",
        "                        'totalDestinationBytes': 'totDB',\n",
        "                        'totalDestinationPackets': 'totDP',\n",
        "                        'totalSourcePackets': 'totSP',\n",
        "                        'sourcePayloadAsBase64': 'sourB64',\n",
        "                        'sourcePayloadAsUTF': 'sourUTF',\n",
        "                        'destinationPayloadAsBase64': 'destB64',\n",
        "                        'destinationPayloadAsUTF': 'destUTF',\n",
        "                        'direction': 'direct',\n",
        "                        'sourceTCPFlagsDescription': 'sourTCPFd',\n",
        "                        'destinationTCPFlagsDescription': 'destTCPFd',\n",
        "                        'protocolName': 'pName',\n",
        "                        'sourcePort': 'sPort',\n",
        "                        'destination': 'dest',\n",
        "                        'destinationPort': 'dPort'})\n",
        "print(\"Preparation process has been finished\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Preparation process has been finished\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "do_MtuN_0mmQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56a623a7-0f93-4629-e568-70842b8e9e2d"
      },
      "source": [
        "dSET.shape, dSET.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((137160, 22),\n",
              " Index(['appName', 'totSB', 'totDB', 'totDP', 'totSP', 'sourB64', 'sourUTF',\n",
              "        'destB64', 'destUTF', 'direct', 'sourTCPFd', 'destTCPFd', 'source',\n",
              "        'pName', 'sPort', 'dest', 'dPort', 'startDateTime', 'stopDateTime',\n",
              "        'Tag', 'IPs_sequence', 'res_port'],\n",
              "       dtype='object'))"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wetTGQ5XfCLD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        },
        "outputId": "2cd38bb0-cb0d-4df0-eccc-9710e9f3ef16"
      },
      "source": [
        "dSET.head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>appName</th>\n",
              "      <th>totSB</th>\n",
              "      <th>totDB</th>\n",
              "      <th>totDP</th>\n",
              "      <th>totSP</th>\n",
              "      <th>sourB64</th>\n",
              "      <th>sourUTF</th>\n",
              "      <th>destB64</th>\n",
              "      <th>destUTF</th>\n",
              "      <th>direct</th>\n",
              "      <th>sourTCPFd</th>\n",
              "      <th>destTCPFd</th>\n",
              "      <th>source</th>\n",
              "      <th>pName</th>\n",
              "      <th>sPort</th>\n",
              "      <th>dest</th>\n",
              "      <th>dPort</th>\n",
              "      <th>startDateTime</th>\n",
              "      <th>stopDateTime</th>\n",
              "      <th>Tag</th>\n",
              "      <th>IPs_sequence</th>\n",
              "      <th>res_port</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Unknown_UDP</td>\n",
              "      <td>2633658</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>28971</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>L2R</td>\n",
              "      <td>N/A</td>\n",
              "      <td>N/A</td>\n",
              "      <td>192.168.5.122</td>\n",
              "      <td>udp_ip</td>\n",
              "      <td>5353</td>\n",
              "      <td>224.0.0.251</td>\n",
              "      <td>5353</td>\n",
              "      <td>2010-06-12T23:57:24</td>\n",
              "      <td>2010-06-13T09:24:52</td>\n",
              "      <td>Normal</td>\n",
              "      <td>192.168.5.122_224.0.0.251_2010-06-12T23</td>\n",
              "      <td>5353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HTTPWeb</td>\n",
              "      <td>64</td>\n",
              "      <td>128</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>L2L</td>\n",
              "      <td>F,A</td>\n",
              "      <td>R</td>\n",
              "      <td>192.168.2.113</td>\n",
              "      <td>tcp_ip</td>\n",
              "      <td>4191</td>\n",
              "      <td>192.168.5.122</td>\n",
              "      <td>80</td>\n",
              "      <td>2010-06-12T23:57:38</td>\n",
              "      <td>2010-06-12T23:59:20</td>\n",
              "      <td>Normal</td>\n",
              "      <td>192.168.2.113_192.168.5.122_2010-06-12T23</td>\n",
              "      <td>4191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>HTTPWeb</td>\n",
              "      <td>128</td>\n",
              "      <td>64</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>L2R</td>\n",
              "      <td>F,A</td>\n",
              "      <td>F,A</td>\n",
              "      <td>192.168.2.113</td>\n",
              "      <td>tcp_ip</td>\n",
              "      <td>4192</td>\n",
              "      <td>207.241.148.80</td>\n",
              "      <td>80</td>\n",
              "      <td>2010-06-12T23:57:40</td>\n",
              "      <td>2010-06-12T23:59:20</td>\n",
              "      <td>Normal</td>\n",
              "      <td>192.168.2.113_207.241.148.80_2010-06-12T23</td>\n",
              "      <td>4192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>HTTPImageTransfer</td>\n",
              "      <td>128</td>\n",
              "      <td>128</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>L2R</td>\n",
              "      <td>F,A</td>\n",
              "      <td>F,A</td>\n",
              "      <td>192.168.2.110</td>\n",
              "      <td>tcp_ip</td>\n",
              "      <td>1864</td>\n",
              "      <td>216.49.88.12</td>\n",
              "      <td>80</td>\n",
              "      <td>2010-06-12T23:57:42</td>\n",
              "      <td>2010-06-12T23:59:22</td>\n",
              "      <td>Normal</td>\n",
              "      <td>192.168.2.110_216.49.88.12_2010-06-12T23</td>\n",
              "      <td>1864</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>SecureWeb</td>\n",
              "      <td>64</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>L2R</td>\n",
              "      <td>F,A</td>\n",
              "      <td>None</td>\n",
              "      <td>192.168.2.113</td>\n",
              "      <td>tcp_ip</td>\n",
              "      <td>4186</td>\n",
              "      <td>63.245.209.72</td>\n",
              "      <td>443</td>\n",
              "      <td>2010-06-12T23:58:11</td>\n",
              "      <td>2010-06-12T23:59:07</td>\n",
              "      <td>Normal</td>\n",
              "      <td>192.168.2.113_63.245.209.72_2010-06-12T23</td>\n",
              "      <td>4186</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             appName  ... res_port\n",
              "0        Unknown_UDP  ...     5353\n",
              "2            HTTPWeb  ...     4191\n",
              "4            HTTPWeb  ...     4192\n",
              "6  HTTPImageTransfer  ...     1864\n",
              "8          SecureWeb  ...     4186\n",
              "\n",
              "[5 rows x 22 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rKce95T6xFVi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7800f805-27a0-4646-a6a6-51b81a97c7e9"
      },
      "source": [
        "\n",
        "print(\"Stage I. Keys building\\n\")\n",
        "key = dSET.groupby('IPs_sequence')[['Tag', 'res_port']].agg({\"Tag\": lambda var_2: \"%s\" % ','.join([var_3 for var_3 in var_2]),\n",
        "          \"res_port\" :lambda var_2: \"%s\" % ','.join([str(var_3) if int(var_3) < 10000 else \"10000\" for var_3 in var_2])})\n",
        "\n",
        "print(\"Unique keys:\\n\" + str(key.count()))\n",
        "attacks = [var_4.split(\",\") for var_4 in key.Tag.tolist()]\n",
        "sequences = [var_4.split(\",\") for var_4 in key.res_port.tolist()]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stage I. Keys building\n",
            "\n",
            "Unique keys:\n",
            "Tag         11243\n",
            "res_port    11243\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8coEIzwnw7My",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f8ddb61d-86de-4f06-a71c-bea9e9b4448f"
      },
      "source": [
        "print(\"Stage II. Label encoding\\n\")\n",
        "U_tokens = list(set([var_5 for var_6 in sequences for var_5 in var_6]))\n",
        "print(\"Number of unique tokens :\", len(U_tokens))\n",
        "LE = LabelEncoder().fit(U_tokens)\n",
        "sequences = [LE.transform(var_7).tolist() for var_7 in sequences]\n",
        "sequences = [[var_6 + 1 for var_6 in var_5] for var_5 in sequences]\n",
        "print(\"Number of sequences :\", len(sequences))\n",
        "sequence_attack = zip(attacks, sequences)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stage II. Label encoding\n",
            "\n",
            "Number of unique tokens : 4168\n",
            "Number of sequences : 11243\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9nf6e4GbwbTF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c515b892-efdc-417b-cfde-e14dbe34b1be"
      },
      "source": [
        "print(\"Stage III. Sequences generating for the future model\\n\")\n",
        "var_8 = np.float32(0)\n",
        "len_sequence = 10\n",
        "print(\"Length of the primary sequence :\", len_sequence)\n",
        "seq_IDX, seq_X, seq_Y, seq_ATT = [], [], [], []\n",
        "for var_10, (var_11, var_12) in enumerate(sequence_attack):\n",
        "    sequence_1 = [np.float32(0)] * (len_sequence) + var_12\n",
        "    sequence_2 = [np.float32(0)] * (len_sequence) + var_11\n",
        "    for var_9 in range(len_sequence, len(sequence_1)):\n",
        "        sequence_3 = sequence_1[(var_9 - len_sequence):(var_9)]\n",
        "        var_14 = []\n",
        "        for var_13 in sequence_3:\n",
        "            try:\n",
        "                var_14.append(var_13)\n",
        "            except:\n",
        "                var_14.append(var_8)\n",
        "        seq_X.append(var_14)\n",
        "        seq_Y.append(sequence_1[var_9])\n",
        "        seq_IDX.append(var_10)\n",
        "        seq_ATT.append(sequence_2[var_9])\n",
        "print(\"Length of X & Y sets :\", len(seq_X))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stage III. Sequences generating for the future model\n",
            "\n",
            "Length of the primary sequence : 10\n",
            "Length of X & Y sets : 137160\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rv7gR5653034",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "592815ff-a86f-4e8a-d319-d6756c843e96"
      },
      "source": [
        "print(\"Stage IV. One-hot-encoder initializing\\n\")\n",
        "OHE = OneHotEncoder(sparse = False, categories = 'auto').fit(np.unique(seq_Y).reshape(-1, 1))\n",
        "\n",
        "X = np.array(seq_X)\n",
        "print(\"Dimensionality size of set X :\", X.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stage IV. One-hot-encoder initializing\n",
            "\n",
            "Dimensionality size of set X : (137160, 10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awr9mi4Aztqc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 539
        },
        "outputId": "27788c8f-9cee-402f-c3d1-ab6d29c87c18"
      },
      "source": [
        "from tensorflow import keras\n",
        "from keras.models import Sequential\n",
        "print(\"Stage V. Model building\\n\")\n",
        "drop_level = 0.35\n",
        "N_neurons = 50 \n",
        "\n",
        "model = Sequential()\n",
        "model.add(layers.Embedding(output_dim = 100,\n",
        "                    input_dim = len(U_tokens) + 1,\n",
        "                    mask_zero = True))\n",
        "\n",
        "model.add(layers.Bidirectional(LSTM(N_neurons, return_sequences = True)))\n",
        "model.add(layers.Dropout(drop_level))\n",
        "\n",
        "\n",
        "model.add(layers.Bidirectional(LSTM(N_neurons, activation = \"relu\", return_sequences = False)))\n",
        "model.add(layers.Dropout(drop_level))\n",
        "\n",
        "model.add(layers.Dense(N_neurons, activation = \"linear\"))\n",
        "model.add(layers.Dropout(drop_level))\n",
        "\n",
        "model.add(layers.Dense(len(U_tokens), activation = \"softmax\"))\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-21-7552eca677fd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSequential\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Stage V. Model building\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdrop_level\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.35\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mN_neurons\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;31m# See b/110718070#comment18 for more details about this import.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_layer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/models.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv2\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmetrics\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmetrics_module\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0moptimizer_v1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/backend.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotobuf\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconfig_pb2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meager\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_config\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend_config\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mImportError\u001b[0m: cannot import name 'get_config' from 'tensorflow.python.eager.context' (/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/context.py)",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PFp4lthL5Sqq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        },
        "outputId": "7ab91f97-36af-401d-fa63-521e8f3aad47"
      },
      "source": [
        "print(\"Stage VI. Compile and fit the model\\n\")\n",
        "\n",
        "batch_size = 512 \n",
        "n_epochs = 10     \n",
        "\n",
        "optim = tf.keras.optimizers.Nadam()   \n",
        "loss_f = tf.keras.metrics.categorical_crossentropy\n",
        "\n",
        "T_data = B_Generator(batch_size, np.asarray(X), np.asarray(seq_Y), OHE)\n",
        "\n",
        "model.compile(loss = loss_f,\n",
        "              optimizer = optim,\n",
        "              metrics = [\"accuracy\"])\n",
        "\n",
        "history = model.fit_generator(T_data.__iter__(),\n",
        "    steps_per_epoch = T_data.n_b,\n",
        "    epochs = n_epochs,\n",
        "    verbose = 1)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stage VI. Compile and fit the model\n",
            "\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-d3924c14d766>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mloss_f\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategorical_crossentropy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mT_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mB_Generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq_Y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOHE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m model.compile(loss = loss_f,\n",
            "\u001b[0;31mNameError\u001b[0m: name 'B_Generator' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZnD4msd55_7x"
      },
      "source": [
        "print(\"Stage VII. Results visualization\\n\")\n",
        "plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3MIvAwFyr1G"
      },
      "source": [
        "print(\"Stage VIII. Model saving & prediction checking\\n\")\n",
        "\n",
        "M_name = \"Model\"\n",
        "\n",
        "filepath = M_name + '.h5'\n",
        "tf.keras.models.save_model(model, filepath, include_optimizer = True, save_format = 'h5', overwrite = True)\n",
        "print(\"Size of the saved model :\", os.stat(filepath).st_size, \"bytes\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sHxL_kecZtS4"
      },
      "source": [
        "model_L = tf.keras.models.load_model(filepath)\n",
        "predicts = model_L.predict(X, batch_size = batch_size)\n",
        "print(\"Dimensionality sizes of model predicts :\", predicts.shape, \"\\n\")\n",
        "print(\"Compare with length of X & Y sets :\\t\", len(seq_X), \"\\nand with number of tokens :\\t\\t\", len(U_tokens))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}