Rap Embedding GRU Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hp5pH1KCwJrC"
   },
   "source": [
    "## Import Librairies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Mjfcl1Od_jyd"
   },
   "outputs": [],
   "source": [
    "import os \n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ast\n",
    "import datetime\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "from tqdm import tqdm \n",
    "\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2955,
     "status": "ok",
     "timestamp": 1675973968840,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "3JSTWLWq1e3W",
    "outputId": "420a2524-a2ab-4371-ce70-3e25be14cf90"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: tensorboard in /usr/local/lib/python3.8/dist-packages (2.9.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (0.6.1)\n",
      "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (1.21.6)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (2.16.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (1.8.1)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (3.19.6)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (0.38.4)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (1.51.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (3.4.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (1.0.1)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (2.25.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (0.4.6)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (57.4.0)\n",
      "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (1.4.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (0.2.8)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (1.15.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.8/dist-packages (from markdown>=2.6.8->tensorboard) (6.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (1.24.3)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (4.0.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard) (3.12.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.8/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (3.2.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jK8LrKg92HTI",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1675973968841,
     "user_tz": -60,
     "elapsed": 17,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     }
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "94fcbf75-9a14-4d32-f034-68ea229bd5cf"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The tensorboard extension is already loaded. To reload it, use:\n",
      "  %reload_ext tensorboard\n"
     ]
    }
   ],
   "source": [
    "# Load the TensorBoard notebook extension\n",
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "M7o7kzk3Ywvi"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import LambdaCallback, EarlyStopping,ModelCheckpoint\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout, Flatten\n",
    "from tensorflow.keras.layers import LSTM, GRU\n",
    "from tensorflow.keras.optimizers import RMSprop, Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1675973968842,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "N2wESKvS2PnO",
    "outputId": "92721889-3a43-41ee-d6cb-355d06469c80"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Version:  2.9.2\n",
      "Eager mode:  True\n",
      "GPU is available\n",
      "Found GPU at: /device:GPU:0\n"
     ]
    }
   ],
   "source": [
    "print(\"Version: \", tf.__version__)\n",
    "print(\"Eager mode: \", tf.executing_eagerly())\n",
    "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")\n",
    "\n",
    "device_name = tf.test.gpu_device_name()\n",
    "if device_name != '/device:GPU:0':\n",
    "    print(SystemError('GPU device not found'))\n",
    "print('Found GPU at: {}'.format(device_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 317,
     "status": "ok",
     "timestamp": 1675973969151,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "pVPhOQdlJNcx",
    "outputId": "668e30cf-49f3-41a4-df90-88670cf68879"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Thu Feb  9 20:19:41 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 510.47.03    Driver Version: 510.47.03    CUDA Version: 11.6     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA A100-SXM...  Off  | 00000000:00:04.0 Off |                    0 |\n",
      "| N/A   29C    P0    50W / 400W |   1656MiB / 40960MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A      4929      C                                    1653MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rfX9_6mUwOEa"
   },
   "source": [
    "## Set up drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-nEUIZu9wY42"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2nznAu2rwby-"
   },
   "outputs": [],
   "source": [
    "DRIVEWORKSPACE_PATH = \"/content/gdrive/Shareddrives/# Zone de Code #/AIRapFR\"\n",
    "DATASETS_DIR_PATH = f\"{DRIVEWORKSPACE_PATH}/data/datasets\"\n",
    "\n",
    "OUTPUTS_DIR_PATH = f\"{DRIVEWORKSPACE_PATH}/ai/outputs\"  #txt files\n",
    "LOGS_DIR =  f\"{DRIVEWORKSPACE_PATH}/ai/logs\"\n",
    "\n",
    "MODELS_DIR_PATH = f\"{DRIVEWORKSPACE_PATH}/ai/models\"\n",
    "LSTM_MODELS_DIR_PATH = f\"{MODELS_DIR_PATH}/lstm\"\n",
    "\n",
    "for p in [DRIVEWORKSPACE_PATH, DATASETS_DIR_PATH, MODELS_DIR_PATH, LSTM_MODELS_DIR_PATH, OUTPUTS_DIR_PATH]:\n",
    "    if not os.path.exists(p):\n",
    "        raise Exception(f\"{p} not exists\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XaFMiXMwzgKa"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(DRIVEWORKSPACE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YDGZnm55wJad"
   },
   "outputs": [],
   "source": [
    "from utils import create_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TnSa0XcJzQ7l"
   },
   "source": [
    "## Pre-training\n",
    "\n",
    "on fr rappers corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zU_oIp-VBPXC"
   },
   "source": [
    "### Data for pretraining 📁"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 34307,
     "status": "ok",
     "timestamp": 1675974005724,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "IARqfBHF319i",
    "outputId": "39fee42a-0074-49e8-b604-cd4e51ddbabb"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Loaded dataset with pickle from /content/gdrive/Shareddrives/# Zone de Code #/AIRapFR/data/datasets/dataset-1273-keryjames-20230208/rap_corpus__26722__tok_rmpunct__min_freq_2__sent_len_10__step_1__p_test_0.1__max_year_2015.pkl.\n",
      " > len(full_dataset): 6642588\n",
      " > len(train): 5978330 # ['sport', 'et', 'en', 'chant', '\\n', 'délinquant', 'rentre', 'à', \"l'\", 'heure'] où \n",
      " > len(test): 664258 # ['gradés', 'dans', 'mon', 'taf', '\\n', 'jalousé', 'par', 'les', 'rappeurs', 'du'] game \n",
      " > len(set_words): 64076 # ['likenon', 'x16', 'amoureusement', 'jonque', 'tripe', 'démontage', 'sites', 'drugstore', 'visqueux', 'ateyaba'] \n",
      " > len(word_index_dict): 64076 # [('06', 1), ('3ash9', 2), ('bolo', 3), ('excentriques', 4), ('schtroumpfettes', 5)] \n"
     ]
    }
   ],
   "source": [
    "from data.load_dataset import load_dataset_with_pickle\n",
    "\n",
    "dataset_dir_path = f\"{DATASETS_DIR_PATH}/dataset-1273-keryjames-20230208\"\n",
    "artist_name = os.path.basename(dataset_dir_path).split(\"-\")[-2]\n",
    "sentences_train_rap, next_words_train_rap, sentences_test_rap, next_words_test_rap, set_words_rap, word_index_dict_union, len_dataset_rap = load_dataset_with_pickle(dataset_dir_path, rap_corpus=True, artist_name_focus=artist_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PyyS7wFGVeGb"
   },
   "outputs": [],
   "source": [
    "word_indices = word_index_dict_union\n",
    "indices_word = {v:k for k,v in word_index_dict_union.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 29,
     "status": "ok",
     "timestamp": 1675974005726,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "7ox9mnUVVb24",
    "outputId": "003f89b6-aa8b-429e-a0a3-add165357583"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "metadata": {},
     "execution_count": 55
    }
   ],
   "source": [
    "SEQUENCE_LEN = len(sentences_train_rap[0])\n",
    "SEQUENCE_LEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5ZJcYE5AwjtR"
   },
   "source": [
    "La fonction `generator` sert à générer des données pour entraîner le modèle réseau de neurones LSTM. Elle prend en entrée des listes de phrases et de mots suivants, ainsi qu'une taille de batch, et renvoie des entrées et sorties pour l'entraînement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 27,
     "status": "ok",
     "timestamp": 1675974005726,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "dDkr7nrIbAFv",
    "outputId": "5e45af86-91f1-42bd-825f-380a869f541d"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(64625, 64076)"
      ]
     },
     "metadata": {},
     "execution_count": 56
    }
   ],
   "source": [
    "len(word_indices), len(set_words_rap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 24,
     "status": "ok",
     "timestamp": 1675974005727,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "VQJgnIcSweet",
    "outputId": "f3bd55b9-c8ac-4314-dfb9-9aaca6a4dd31"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(128, 10, 64625)\n",
      "(128, 64625)\n"
     ]
    }
   ],
   "source": [
    "def data_generator(sentence_list, next_word_list, batch_size, sequence_length, words_dict):\n",
    "    index = 0\n",
    "    while True:\n",
    "        x = np.zeros((batch_size, sequence_length, len(words_dict)), dtype=bool)\n",
    "        y = np.zeros((batch_size, len(words_dict)), dtype=bool)\n",
    "        for i in range(batch_size):\n",
    "            for t, w in enumerate(sentence_list[index]):\n",
    "                x[i, t, words_dict[w]] = 1\n",
    "            y[i, words_dict[next_word_list[index]]] = 1\n",
    "\n",
    "            index = (index + 1) % len(sentence_list)\n",
    "        yield x, y\n",
    "\n",
    "partial_data_generator = partial(data_generator,sequence_length=SEQUENCE_LEN,words_dict=word_indices)\n",
    "\n",
    "for x,y in partial_data_generator(sentences_train_rap, next_words_train_rap, batch_size=128):\n",
    "    print(x.shape)\n",
    "    print(y.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# Data generator for fit and evaluate\n",
    "def generator_embedding(sentence_list, next_words_list, batch_size, sequence_length, words_dict):\n",
    "    index = 0\n",
    "    while True:\n",
    "        x = np.zeros((batch_size, sequence_length), dtype=np.bool)\n",
    "        y = np.zeros((batch_size), dtype=np.bool)\n",
    "        for i in range(batch_size):\n",
    "            for t, w in enumerate(sentence_list[index % len(sentence_list)]):\n",
    "                x[i, t] = words_dict[w]\n",
    "            y[i] = next_words_list[index % len(sentence_list)]\n",
    "            index = index + 1\n",
    "        yield x, y\n",
    "        "
   ],
   "metadata": {
    "id": "O1WybRMTwi65"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9l5aqoy-wvOv"
   },
   "source": [
    "### Building the LSTM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_usBy2V_x8Pp"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Reshape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cXijBvQcpcSE"
   },
   "source": [
    "### Building the GRU Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Q9hjWH6Bpcvm"
   },
   "outputs": [],
   "source": [
    "def build_GRU(max_length, len_vocab, dropout=0.2,optimiser_name=\"rmsprop\",optimizer_lr=0.01):\n",
    "    \"\"\"\n",
    "    Build the GRU network structure and compile the model\n",
    "    \"\"\"\n",
    "    model = Sequential(name=f\"model_gru1__{max_length}_{len_vocab}_{dropout}_{optimiser_name}_{optimizer_lr}\")\n",
    "    model.add(GRU(128, input_shape=(max_length, len_vocab)))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Reshape((1, 128)))\n",
    "    model.add(GRU(128))\n",
    "    model.add(Dropout(dropout))\n",
    "\n",
    "    model.add(Dense(len_vocab))\n",
    "    model.add(Activation('softmax'))\n",
    "    optimizer=Adam(learning_rate=optimizer_lr) if optimiser_name == \"adam\" else RMSprop(learning_rate=optimizer_lr)\n",
    "    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=[\"accuracy\"])\n",
    "    return model\n",
    "\n",
    "def build_GRU2(max_length, len_vocab, dropout=0.2):\n",
    "    \"\"\"\n",
    "    Build the GRU network structure and compile the model\n",
    "    \"\"\"\n",
    "    model = Sequential(name=f\"model_gru2__{max_length}_{len_vocab}_{dropout}_adam\")\n",
    "    model.add(GRU(32, input_shape=(max_length, len_vocab)))\n",
    "    model.add(Dropout(dropout))\n",
    "    #model.add(Reshape((1, 128)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(len_vocab+50, activation = 'softmax'))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Dense(dropout, activation = 'softmax'))\n",
    "    model.compile(optimizer=\"Adam\", loss='categorical_crossentropy', metrics=[\"accuracy\"])\n",
    "    return model\n",
    "\n",
    "\n",
    "def build_EMB_GRU(max_length, len_vocab, dropout=0.2):\n",
    "    \"\"\"\n",
    "    Build the GRU network structure and compile the model\n",
    "    \"\"\"\n",
    "    model = Sequential(name=f\"model_emb_gru__{max_length}_{len_vocab}_{dropout}_adam\")\n",
    "\n",
    "    model.add(Embedding(len_vocab, 64))\n",
    "    model.add(Dropout(dropout))\n",
    "    #model.add(Reshape((1, 128)))\n",
    "    model.add(GRU(64))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(len_vocab))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(optimizer=\"Adam\", loss='categorical_crossentropy', metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 300,
     "status": "ok",
     "timestamp": 1675974008463,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "xSihabOwsq9H",
    "outputId": "17b9f0f7-a8ae-48b9-ae60-1887dae2ca77"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Model: \"model_gru1__10_64625_0.2_rmsprop_0.01\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " gru_10 (GRU)                (None, 128)               24865920  \n",
      "                                                                 \n",
      " dropout_14 (Dropout)        (None, 128)               0         \n",
      "                                                                 \n",
      " reshape_3 (Reshape)         (None, 1, 128)            0         \n",
      "                                                                 \n",
      " gru_11 (GRU)                (None, 128)               99072     \n",
      "                                                                 \n",
      " dropout_15 (Dropout)        (None, 128)               0         \n",
      "                                                                 \n",
      " dense_9 (Dense)             (None, 64625)             8336625   \n",
      "                                                                 \n",
      " activation_5 (Activation)   (None, 64625)             0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 33,301,617\n",
      "Trainable params: 33,301,617\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_gru_1 = build_GRU(SEQUENCE_LEN, len_vocab=len(word_indices))\n",
    "model_gru_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "model_gru_2 = build_GRU2(SEQUENCE_LEN, len_vocab=len(word_indices))\n",
    "model_gru_2.summary()"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1S0pU37tzjCr",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1675974008681,
     "user_tz": -60,
     "elapsed": 220,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     }
    },
    "outputId": "693b7f98-190a-4cf8-f087-97a30f2a60bb"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Model: \"model_gru2__10_64625_0.2_adam\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " gru_12 (GRU)                (None, 32)                6207264   \n",
      "                                                                 \n",
      " dropout_16 (Dropout)        (None, 32)                0         \n",
      "                                                                 \n",
      " flatten_2 (Flatten)         (None, 32)                0         \n",
      "                                                                 \n",
      " dense_10 (Dense)            (None, 64675)             2134275   \n",
      "                                                                 \n",
      " dropout_17 (Dropout)        (None, 64675)             0         \n",
      "                                                                 \n",
      " dense_11 (Dense)            (None, 0)                 0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 8,341,539\n",
      "Trainable params: 8,341,539\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "model_emb_gru = build_EMB_GRU(SEQUENCE_LEN, len_vocab=len(word_indices))\n",
    "model_emb_gru.summary()"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BdVII5lI0AxD",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1675974008900,
     "user_tz": -60,
     "elapsed": 220,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     }
    },
    "outputId": "c0503187-5b9f-49df-f961-c8d611ec4f66"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Model: \"model_emb_gru__10_64625_0.2_adam\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_2 (Embedding)     (None, None, 64)          4136000   \n",
      "                                                                 \n",
      " dropout_18 (Dropout)        (None, None, 64)          0         \n",
      "                                                                 \n",
      " gru_13 (GRU)                (None, 64)                24960     \n",
      "                                                                 \n",
      " dropout_19 (Dropout)        (None, 64)                0         \n",
      "                                                                 \n",
      " dense_12 (Dense)            (None, 64625)             4200625   \n",
      "                                                                 \n",
      " activation_6 (Activation)   (None, 64625)             0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 8,361,585\n",
      "Trainable params: 8,361,585\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "In9E2AZkXrjE"
   },
   "source": [
    "### Training functions and params ⚙"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vEhX0SjIYH9L"
   },
   "source": [
    "pre training params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1675974008901,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "2nGF248gx2V7",
    "outputId": "2201dddd-11ba-4256-84ef-018f05e2c068"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "256"
      ]
     },
     "metadata": {},
     "execution_count": 64
    }
   ],
   "source": [
    "2**8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1675974008901,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "FZm5vNteYDU6",
    "outputId": "6ea6daf4-28a6-4e91-c62f-e892c41d37d5"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(6486, 512)"
      ]
     },
     "metadata": {},
     "execution_count": 65
    }
   ],
   "source": [
    "BATCH_SIZE = 2**10\n",
    "EPOCHS = 50\n",
    "RESET_PRETRAINING = False\n",
    "\n",
    "len_dataset_rap//BATCH_SIZE, 2**9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YX30W8ELYK_H"
   },
   "source": [
    "fine tuning params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6YQktJAxYRFn"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE_FINE = 256\n",
    "EPOCHS_FINE = 10\n",
    "RESET_FINE_TRAINING = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0SK7tX1LYXUD"
   },
   "source": [
    "train fct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "99abns88Xu0_"
   },
   "outputs": [],
   "source": [
    "def train_model_bis(model,epochs,batch_size,len_sentences_dataset,sentences_train, next_words_train,sentences_test, next_words_test,callbacks_list = None ):\n",
    "    #todo : n_teps\n",
    "    n_steps = len_sentences_dataset//batch_size + 1\n",
    "    #n_steps = 2**9\n",
    "    \n",
    "    history = model.fit(partial_data_generator(sentences_train, next_words_train, batch_size=batch_size),\n",
    "                              steps_per_epoch=n_steps,\n",
    "                              epochs=epochs, callbacks=callbacks_list,\n",
    "\n",
    "                              validation_data=partial_data_generator(sentences_test, next_words_test, batch_size=batch_size), \n",
    "                              validation_steps=n_steps )\n",
    "    return history, model\n",
    "    \n",
    "train_model = partial(train_model_bis,len_sentences_dataset=len_dataset_rap)"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "def train_model_bis2(model,epochs,batch_size,len_sentences_dataset,sentences_train, next_words_train,sentences_test, next_words_test,callbacks_list = None ):\n",
    "    #todo : n_teps\n",
    "    n_steps = len_sentences_dataset//batch_size + 1\n",
    "    #n_steps = 2**9\n",
    "    \n",
    "    history = model.fit(partial_data_generator(sentences_train, next_words_train, batch_size=batch_size),\n",
    "                              steps_per_epoch=n_steps,\n",
    "                              epochs=epochs, callbacks=callbacks_list,\n",
    "\n",
    "                              validation_data=partial_data_generator(sentences_test, next_words_test, batch_size=batch_size), \n",
    "                              validation_steps=n_steps )\n",
    "    return history, model\n",
    "    \n",
    "train_model2 = partial(train_model_bis,len_sentences_dataset=len_dataset_rap)"
   ],
   "metadata": {
    "id": "Ac2rQO174a17"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HSek9hKE3IeR"
   },
   "source": [
    "### Generate Text & Tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1675974008902,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "7Kn6l3ZliwLp",
    "outputId": "f7a5b277-de0b-4bf4-fe56-cba9bef6f22b"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "metadata": {},
     "execution_count": 68
    }
   ],
   "source": [
    "SEQUENCE_LEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3ZUGpnmv9HMw"
   },
   "outputs": [],
   "source": [
    "def sample(preds, temperature=1.0):\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.where(preds == 0, 1e-8, preds) # remplacer toutes les valeurs égales à zéro par une petite valeur proche de zéro\n",
    "    preds = np.log(preds) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1) #todo : prendre le deuxième\n",
    "    index_max_1 = np.argmax(probas) \n",
    "    \n",
    "    # Trier les prédictions pour obtenir les indices des deux plus grands éléments\n",
    "    sorted_indices = np.argsort(preds)[::-1][:2]\n",
    "    \n",
    "    return sorted_indices[0], sorted_indices[1]\n",
    "\n",
    "\n",
    "def generate_text(model, starting_words,window_size, words_indices, indices_words, max_length=50, diversity=1):\n",
    "    assert max_length > len(starting_words),f\"max_length {max_length} must > len(starting_words) {len(starting_words)}\"\n",
    "    length = max_length - len(starting_words)\n",
    "    \n",
    "    generated = starting_words[:]\n",
    "    sentence = starting_words[-window_size:]\n",
    "    for i in range(length):\n",
    "        x_pred = np.zeros((1, window_size, len(words_indices)))\n",
    "        #print(sentence)\n",
    "        for t, word in enumerate(sentence):\n",
    "            x_pred[0, t, words_indices[word]] = 1.\n",
    "\n",
    "        preds = model.predict(x_pred, verbose=0)[0]\n",
    "        \n",
    "        next_index, else_next_index = sample(preds, diversity)\n",
    "        #print([indices_words[next_index],indices_words[else_next_index]])\n",
    "        next_word = indices_words[else_next_index if sentence[-1] == \"\\n\" and indices_word[next_index] == \"\\n\"  else next_index]\n",
    "        #print(generated)\n",
    "        generated.append(next_word)\n",
    "        sentence = sentence[-(window_size-1):] + [next_word]\n",
    "    return generated\n",
    "\n",
    "#generate_text(model_lstm_1, ['je',\"suis\"],window_size=SEQUENCE_LEN, words_indices=word_indices, indices_words=indices_word, max_length=25,diversity=.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Qh-YXSZaegfp"
   },
   "outputs": [],
   "source": [
    "def get_str_time_now():\n",
    "    return datetime.datetime.now().strftime(\"%Y%m%d%H%M\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-_kTST9sXZFo"
   },
   "outputs": [],
   "source": [
    "def on_epoch_end(epoch, logs, model, sequence_len, words_indices, indices_words,start_time_str, max_length=15,epochs_looked = [0,25,50]):\n",
    "    if epochs_looked is None:\n",
    "        return\n",
    "    if epoch in epochs_looked:\n",
    "        first_words = ['je',\"suis\"]\n",
    "        output_filepath = os.path.join(OUTPUTS_DIR_PATH,f\"outputs__{model.name}__{start_time_str}.txt\")\n",
    "        \n",
    "\n",
    "        with open(output_filepath, 'w' if not os.path.exists(output_filepath) else 'a', encoding='utf8') as output_file:\n",
    "            h1 = f'\\n----- Generating text after Epoch: {epoch} ({\" \".join(first_words)})'\n",
    "            print(h1)\n",
    "            output_file.write(h1)\n",
    "            for diversity in [0.2, 0.5, 1.0]:\n",
    "                h2 = f'----- Diversity: {diversity} ----- '\n",
    "                print(h2)\n",
    "                output_file.write(h2)\n",
    "                generated_text_list = generate_text(model, first_words ,window_size=sequence_len, words_indices=word_indices, indices_words=indices_word, diversity=diversity, max_length=max_length)\n",
    "                generated_text = ' '.join(generated_text_list)\n",
    "                print(generated_text)\n",
    "                output_file.write(generated_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "L61EZCEWcQQS"
   },
   "outputs": [],
   "source": [
    "def plot_learning_curves(history, title=\"\"):\n",
    "    acc      = history.history[\"accuracy\"]\n",
    "    loss     = history.history[\"loss\"]\n",
    "    val_acc  = history.history[\"val_accuracy\"]\n",
    "    val_loss = history.history[\"val_loss\"]\n",
    "    epochs = range(len(acc))\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,5))\n",
    "    \n",
    "    fig.suptitle(title, fontsize=\"x-large\")\n",
    "    \n",
    "    ax1.plot(epochs, acc, label=\"Entraînement\")\n",
    "    ax1.plot(epochs, val_acc, label=\"Validation\")\n",
    "    ax1.set_title(\"Accuracy - Données entraînement vs. validation.\")\n",
    "    ax1.set_ylabel(\"Accuracy (%)\")\n",
    "    ax1.set_xlabel(\"Epoch\")\n",
    "    ax1.legend()\n",
    "    \n",
    "    ax2.plot(epochs, loss, label=\"Entraînement\")\n",
    "    ax2.plot(epochs, val_loss, label=\"Validation\")\n",
    "    ax2.set_title(\"Perte - Données entraînement vs. validation.\")\n",
    "    ax2.set_ylabel('Perte')\n",
    "    ax2.set_xlabel('Epoch')\n",
    "    ax2.legend()\n",
    "\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UHvOiHgiZcL9"
   },
   "source": [
    "### TensorBoard 📊"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SMexQBMXZf9Z"
   },
   "outputs": [],
   "source": [
    "%tensorboard --logdir \"$LOGS_DIR\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qQvekiCVw3zu"
   },
   "source": [
    "### Pre-training 🔧 (model_gru_1)\n",
    "\n",
    "on fr rappers corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yZjlY35H0Nhx"
   },
   "source": [
    "#### checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 513,
     "status": "ok",
     "timestamp": 1675974051570,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "id": "mjcIop6Lczdj",
    "outputId": "8f786f9e-046b-4ba2-bf65-41be6bd55959"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Pas de poids précédents trouvés, entraînement à partir de zéro\n"
     ]
    }
   ],
   "source": [
    "weights_file = create_dir(LSTM_MODELS_DIR_PATH ,\"pre-trained\") + f\"/{model_gru_1.name}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(weights_file, monitor='val_loss',  save_best_only=True, save_weights_only=False) \n",
    "\n",
    "if os.path.exists(weights_file) and not RESET_PRETRAINING:\n",
    "    model_gru_1.load_weights(weights_file)\n",
    "    print(\"Chargé les poids précédents\")\n",
    "else:\n",
    "    print(\"Pas de poids précédents trouvés, entraînement à partir de zéro\")\n",
    "                                               "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NSsRLjs5719M"
   },
   "source": [
    "#### early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "j1t9R1MUx3lT"
   },
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_G3BbnZo74tv"
   },
   "source": [
    "#### print on epoch end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ABAR7YShwRKa"
   },
   "outputs": [],
   "source": [
    "\n",
    "print_callback_gru_1 = LambdaCallback(on_epoch_end=partial(on_epoch_end, model=model_gru_1, sequence_len=SEQUENCE_LEN, words_indices=word_indices, indices_words=indices_word,start_time_str=get_str_time_now(), max_length=15,epochs_looked = None)\n",
    ")  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wQMP8BNH8CFl"
   },
   "source": [
    "#### tensorboard "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "csEJkFfu7tvf"
   },
   "outputs": [],
   "source": [
    "logdir = os.path.join(LOGS_DIR, datetime.datetime.now().strftime(f\"{model_gru_1.name} pre-trained %Y%m%d-%H%M%S\"))\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hXot8S10xHyg"
   },
   "source": [
    "#### 🔥 Launch pre-training 🦾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NfpzULjDDpt8",
    "outputId": "96055e10-52c1-4856-c140-d159238950ac"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/50\n",
      "3351/6487 [==============>...............] - ETA: 15:22 - loss: 5.8252 - accuracy: 0.1786"
     ]
    }
   ],
   "source": [
    "history_gru_1, model_gru_1 = train_model(model=model_gru_1,epochs=EPOCHS,batch_size=BATCH_SIZE, sentences_train=sentences_train_rap, next_words_train=next_words_train_rap,sentences_test=sentences_test_rap, next_words_test=next_words_test_rap,callbacks_list=[print_callback_gru_1, checkpoint, early_stopping, tensorboard_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EjuV1d0dchhh"
   },
   "outputs": [],
   "source": [
    "plot_learning_curves(history_gru_1, title=f\"Modèle GRU 1 ({model_gru_1.name})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jT7hQtVFauez"
   },
   "source": [
    "### ... free space 🗑️"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GMbimZhGayBt"
   },
   "outputs": [],
   "source": [
    "del model_gru_1"
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "JSG77S5V10Nh"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HZXXOQKu10cX"
   },
   "source": [
    "### Pre-training 🔧 (model_gru_2)\n",
    "\n",
    "on fr rappers corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HpI2tSfR10cX"
   },
   "source": [
    "#### checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 513,
     "status": "ok",
     "timestamp": 1675974051570,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "outputId": "8f786f9e-046b-4ba2-bf65-41be6bd55959",
    "id": "lZ6qgcV110cY"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Pas de poids précédents trouvés, entraînement à partir de zéro\n"
     ]
    }
   ],
   "source": [
    "weights_file = create_dir(LSTM_MODELS_DIR_PATH ,\"pre-trained\") + f\"/{model_gru_2.name}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(weights_file, monitor='val_loss',  save_best_only=True, save_weights_only=False) \n",
    "\n",
    "if os.path.exists(weights_file) and not RESET_PRETRAINING:\n",
    "    model_gru_2.load_weights(weights_file)\n",
    "    print(\"Chargé les poids précédents\")\n",
    "else:\n",
    "    print(\"Pas de poids précédents trouvés, entraînement à partir de zéro\")\n",
    "                                               "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VLqVdJhS10cZ"
   },
   "source": [
    "#### early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "73-iEkCm10ca"
   },
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kh3fpor110ca"
   },
   "source": [
    "#### print on epoch end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Al0dSn7U10ca"
   },
   "outputs": [],
   "source": [
    "\n",
    "print_callback_gru_2 = LambdaCallback(on_epoch_end=partial(on_epoch_end, model=model_gru_2, sequence_len=SEQUENCE_LEN, words_indices=word_indices, indices_words=indices_word,start_time_str=get_str_time_now(), max_length=15,epochs_looked = None)\n",
    ")  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oVRlP89j10ca"
   },
   "source": [
    "#### tensorboard "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ug--5Lq010ca"
   },
   "outputs": [],
   "source": [
    "logdir = os.path.join(LOGS_DIR, datetime.datetime.now().strftime(f\"{model_gru_2.name} pre-trained %Y%m%d-%H%M%S\"))\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hdNv4C7310cb"
   },
   "source": [
    "#### 🔥 Launch pre-training 🦾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "96055e10-52c1-4856-c140-d159238950ac",
    "id": "G8I1pUD810cb"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/50\n",
      " 126/6487 [..............................] - ETA: 30:34 - loss: 6.9683 - accuracy: 0.0992"
     ]
    }
   ],
   "source": [
    "history_gru_2, model_gru_2 = train_model(model=model_gru_2,epochs=EPOCHS,batch_size=BATCH_SIZE, sentences_train=sentences_train_rap, next_words_train=next_words_train_rap,sentences_test=sentences_test_rap, next_words_test=next_words_test_rap,callbacks_list=[print_callback_gru_2, checkpoint, early_stopping, tensorboard_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dwMeg8Wr10cb"
   },
   "outputs": [],
   "source": [
    "plot_learning_curves(history_gru_1, title=f\"Modèle GRU 2 ({model_gru_1.name})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uh-LHi5V10cb"
   },
   "source": [
    "### ... free space 🗑️"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "daWRpYuZ10cb"
   },
   "outputs": [],
   "source": [
    "del model_gru_2"
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "aU4QbRW63J1_"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "28uMBxgL3KMW"
   },
   "source": [
    "### Pre-training 🔧 (model_emb_gru)\n",
    "\n",
    "on fr rappers corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hxN3vn5P3KMX"
   },
   "source": [
    "#### checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 513,
     "status": "ok",
     "timestamp": 1675974051570,
     "user": {
      "displayName": "M SAID",
      "userId": "16677121514363856115"
     },
     "user_tz": -60
    },
    "outputId": "8f786f9e-046b-4ba2-bf65-41be6bd55959",
    "id": "04dwrxej3KMX"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Pas de poids précédents trouvés, entraînement à partir de zéro\n"
     ]
    }
   ],
   "source": [
    "weights_file = create_dir(LSTM_MODELS_DIR_PATH ,\"pre-trained\") + f\"/{model_emb_gru.name}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(weights_file, monitor='val_loss',  save_best_only=True, save_weights_only=False) \n",
    "\n",
    "if os.path.exists(weights_file) and not RESET_PRETRAINING:\n",
    "    model_emb_gru.load_weights(weights_file)\n",
    "    print(\"Chargé les poids précédents\")\n",
    "else:\n",
    "    print(\"Pas de poids précédents trouvés, entraînement à partir de zéro\")\n",
    "                                               "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QjoIlGoF3KMX"
   },
   "source": [
    "#### early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FICpjnn33KMX"
   },
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZwKsyLAO3KMX"
   },
   "source": [
    "#### print on epoch end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zj2fbBp23KMX"
   },
   "outputs": [],
   "source": [
    "\n",
    "print_callback_emb_gru = LambdaCallback(on_epoch_end=partial(on_epoch_end, model=model_emb_gru, sequence_len=SEQUENCE_LEN, words_indices=word_indices, indices_words=indices_word,start_time_str=get_str_time_now(), max_length=15,epochs_looked = None)\n",
    ")  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I-VGR23n3KMY"
   },
   "source": [
    "#### tensorboard "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vHdAoQdV3KMY"
   },
   "outputs": [],
   "source": [
    "logdir = os.path.join(LOGS_DIR, datetime.datetime.now().strftime(f\"{model_emb_gru.name} pre-trained %Y%m%d-%H%M%S\"))\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SltOBfLZ3KMY"
   },
   "source": [
    "#### 🔥 Launch pre-training 🦾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "96055e10-52c1-4856-c140-d159238950ac",
    "id": "g-p-O1cW3KMY"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/50\n",
      " 126/6487 [..............................] - ETA: 30:34 - loss: 6.9683 - accuracy: 0.0992"
     ]
    }
   ],
   "source": [
    "history_gru_2, model_emb_gru = train_model(model=model_emb_gru,epochs=EPOCHS,batch_size=BATCH_SIZE, sentences_train=sentences_train_rap, next_words_train=next_words_train_rap,sentences_test=sentences_test_rap, next_words_test=next_words_test_rap,callbacks_list=[print_callback_emb_gru, checkpoint, early_stopping, tensorboard_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YFOvnb5N3KMY"
   },
   "outputs": [],
   "source": [
    "plot_learning_curves(history_gru_1, title=f\"Modèle EMB GRU ({model_emb_gru.name})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AwtIPA5j3KMY"
   },
   "source": [
    "### ... free space 🗑️"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NF47dcQt3KMY"
   },
   "outputs": [],
   "source": [
    "del model_emb_gru"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YCJ4EZGJB_vc"
   },
   "source": [
    "## Fine-tuning\n",
    " on 1 artist corpus "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Jhtonhe640c7"
   },
   "source": [
    "##### Data for finetuning 📁"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zfPh1YWUS2nT"
   },
   "outputs": [],
   "source": [
    "sentences_train_artist, next_words_train_artist, sentences_test_artist, next_words_test_artist, set_words_artist, word_index_dict_union, len_dataset = load_dataset_with_pickle(dataset_dir_path, rap_corpus=False, artist_name_focus=artist_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VOJD5a52xsTM"
   },
   "source": [
    "##### load pre-trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OepujGs0x0zb"
   },
   "outputs": [],
   "source": [
    "# Charger les poids du modèle précédemment entraîné\n",
    "weights_file = LSTM_MODELS_DIR_PATH + \"/pre-trained/model_lstm_1.hdf5\"\n",
    "#del model_lstm_1\n",
    "model_pretrained_lstm_1 = load_model(weights_file)\n",
    "print(model_pretrained_lstm_1)\n",
    "for diversity in [0.2,0.5,0.8,1.0]:\n",
    "    print(f'----- Diversity: {diversity}')\n",
    "    generated_text = generate_text(model_pretrained_lstm_1, ['je',\"suis\"],window_size=SEQUENCE_LEN, words_indices=word_indices, indices_words=indices_word, diversity=diversity, max_length=100)\n",
    "    \n",
    "    print(' '.join(generated_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3zCY2RSOfuyB"
   },
   "source": [
    "##### checkpoint final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wbyx79V2fv30"
   },
   "outputs": [],
   "source": [
    "weights_file = LSTM_MODELS_DIR_PATH + f\"/model_lstm_1_{artist_name}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(weights_file, monitor='val_loss',  save_best_only=True, save_weights_only=False) \n",
    "\n",
    "if os.path.exists(weights_file) and not RESET_FINE_TRAINING:\n",
    "    model_lstm_1.load_weights(weights_file)\n",
    "    print(\"Chargé les poids précédents\")\n",
    "else:\n",
    "    print(\"Pas de poids précédents trouvés, entraînement à partir de zéro\")\n",
    "                                               "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VZ3QyY3tlVFs"
   },
   "source": [
    "##### early stopping final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gxkS6aBTkgV4"
   },
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "juRSBd9yHKgW"
   },
   "source": [
    "##### tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QfbZLdXuHMfV"
   },
   "outputs": [],
   "source": [
    "logdir = os.path.join(\"logs\", datetime.datetime.now().strftime(\"model_lstm_1 final %Y%m%d-%H%M%S\"))\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7cThKxbm7PEr"
   },
   "source": [
    "#### 🔥 Launch training 🦾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "R37xEMYAzmxb"
   },
   "outputs": [],
   "source": [
    "# Compiler à nouveau le modèle\n",
    "model_pretrained_lstm_1.compile(optimizer=RMSprop(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Entraîner le modèle sur les données de l'artiste particulier\n",
    "history_final_lstm_1,model_final_lstm_1 = train_model(model=model_lstm_1,epochs=EPOCHS_FINE,batch_size=BATCH_SIZE_FINE, sentences_train=sentences_train_artist, next_words_train=next_words_train_artist,\n",
    "                             sentences_test=sentences_test_artist, next_words_test=next_words_test_artist, callbacks_list=[print_callback_lstm_1,checkpoint,early_stopping,tensorboard_callback ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "art_J2wwIvaC"
   },
   "outputs": [],
   "source": [
    "plot_learning_curves(history_final_lstm_1, title=\"Modèle Final LSTM\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eOUgayzwxnEu"
   },
   "source": [
    "##### save final 💾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UXjUPaC_Huij"
   },
   "outputs": [],
   "source": [
    "weights_file = LSTM_MODELS_DIR_PATH + \"/\" + f\"{model_final_lstm_1.name}.hdf5\"\n",
    "model_final_lstm_1.save(weights_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qCrb76oDJBQ4"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "HZXXOQKu10cX"
   ],
   "machine_shape": "hm",
   "provenance": [],
   "toc_visible": true
  },
  "gpuClass": "premium",
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}