Arabic_lyrics_RNN_word_level.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import helper\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from distutils.version import LooseVersion\n",
    "import warnings\n",
    "import tensorflow as tf\n",
    "import re\n",
    "\n",
    "data_dir = 'aggregate_lyrics.txt'\n",
    "text = helper.load_data(data_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Stats\n",
      "Roughly the number of unique words: 109977\n",
      "The sentences 0 to 10:\n",
      "اروح لاحبابي و الاقي الفرح ساكن عينهم \n",
      " ابتسم لافراحهم و انا من الهم احترق \n",
      " و اسال جروحي من تري حس بعذابي منهم \n",
      " و بالحقيقه انصدم محدن معه همي فرق \n",
      "دورت في كل الوجيه حسيت غربه بينهم \n",
      " مع الاسف محدن ابد حس بعذاباتي و رق \n",
      " جيت اتعثر بالتعب ابي اشوف يدينهم \n",
      " ماكنت ابي الا احد يحس بي لو مانطق \n",
      " و حز فيني اني رجعت لكن رجعت بدونهم \n",
      " يحز في نفسي بانه ماسوي جرحي صدق \n"
     ]
    }
   ],
   "source": [
    "view_sentence_range = (0, 10)\n",
    "\n",
    "print('Dataset Stats')\n",
    "print('Roughly the number of unique words: {}'.format(len({word: None for word in text.split()})))\n",
    "print('The sentences {} to {}:'.format(*view_sentence_range))\n",
    "print('\\n'.join(text.split('\\n')[view_sentence_range[0]:view_sentence_range[1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_lookup_tables(text):\n",
    "    \"\"\"\n",
    "    Create lookup tables for vocabulary\n",
    "    :param text: The text of lyrics split into words\n",
    "    :return: A tuple of dicts (vocab_to_int, int_to_vocab)\n",
    "    \"\"\"\n",
    "    \n",
    "    word_counts = set(text)\n",
    "    int_to_vocab = {ii: word for ii, word in enumerate(word_counts)}\n",
    "    vocab_to_int = {word: ii for ii, word in int_to_vocab.items()}    \n",
    "    \n",
    "    return (vocab_to_int, int_to_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def token_lookup():\n",
    "    \"\"\"\n",
    "    Generate a dict to turn punctuation into a token.\n",
    "    :return: Tokenize dictionary where the key is the punctuation and the value is the token\n",
    "    \"\"\"\n",
    "    tokens = {'.':'||Period||',\n",
    "              ',':'||Comma||',\n",
    "              '\"':'||Quotation_Mark||',\n",
    "              ';':'||Semicolon||',\n",
    "              '!':'||Exclamation_Mark||',\n",
    "              '؟':'||Question_Mark||',\n",
    "              '(':'||Left_Parantheses||',\n",
    "              ')':'||Right_Parantheses||',\n",
    "              '--':'||Dash||',\n",
    "              '\\n':'||Return||'\n",
    "             }\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "helper.preprocess_and_save_data(data_dir, token_lookup, create_lookup_tables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Check Point\n",
    "This is your first checkpoint. If you ever decide to come back to this notebook or have to restart the notebook, you can start from here. The preprocessed data has been saved to disk."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.0.1\n",
      "Default GPU Device: /gpu:0\n"
     ]
    }
   ],
   "source": [
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer'\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))\n",
    "\n",
    "# Check for a GPU\n",
    "if not tf.test.gpu_device_name():\n",
    "    warnings.warn('No GPU found. Please use a GPU to train your neural network.')\n",
    "else:\n",
    "    print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_inputs():\n",
    "    \"\"\"\n",
    "    Create TF Placeholders for input, targets, and learning rate.\n",
    "    :return: Tuple (input, targets, learning rate)\n",
    "    \"\"\"\n",
    "    inputs         = tf.placeholder(tf.int32,[None, None], name='input')\n",
    "    targets        = tf.placeholder(tf.int32, [None, None], name='target')\n",
    "    learning_rate  = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    return (inputs, targets, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_init_cell(batch_size, rnn_size):\n",
    "    \"\"\"\n",
    "    Create an RNN Cell and initialize it.\n",
    "    :param batch_size: Size of batches\n",
    "    :param rnn_size: Size of RNNs\n",
    "    :return: Tuple (cell, initialize state)\n",
    "    \"\"\"\n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n",
    "    Cell = tf.contrib.rnn.MultiRNNCell([lstm] * 2)\n",
    "    InitialState = Cell.zero_state(batch_size, tf.float32)\n",
    "    InitialState = tf.identity(InitialState, name='initial_state')\n",
    "    \n",
    "    return (Cell, InitialState)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_embed(input_data, vocab_size, embed_dim):\n",
    "    \"\"\"\n",
    "    Create embedding for <input_data>.\n",
    "    :param input_data: TF placeholder for text input.\n",
    "    :param vocab_size: Number of words in vocabulary.\n",
    "    :param embed_dim: Number of embedding dimensions\n",
    "    :return: Embedded input.\n",
    "    \"\"\"\n",
    "    embedding = tf.Variable(tf.random_uniform((vocab_size, embed_dim), -1, 1), dtype=tf.float32)\n",
    "    embed = tf.nn.embedding_lookup(embedding, input_data)\n",
    "    \n",
    "    return embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def build_rnn(cell, inputs):\n",
    "    \"\"\"\n",
    "    Create a RNN using a RNN Cell\n",
    "    :param cell: RNN Cell\n",
    "    :param inputs: Input text data\n",
    "    :return: Tuple (Outputs, Final State)\n",
    "    \"\"\"\n",
    "\n",
    "    Outputs, FinalState = tf.nn.dynamic_rnn(cell, inputs, dtype=tf.float32) \n",
    "    FinalState =  tf.identity(FinalState, name='final_state')\n",
    "    return (Outputs, FinalState)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def build_nn(cell, rnn_size, input_data, vocab_size):\n",
    "    \"\"\"\n",
    "    Build part of the neural network\n",
    "    :param cell: RNN cell\n",
    "    :param rnn_size: Size of rnns\n",
    "    :param input_data: Input data\n",
    "    :param vocab_size: Vocabulary size\n",
    "    :return: Tuple (Logits, FinalState)\n",
    "    \"\"\"\n",
    "    \n",
    "    embedded = get_embed(input_data=input_data, vocab_size=vocab_size, embed_dim=200)\n",
    "    outputs, FinalState = build_rnn(cell=cell, inputs=embedded)\n",
    "    batch_size, embed_size = input_data.get_shape()\n",
    "    logits = tf.contrib.layers.fully_connected(outputs, vocab_size, activation_fn=None)\n",
    "\n",
    "    return (logits, FinalState)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_batches(int_text, batch_size, seq_length):\n",
    "    \"\"\"\n",
    "    Return batches of input and target\n",
    "    :param int_text: Text with the words replaced by their ids\n",
    "    :param batch_size: The size of batch\n",
    "    :param seq_length: The length of sequence\n",
    "    :return: Batches as a Numpy array\n",
    "    \"\"\"\n",
    "    n_batches = len(int_text)//(batch_size*seq_length)\n",
    "\n",
    "    valid_text = int_text[:n_batches*batch_size*seq_length+1]\n",
    "    \n",
    "    result = np.ndarray((n_batches,2,batch_size,seq_length), dtype=int)\n",
    "    step = n_batches*seq_length    \n",
    "    \n",
    "    #print(valid_text)\n",
    "    \n",
    "    for batch in range(n_batches):\n",
    "        batch_walk = batch*seq_length\n",
    "        x = []\n",
    "        y = []\n",
    "        for binn in range(batch_size):\n",
    "            idx = batch_walk + binn * step    # start from this index\n",
    "            result[batch][0][binn] = valid_text[idx   : idx    +seq_length]\n",
    "            result[batch][1][binn] = valid_text[idx+1 : idx+1  +seq_length]   \n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 10\n",
    "batch_size = 128\n",
    "rnn_size = 256\n",
    "seq_length = 50\n",
    "learning_rate = 0.01\n",
    "show_every_n_batches = 10\n",
    "save_dir = './save'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib import seq2seq\n",
    "\n",
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    vocab_size = len(int_to_vocab)\n",
    "    input_text, targets, lr = get_inputs()\n",
    "    input_data_shape = tf.shape(input_text)\n",
    "    cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)\n",
    "    logits, final_state = build_nn(cell, rnn_size, input_text, vocab_size)\n",
    "\n",
    "    # Probabilities for generating words\n",
    "    probs = tf.nn.softmax(logits, name='probs')\n",
    "\n",
    "    # Loss function\n",
    "    cost = seq2seq.sequence_loss(\n",
    "        logits,\n",
    "        targets,\n",
    "        tf.ones([input_data_shape[0], input_data_shape[1]]))\n",
    "\n",
    "    # Optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(lr)\n",
    "\n",
    "    # Gradient Clipping\n",
    "    gradients = optimizer.compute_gradients(cost)\n",
    "    capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients]\n",
    "    train_op = optimizer.apply_gradients(capped_gradients)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Train\n",
    "Train the neural network on the preprocessed data.  If you have a hard time getting a good loss, check the [forms](https://discussions.udacity.com/) to see if anyone is having the same problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   0 Batch    0/305   train_loss = 11.608\n",
      "Epoch   0 Batch   10/305   train_loss = 8.912\n",
      "Epoch   0 Batch   20/305   train_loss = 8.669\n",
      "Epoch   0 Batch   30/305   train_loss = 8.601\n",
      "Epoch   0 Batch   40/305   train_loss = 8.382\n",
      "Epoch   0 Batch   50/305   train_loss = 8.545\n",
      "Epoch   0 Batch   60/305   train_loss = 8.364\n",
      "Epoch   0 Batch   70/305   train_loss = 8.196\n",
      "Epoch   0 Batch   80/305   train_loss = 8.013\n",
      "Epoch   0 Batch   90/305   train_loss = 8.223\n",
      "Epoch   0 Batch  100/305   train_loss = 8.213\n",
      "Epoch   0 Batch  110/305   train_loss = 8.194\n",
      "Epoch   0 Batch  120/305   train_loss = 8.036\n",
      "Epoch   0 Batch  130/305   train_loss = 8.336\n",
      "Epoch   0 Batch  140/305   train_loss = 8.087\n",
      "Epoch   0 Batch  150/305   train_loss = 8.249\n",
      "Epoch   0 Batch  160/305   train_loss = 8.079\n",
      "Epoch   0 Batch  170/305   train_loss = 8.120\n",
      "Epoch   0 Batch  180/305   train_loss = 8.019\n",
      "Epoch   0 Batch  190/305   train_loss = 7.910\n",
      "Epoch   0 Batch  200/305   train_loss = 8.215\n",
      "Epoch   0 Batch  210/305   train_loss = 8.133\n",
      "Epoch   0 Batch  220/305   train_loss = 8.071\n",
      "Epoch   0 Batch  230/305   train_loss = 8.113\n",
      "Epoch   0 Batch  240/305   train_loss = 8.118\n",
      "Epoch   0 Batch  250/305   train_loss = 7.834\n",
      "Epoch   0 Batch  260/305   train_loss = 7.922\n",
      "Epoch   0 Batch  270/305   train_loss = 7.947\n",
      "Epoch   0 Batch  280/305   train_loss = 7.999\n",
      "Epoch   0 Batch  290/305   train_loss = 8.036\n",
      "Epoch   0 Batch  300/305   train_loss = 7.879\n",
      "Epoch   1 Batch    5/305   train_loss = 7.854\n",
      "Epoch   1 Batch   15/305   train_loss = 7.912\n",
      "Epoch   1 Batch   25/305   train_loss = 7.786\n",
      "Epoch   1 Batch   35/305   train_loss = 7.985\n",
      "Epoch   1 Batch   45/305   train_loss = 7.846\n",
      "Epoch   1 Batch   55/305   train_loss = 7.770\n",
      "Epoch   1 Batch   65/305   train_loss = 7.852\n",
      "Epoch   1 Batch   75/305   train_loss = 7.761\n",
      "Epoch   1 Batch   85/305   train_loss = 7.420\n",
      "Epoch   1 Batch   95/305   train_loss = 7.431\n",
      "Epoch   1 Batch  105/305   train_loss = 7.486\n",
      "Epoch   1 Batch  115/305   train_loss = 7.301\n",
      "Epoch   1 Batch  125/305   train_loss = 7.345\n",
      "Epoch   1 Batch  135/305   train_loss = 7.272\n",
      "Epoch   1 Batch  145/305   train_loss = 7.303\n",
      "Epoch   1 Batch  155/305   train_loss = 7.131\n",
      "Epoch   1 Batch  165/305   train_loss = 7.181\n",
      "Epoch   1 Batch  175/305   train_loss = 7.289\n",
      "Epoch   1 Batch  185/305   train_loss = 7.160\n",
      "Epoch   1 Batch  195/305   train_loss = 7.114\n",
      "Epoch   1 Batch  205/305   train_loss = 7.134\n",
      "Epoch   1 Batch  215/305   train_loss = 7.233\n",
      "Epoch   1 Batch  225/305   train_loss = 7.117\n",
      "Epoch   1 Batch  235/305   train_loss = 7.194\n",
      "Epoch   1 Batch  245/305   train_loss = 7.020\n",
      "Epoch   1 Batch  255/305   train_loss = 6.988\n",
      "Epoch   1 Batch  265/305   train_loss = 7.054\n",
      "Epoch   1 Batch  275/305   train_loss = 6.964\n",
      "Epoch   1 Batch  285/305   train_loss = 7.024\n",
      "Epoch   1 Batch  295/305   train_loss = 7.059\n",
      "Epoch   2 Batch    0/305   train_loss = 6.964\n",
      "Epoch   2 Batch   10/305   train_loss = 7.152\n",
      "Epoch   2 Batch   20/305   train_loss = 7.094\n",
      "Epoch   2 Batch   30/305   train_loss = 7.103\n",
      "Epoch   2 Batch   40/305   train_loss = 7.029\n",
      "Epoch   2 Batch   50/305   train_loss = 7.298\n",
      "Epoch   2 Batch   60/305   train_loss = 7.131\n",
      "Epoch   2 Batch   70/305   train_loss = 7.092\n",
      "Epoch   2 Batch   80/305   train_loss = 6.889\n",
      "Epoch   2 Batch   90/305   train_loss = 7.070\n",
      "Epoch   2 Batch  100/305   train_loss = 7.049\n",
      "Epoch   2 Batch  110/305   train_loss = 7.119\n",
      "Epoch   2 Batch  120/305   train_loss = 6.995\n",
      "Epoch   2 Batch  130/305   train_loss = 7.163\n",
      "Epoch   2 Batch  140/305   train_loss = 7.128\n",
      "Epoch   2 Batch  150/305   train_loss = 7.169\n",
      "Epoch   2 Batch  160/305   train_loss = 7.100\n",
      "Epoch   2 Batch  170/305   train_loss = 7.208\n",
      "Epoch   2 Batch  180/305   train_loss = 7.091\n",
      "Epoch   2 Batch  190/305   train_loss = 6.957\n",
      "Epoch   2 Batch  200/305   train_loss = 7.191\n",
      "Epoch   2 Batch  210/305   train_loss = 7.171\n",
      "Epoch   2 Batch  220/305   train_loss = 7.127\n",
      "Epoch   2 Batch  230/305   train_loss = 7.064\n",
      "Epoch   2 Batch  240/305   train_loss = 7.083\n",
      "Epoch   2 Batch  250/305   train_loss = 6.882\n",
      "Epoch   2 Batch  260/305   train_loss = 6.947\n",
      "Epoch   2 Batch  270/305   train_loss = 6.931\n",
      "Epoch   2 Batch  280/305   train_loss = 6.958\n",
      "Epoch   2 Batch  290/305   train_loss = 6.989\n",
      "Epoch   2 Batch  300/305   train_loss = 6.964\n",
      "Epoch   3 Batch    5/305   train_loss = 7.119\n",
      "Epoch   3 Batch   15/305   train_loss = 7.091\n",
      "Epoch   3 Batch   25/305   train_loss = 7.030\n",
      "Epoch   3 Batch   35/305   train_loss = 7.120\n",
      "Epoch   3 Batch   45/305   train_loss = 7.079\n",
      "Epoch   3 Batch   55/305   train_loss = 7.088\n",
      "Epoch   3 Batch   65/305   train_loss = 7.070\n",
      "Epoch   3 Batch   75/305   train_loss = 7.139\n",
      "Epoch   3 Batch   85/305   train_loss = 7.010\n",
      "Epoch   3 Batch   95/305   train_loss = 7.078\n",
      "Epoch   3 Batch  105/305   train_loss = 7.204\n",
      "Epoch   3 Batch  115/305   train_loss = 7.028\n",
      "Epoch   3 Batch  125/305   train_loss = 6.988\n",
      "Epoch   3 Batch  135/305   train_loss = 6.902\n",
      "Epoch   3 Batch  145/305   train_loss = 6.964\n",
      "Epoch   3 Batch  155/305   train_loss = 6.885\n",
      "Epoch   3 Batch  165/305   train_loss = 6.957\n",
      "Epoch   3 Batch  175/305   train_loss = 7.080\n",
      "Epoch   3 Batch  185/305   train_loss = 7.071\n",
      "Epoch   3 Batch  195/305   train_loss = 6.976\n",
      "Epoch   3 Batch  205/305   train_loss = 6.943\n",
      "Epoch   3 Batch  215/305   train_loss = 7.150\n",
      "Epoch   3 Batch  225/305   train_loss = 6.957\n",
      "Epoch   3 Batch  235/305   train_loss = 7.024\n",
      "Epoch   3 Batch  245/305   train_loss = 6.869\n",
      "Epoch   3 Batch  255/305   train_loss = 6.890\n",
      "Epoch   3 Batch  265/305   train_loss = 6.907\n",
      "Epoch   3 Batch  275/305   train_loss = 6.704\n",
      "Epoch   3 Batch  285/305   train_loss = 6.843\n",
      "Epoch   3 Batch  295/305   train_loss = 6.894\n",
      "Epoch   4 Batch    0/305   train_loss = 6.802\n",
      "Epoch   4 Batch   10/305   train_loss = 6.975\n",
      "Epoch   4 Batch   20/305   train_loss = 6.834\n",
      "Epoch   4 Batch   30/305   train_loss = 6.942\n",
      "Epoch   4 Batch   40/305   train_loss = 6.930\n",
      "Epoch   4 Batch   50/305   train_loss = 7.141\n",
      "Epoch   4 Batch   60/305   train_loss = 7.078\n",
      "Epoch   4 Batch   70/305   train_loss = 6.964\n",
      "Epoch   4 Batch   80/305   train_loss = 6.814\n",
      "Epoch   4 Batch   90/305   train_loss = 6.977\n",
      "Epoch   4 Batch  100/305   train_loss = 7.031\n",
      "Epoch   4 Batch  110/305   train_loss = 7.012\n",
      "Epoch   4 Batch  120/305   train_loss = 6.886\n",
      "Epoch   4 Batch  130/305   train_loss = 7.028\n",
      "Epoch   4 Batch  140/305   train_loss = 6.947\n",
      "Epoch   4 Batch  150/305   train_loss = 7.087\n",
      "Epoch   4 Batch  160/305   train_loss = 7.019\n",
      "Epoch   4 Batch  170/305   train_loss = 7.034\n",
      "Epoch   4 Batch  180/305   train_loss = 6.936\n",
      "Epoch   4 Batch  190/305   train_loss = 6.785\n",
      "Epoch   4 Batch  200/305   train_loss = 6.885\n",
      "Epoch   4 Batch  210/305   train_loss = 7.008\n",
      "Epoch   4 Batch  220/305   train_loss = 6.962\n",
      "Epoch   4 Batch  230/305   train_loss = 6.896\n",
      "Epoch   4 Batch  240/305   train_loss = 6.940\n",
      "Epoch   4 Batch  250/305   train_loss = 6.824\n",
      "Epoch   4 Batch  260/305   train_loss = 6.760\n",
      "Epoch   4 Batch  270/305   train_loss = 6.810\n",
      "Epoch   4 Batch  280/305   train_loss = 6.838\n",
      "Epoch   4 Batch  290/305   train_loss = 6.776\n",
      "Epoch   4 Batch  300/305   train_loss = 6.824\n",
      "Epoch   5 Batch    5/305   train_loss = 6.692\n",
      "Epoch   5 Batch   15/305   train_loss = 6.872\n",
      "Epoch   5 Batch   25/305   train_loss = 6.964\n",
      "Epoch   5 Batch   35/305   train_loss = 7.055\n",
      "Epoch   5 Batch   45/305   train_loss = 6.993\n",
      "Epoch   5 Batch   55/305   train_loss = 6.962\n",
      "Epoch   5 Batch   65/305   train_loss = 6.940\n",
      "Epoch   5 Batch   75/305   train_loss = 6.919\n",
      "Epoch   5 Batch   85/305   train_loss = 6.701\n",
      "Epoch   5 Batch   95/305   train_loss = 6.703\n",
      "Epoch   5 Batch  105/305   train_loss = 6.830\n",
      "Epoch   5 Batch  115/305   train_loss = 6.723\n",
      "Epoch   5 Batch  125/305   train_loss = 6.774\n",
      "Epoch   5 Batch  135/305   train_loss = 6.692\n",
      "Epoch   5 Batch  145/305   train_loss = 6.734\n",
      "Epoch   5 Batch  155/305   train_loss = 6.581\n",
      "Epoch   5 Batch  165/305   train_loss = 6.635\n",
      "Epoch   5 Batch  175/305   train_loss = 6.736\n",
      "Epoch   5 Batch  185/305   train_loss = 6.580\n",
      "Epoch   5 Batch  195/305   train_loss = 6.619\n",
      "Epoch   5 Batch  205/305   train_loss = 6.611\n",
      "Epoch   5 Batch  215/305   train_loss = 6.738\n",
      "Epoch   5 Batch  225/305   train_loss = 6.582\n",
      "Epoch   5 Batch  235/305   train_loss = 6.787\n",
      "Epoch   5 Batch  245/305   train_loss = 6.665\n",
      "Epoch   5 Batch  255/305   train_loss = 6.603\n",
      "Epoch   5 Batch  265/305   train_loss = 6.622\n",
      "Epoch   5 Batch  275/305   train_loss = 6.493\n",
      "Epoch   5 Batch  285/305   train_loss = 6.489\n",
      "Epoch   5 Batch  295/305   train_loss = 6.515\n",
      "Epoch   6 Batch    0/305   train_loss = 6.524\n",
      "Epoch   6 Batch   10/305   train_loss = 6.838\n",
      "Epoch   6 Batch   20/305   train_loss = 6.533\n",
      "Epoch   6 Batch   30/305   train_loss = 6.551\n",
      "Epoch   6 Batch   40/305   train_loss = 6.570\n",
      "Epoch   6 Batch   50/305   train_loss = 6.693\n",
      "Epoch   6 Batch   60/305   train_loss = 6.631\n",
      "Epoch   6 Batch   70/305   train_loss = 6.602\n",
      "Epoch   6 Batch   80/305   train_loss = 6.365\n",
      "Epoch   6 Batch   90/305   train_loss = 6.609\n",
      "Epoch   6 Batch  100/305   train_loss = 6.603\n",
      "Epoch   6 Batch  110/305   train_loss = 6.672\n",
      "Epoch   6 Batch  120/305   train_loss = 6.485\n",
      "Epoch   6 Batch  130/305   train_loss = 6.563\n",
      "Epoch   6 Batch  140/305   train_loss = 6.623\n",
      "Epoch   6 Batch  150/305   train_loss = 6.592\n",
      "Epoch   6 Batch  160/305   train_loss = 6.461\n",
      "Epoch   6 Batch  170/305   train_loss = 6.658\n",
      "Epoch   6 Batch  180/305   train_loss = 6.442\n",
      "Epoch   6 Batch  190/305   train_loss = 6.405\n",
      "Epoch   6 Batch  200/305   train_loss = 6.771\n",
      "Epoch   6 Batch  210/305   train_loss = 6.565\n",
      "Epoch   6 Batch  220/305   train_loss = 6.600\n",
      "Epoch   6 Batch  230/305   train_loss = 6.507\n",
      "Epoch   6 Batch  240/305   train_loss = 6.586\n",
      "Epoch   6 Batch  250/305   train_loss = 6.514\n",
      "Epoch   6 Batch  260/305   train_loss = 6.543\n",
      "Epoch   6 Batch  270/305   train_loss = 6.525\n",
      "Epoch   6 Batch  280/305   train_loss = 6.477\n",
      "Epoch   6 Batch  290/305   train_loss = 6.441\n",
      "Epoch   6 Batch  300/305   train_loss = 6.468\n",
      "Epoch   7 Batch    5/305   train_loss = 6.596\n",
      "Epoch   7 Batch   15/305   train_loss = 6.770\n",
      "Epoch   7 Batch   25/305   train_loss = 6.572\n",
      "Epoch   7 Batch   35/305   train_loss = 6.533\n",
      "Epoch   7 Batch   45/305   train_loss = 6.520\n",
      "Epoch   7 Batch   55/305   train_loss = 6.574\n",
      "Epoch   7 Batch   65/305   train_loss = 6.624\n",
      "Epoch   7 Batch   75/305   train_loss = 6.604\n",
      "Epoch   7 Batch   85/305   train_loss = 6.353\n",
      "Epoch   7 Batch   95/305   train_loss = 6.351\n",
      "Epoch   7 Batch  105/305   train_loss = 6.455\n",
      "Epoch   7 Batch  115/305   train_loss = 6.330\n",
      "Epoch   7 Batch  125/305   train_loss = 6.398\n",
      "Epoch   7 Batch  135/305   train_loss = 6.335\n",
      "Epoch   7 Batch  145/305   train_loss = 6.430\n",
      "Epoch   7 Batch  155/305   train_loss = 6.236\n",
      "Epoch   7 Batch  165/305   train_loss = 6.392\n",
      "Epoch   7 Batch  175/305   train_loss = 6.424\n",
      "Epoch   7 Batch  185/305   train_loss = 6.320\n",
      "Epoch   7 Batch  195/305   train_loss = 6.311\n",
      "Epoch   7 Batch  205/305   train_loss = 6.357\n",
      "Epoch   7 Batch  215/305   train_loss = 6.638\n",
      "Epoch   7 Batch  225/305   train_loss = 6.557\n",
      "Epoch   7 Batch  235/305   train_loss = 6.554\n",
      "Epoch   7 Batch  245/305   train_loss = 6.354\n",
      "Epoch   7 Batch  255/305   train_loss = 6.398\n",
      "Epoch   7 Batch  265/305   train_loss = 6.406\n",
      "Epoch   7 Batch  275/305   train_loss = 6.337\n",
      "Epoch   7 Batch  285/305   train_loss = 6.366\n",
      "Epoch   7 Batch  295/305   train_loss = 6.373\n",
      "Epoch   8 Batch    0/305   train_loss = 6.381\n",
      "Epoch   8 Batch   10/305   train_loss = 6.471\n",
      "Epoch   8 Batch   20/305   train_loss = 6.474\n",
      "Epoch   8 Batch   30/305   train_loss = 6.369\n",
      "Epoch   8 Batch   40/305   train_loss = 6.416\n",
      "Epoch   8 Batch   50/305   train_loss = 6.503\n",
      "Epoch   8 Batch   60/305   train_loss = 6.432\n",
      "Epoch   8 Batch   70/305   train_loss = 6.325\n",
      "Epoch   8 Batch   80/305   train_loss = 6.194\n",
      "Epoch   8 Batch   90/305   train_loss = 6.356\n",
      "Epoch   8 Batch  100/305   train_loss = 6.358\n",
      "Epoch   8 Batch  110/305   train_loss = 6.417\n",
      "Epoch   8 Batch  120/305   train_loss = 6.268\n",
      "Epoch   8 Batch  130/305   train_loss = 6.328\n",
      "Epoch   8 Batch  140/305   train_loss = 6.384\n",
      "Epoch   8 Batch  150/305   train_loss = 6.531\n",
      "Epoch   8 Batch  160/305   train_loss = 6.373\n",
      "Epoch   8 Batch  170/305   train_loss = 6.571\n",
      "Epoch   8 Batch  180/305   train_loss = 6.297\n",
      "Epoch   8 Batch  190/305   train_loss = 6.210\n",
      "Epoch   8 Batch  200/305   train_loss = 6.375\n",
      "Epoch   8 Batch  210/305   train_loss = 6.339\n",
      "Epoch   8 Batch  220/305   train_loss = 6.332\n",
      "Epoch   8 Batch  230/305   train_loss = 6.355\n",
      "Epoch   8 Batch  240/305   train_loss = 6.380\n",
      "Epoch   8 Batch  250/305   train_loss = 6.259\n",
      "Epoch   8 Batch  260/305   train_loss = 6.273\n",
      "Epoch   8 Batch  270/305   train_loss = 6.246\n",
      "Epoch   8 Batch  280/305   train_loss = 6.275\n",
      "Epoch   8 Batch  290/305   train_loss = 6.265\n",
      "Epoch   8 Batch  300/305   train_loss = 6.215\n",
      "Epoch   9 Batch    5/305   train_loss = 6.334\n",
      "Epoch   9 Batch   15/305   train_loss = 6.307\n",
      "Epoch   9 Batch   25/305   train_loss = 6.321\n",
      "Epoch   9 Batch   35/305   train_loss = 6.352\n",
      "Epoch   9 Batch   45/305   train_loss = 6.245\n",
      "Epoch   9 Batch   55/305   train_loss = 6.345\n",
      "Epoch   9 Batch   65/305   train_loss = 6.434\n",
      "Epoch   9 Batch   75/305   train_loss = 6.279\n",
      "Epoch   9 Batch   85/305   train_loss = 6.151\n",
      "Epoch   9 Batch   95/305   train_loss = 6.201\n",
      "Epoch   9 Batch  105/305   train_loss = 6.297\n",
      "Epoch   9 Batch  115/305   train_loss = 6.241\n",
      "Epoch   9 Batch  125/305   train_loss = 6.278\n",
      "Epoch   9 Batch  135/305   train_loss = 6.195\n",
      "Epoch   9 Batch  145/305   train_loss = 6.351\n",
      "Epoch   9 Batch  155/305   train_loss = 6.226\n",
      "Epoch   9 Batch  165/305   train_loss = 6.293\n",
      "Epoch   9 Batch  175/305   train_loss = 6.405\n",
      "Epoch   9 Batch  185/305   train_loss = 6.371\n",
      "Epoch   9 Batch  195/305   train_loss = 6.252\n",
      "Epoch   9 Batch  205/305   train_loss = 6.248\n",
      "Epoch   9 Batch  215/305   train_loss = 6.284\n",
      "Epoch   9 Batch  225/305   train_loss = 6.167\n",
      "Epoch   9 Batch  235/305   train_loss = 6.285\n",
      "Epoch   9 Batch  245/305   train_loss = 6.120\n",
      "Epoch   9 Batch  255/305   train_loss = 6.190\n",
      "Epoch   9 Batch  265/305   train_loss = 6.172\n",
      "Epoch   9 Batch  275/305   train_loss = 6.086\n",
      "Epoch   9 Batch  285/305   train_loss = 6.136\n",
      "Epoch   9 Batch  295/305   train_loss = 6.127\n",
      "Model Trained and Saved\n"
     ]
    }
   ],
   "source": [
    "batches = get_batches(int_text, batch_size, seq_length)\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    for epoch_i in range(num_epochs):\n",
    "        state = sess.run(initial_state, {input_text: batches[0][0]})\n",
    "\n",
    "        for batch_i, (x, y) in enumerate(batches):\n",
    "            feed = {\n",
    "                input_text: x,\n",
    "                targets: y,\n",
    "                initial_state: state,\n",
    "                lr: learning_rate}\n",
    "            train_loss, state, _ = sess.run([cost, final_state, train_op], feed)\n",
    "\n",
    "            # Show every <show_every_n_batches> batches\n",
    "            if (epoch_i * len(batches) + batch_i) % show_every_n_batches == 0:\n",
    "                print('Epoch {:>3} Batch {:>4}/{}   train_loss = {:.3f}'.format(\n",
    "                    epoch_i,\n",
    "                    batch_i,\n",
    "                    len(batches),\n",
    "                    train_loss))\n",
    "\n",
    "    # Save Model\n",
    "    saver = tf.train.Saver()\n",
    "    saver.save(sess, save_dir)\n",
    "    print('Model Trained and Saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "helper.save_params((seq_length, save_dir))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()\n",
    "seq_length, load_dir = helper.load_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('لانظرتك', 0),\n",
       " ('يسعوا', 1),\n",
       " ('بسمايا', 2),\n",
       " ('قضاها', 3),\n",
       " ('الاقزام', 4),\n",
       " ('رياض', 5),\n",
       " ('قرانا', 6),\n",
       " ('فطان', 7),\n",
       " ('اعشقيني', 8),\n",
       " ('لاقدامي', 9)]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(vocab_to_int.items())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_tensors(loaded_graph):\n",
    "    \"\"\"\n",
    "    Get input, initial state, final state, and probabilities tensor from <loaded_graph>\n",
    "    :param loaded_graph: TensorFlow graph loaded from file\n",
    "    :return: Tuple (InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)\n",
    "    \"\"\"\n",
    "    InputTensor = loaded_graph.get_tensor_by_name(name='input:0')\n",
    "    InitialStateTensor = loaded_graph.get_tensor_by_name(name='initial_state:0')\n",
    "    FinalStateTensor = loaded_graph.get_tensor_by_name(name='final_state:0')\n",
    "    ProbsTensor = loaded_graph.get_tensor_by_name(name='probs:0')\n",
    "    return (InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def pick_word(preds, int_to_vocab, top_n=10):\n",
    "    \"\"\"\n",
    "    Pick the next word in the generated text\n",
    "    :param probabilities: Probabilites of the next word\n",
    "    :param int_to_vocab: Dictionary of word ids as the keys and words as the values\n",
    "    :return: String of the predicted word\n",
    "    \"\"\"\n",
    "    p = np.squeeze(preds)\n",
    "    p[np.argsort(p)[:-top_n]] = 0\n",
    "    p = p / np.sum(p)\n",
    "    c = np.random.choice(list(int_to_vocab.values()), 1, p=p)[0]\n",
    "    \n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "عمري و ياك\n",
      " انا و انا ما لا و انت انا\n",
      " مش انا مش قادر مش في بالي\n",
      " انا الي في قلبي و انا ما كان\n",
      " يا ما كان ايه انا ما تسلمولي\n",
      " انا انت حبيبي\n",
      " مش مكن جرالي و انا انت انا حبيبي انت\n",
      " انت الي انا ما انا ما كان\n"
     ]
    }
   ],
   "source": [
    "gen_length = 60\n",
    "prime_word = u'عمري'\n",
    "\n",
    "loaded_graph = tf.Graph()\n",
    "with tf.Session(graph=loaded_graph) as sess:\n",
    "    # Load saved model\n",
    "    loader = tf.train.import_meta_graph(load_dir + '.meta')\n",
    "    loader.restore(sess, load_dir)\n",
    "\n",
    "    # Get Tensors from loaded model\n",
    "    input_text, initial_state, final_state, probs = get_tensors(loaded_graph)\n",
    "\n",
    "    # Sentences generation setup\n",
    "    gen_sentences = [prime_word]\n",
    "\n",
    "    prev_state = sess.run(initial_state, {input_text: np.array([[1]])})\n",
    "\n",
    "    # Generate sentences\n",
    "    for n in range(gen_length):\n",
    "        # Dynamic Input\n",
    "        dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]\n",
    "        dyn_seq_length = len(dyn_input[0])\n",
    "        \n",
    "        # Get Prediction\n",
    "        probabilities, prev_state = sess.run(\n",
    "            [probs, final_state],\n",
    "            {input_text: dyn_input, initial_state: prev_state})\n",
    "        \n",
    "        pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)\n",
    "        if pred_word == gen_sentences[len(gen_sentences)-1]:\n",
    "            continue\n",
    "        gen_sentences.append(pred_word)\n",
    "    \n",
    "    # Remove tokens\n",
    "    lyrics = ' '.join(gen_sentences)\n",
    "    for key, token in token_dict.items():\n",
    "        lyrics = lyrics.replace(' ' + token, key)\n",
    "    lyrics = re.sub('(\\n){2,}', '\\n', lyrics)\n",
    "        \n",
    "    print(lyrics.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}