swahilli_speech_recognition.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Swahili_speech_to_text (2).ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zelalemgetahun9374/Swahili-Speech-To-Text/blob/main/swahilli_speech_recognition.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8DB4X_g6ovq6"
      },
      "source": [
        "import os\n",
        "import pickle\n",
        "import librosa\n",
        "import pandas as pd\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore', category=DeprecationWarning)\n",
        "warnings.filterwarnings('ignore', category=FutureWarning)\n",
        "\n",
        "from IPython.display import Audio\n",
        "\n",
        "import librosa.display\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow.keras.models import Model, Sequential\n",
        "from tensorflow.keras.layers import * \n",
        "from tensorflow.keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, TensorBoard\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.optimizers import SGD, Adam, RMSprop\n",
        "from tensorflow.keras import backend as K"
      ],
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t20ZPt-MU6nQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "49174155-4d90-42c7-da99-234e7ba6cfea"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zVfvp0tkJUGw"
      },
      "source": [
        "os.chdir(\"/content/drive/MyDrive/structured_data/train/wav\")"
      ],
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZOWtGAdRiumj"
      },
      "source": [
        "## Inputs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZP9qdhcRiyUG",
        "outputId": "4334d91a-9af0-4f28-cc75-e44843c1245f"
      },
      "source": [
        "rows = []\n",
        "parent_dir = \"./\"\n",
        "files = os.listdir(parent_dir)\n",
        "for f in files[:1100]:\n",
        "    audio, fs = librosa.load(f\"{f}\")\n",
        "    filename = f.split('.')[0]\n",
        "    row = {'filename': filename, 'audio': audio}\n",
        "    rows.append(row)\n",
        "rows[:5]"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'audio': array([-0.01944764, -0.02380558, -0.01945851, ..., -0.02546581,\n",
              "         -0.01958567, -0.00813534], dtype=float32),\n",
              "  'filename': 'SWH-15-20101109_emission_swahili_15h00_-_16h00_tu_20101109_part198'},\n",
              " {'audio': array([ 6.4895018e-03, -9.7726457e-05, -6.6772741e-03, ...,\n",
              "          4.5560386e-02,  3.6010724e-02,  0.0000000e+00], dtype=float32),\n",
              "  'filename': 'SWH-15-20101109_emission_swahili_15h00_-_16h00_tu_20101109_part411'},\n",
              " {'audio': array([-0.01290035, -0.01468785, -0.01265284, ..., -0.16431093,\n",
              "         -0.11925378, -0.05920244], dtype=float32),\n",
              "  'filename': 'SWH-15-20101109_emission_swahili_15h00_-_16h00_tu_20101109_part23'},\n",
              " {'audio': array([-0.0225578 , -0.02763871, -0.01984122, ..., -0.08003497,\n",
              "         -0.08758974,  0.        ], dtype=float32),\n",
              "  'filename': 'SWH-15-20101109_emission_swahili_15h00_-_16h00_tu_20101109_part233'},\n",
              " {'audio': array([0.00551369, 0.00701467, 0.00613299, ..., 0.07596262, 0.06502344,\n",
              "         0.        ], dtype=float32),\n",
              "  'filename': 'SWH-15-20101109_emission_swahili_15h00_-_16h00_tu_20101109_part164'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TeGMXkLWJ4np",
        "outputId": "a9e73d27-9ae7-4087-d2a2-9412b4ef539e"
      },
      "source": [
        "len(files)"
      ],
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10180"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c8NByqTOUhRd",
        "outputId": "de238818-a14b-4eeb-ad0f-c78a6dc42b19"
      },
      "source": [
        "sample_audios = []\n",
        "for row in rows:\n",
        "    audio = row['audio']\n",
        "    sample_audios.append(audio)\n",
        "sample_audios[:5]"
      ],
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([-0.01944764, -0.02380558, -0.01945851, ..., -0.02546581,\n",
              "        -0.01958567, -0.00813534], dtype=float32),\n",
              " array([ 6.4895018e-03, -9.7726457e-05, -6.6772741e-03, ...,\n",
              "         4.5560386e-02,  3.6010724e-02,  0.0000000e+00], dtype=float32),\n",
              " array([-0.01290035, -0.01468785, -0.01265284, ..., -0.16431093,\n",
              "        -0.11925378, -0.05920244], dtype=float32),\n",
              " array([-0.0225578 , -0.02763871, -0.01984122, ..., -0.08003497,\n",
              "        -0.08758974,  0.        ], dtype=float32),\n",
              " array([0.00551369, 0.00701467, 0.00613299, ..., 0.07596262, 0.06502344,\n",
              "        0.        ], dtype=float32)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8H6faM08QvZ2"
      },
      "source": [
        "meta_df = pd.read_csv('/content/drive/MyDrive/metadata.csv')"
      ],
      "execution_count": 88,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "bVVnaH5DT6DG",
        "outputId": "e8e2a725-0c0e-4cb3-c296-9b7bf7416711"
      },
      "source": [
        "meta_df.head()"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>filename</th>\n",
              "      <th>transcription</th>\n",
              "      <th>filepath</th>\n",
              "      <th>sample_rate</th>\n",
              "      <th>duration</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SWH-05-20101106_16k-emission_swahili_05h30_-_0...</td>\n",
              "      <td>rais wa tanzania jakaya mrisho kikwete</td>\n",
              "      <td>SWH-05-20101106/SWH-05-20101106_16k-emission_s...</td>\n",
              "      <td>16000</td>\n",
              "      <td>3.14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>SWH-05-20101106_16k-emission_swahili_05h30_-_0...</td>\n",
              "      <td>yanayo andaliwa nami pendo pondo idhaa ya kisw...</td>\n",
              "      <td>SWH-05-20101106/SWH-05-20101106_16k-emission_s...</td>\n",
              "      <td>16000</td>\n",
              "      <td>3.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>SWH-05-20101106_16k-emission_swahili_05h30_-_0...</td>\n",
              "      <td>inayokutangazia moja kwa moja kutoka jijini da...</td>\n",
              "      <td>SWH-05-20101106/SWH-05-20101106_16k-emission_s...</td>\n",
              "      <td>16000</td>\n",
              "      <td>3.65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>SWH-05-20101106_16k-emission_swahili_05h30_-_0...</td>\n",
              "      <td>juma hili bara la afrika limeshuhudia raia wa ...</td>\n",
              "      <td>SWH-05-20101106/SWH-05-20101106_16k-emission_s...</td>\n",
              "      <td>16000</td>\n",
              "      <td>3.90</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>SWH-05-20101106_16k-emission_swahili_05h30_-_0...</td>\n",
              "      <td>wakipiga kura ya maoni ilikufanya mabadiliko ya</td>\n",
              "      <td>SWH-05-20101106/SWH-05-20101106_16k-emission_s...</td>\n",
              "      <td>16000</td>\n",
              "      <td>2.94</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            filename  ... duration\n",
              "0  SWH-05-20101106_16k-emission_swahili_05h30_-_0...  ...     3.14\n",
              "1  SWH-05-20101106_16k-emission_swahili_05h30_-_0...  ...     3.10\n",
              "2  SWH-05-20101106_16k-emission_swahili_05h30_-_0...  ...     3.65\n",
              "3  SWH-05-20101106_16k-emission_swahili_05h30_-_0...  ...     3.90\n",
              "4  SWH-05-20101106_16k-emission_swahili_05h30_-_0...  ...     2.94\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gNHLp5mHSXbT",
        "outputId": "55dd9fac-fd63-4887-c940-51b874f01d61"
      },
      "source": [
        "txts = []\n",
        "for row in rows:\n",
        "    filename = row['filename']\n",
        "    filter = meta_df[meta_df['filename'] == filename]\n",
        "    txt = filter[['transcription']].values\n",
        "    txts.append(txt)\n",
        "\n",
        "txts[:5]"
      ],
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([['janga la ukimwi unajua sana sana tunavyojua lilitokea uganda sehemu za rakai']],\n",
              "       dtype=object),\n",
              " array([['hali ya mambo baada tu ya kupata mada hii kwamba']], dtype=object),\n",
              " array([['mamlaka za afya nchini haiti zinathibitisha kuwa kesi mia moja na ishirini zimeripotiwa']],\n",
              "       dtype=object),\n",
              " array([['watu wafuga lakini pigo']], dtype=object),\n",
              " array([['kwanza kutolewa kwa matokeo ya awali lakini hadi sasa']],\n",
              "       dtype=object)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QtbOxUt-Vvzr"
      },
      "source": [
        "txts = np.array(txts).reshape(-1)"
      ],
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TdRmO_PMWK7M",
        "outputId": "2d701ce6-691d-4960-e93c-6fb7b20da81c"
      },
      "source": [
        "txts[:5]"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['janga la ukimwi unajua sana sana tunavyojua lilitokea uganda sehemu za rakai',\n",
              "       'hali ya mambo baada tu ya kupata mada hii kwamba',\n",
              "       'mamlaka za afya nchini haiti zinathibitisha kuwa kesi mia moja na ishirini zimeripotiwa',\n",
              "       'watu wafuga lakini pigo',\n",
              "       'kwanza kutolewa kwa matokeo ya awali lakini hadi sasa'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bj7dp_XDZFpb"
      },
      "source": [
        "clean_txts = []\n",
        "alphabets = 'a b c d e f g h i j k l m n o p q r s t u v w x y z'.split()\n",
        "for txt in txts:\n",
        "    clean_txt = []\n",
        "    for c in txt:\n",
        "        if c not in alphabets and c != ' ':\n",
        "            continue\n",
        "        clean_txt.append(c)\n",
        "    clean_txt = ''.join(clean_txt)\n",
        "    clean_txts.append(clean_txt)"
      ],
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hssO43laaId0",
        "outputId": "f8ce24c6-40fa-47ca-ad6c-9046d1063b54"
      },
      "source": [
        "clean_txts[:5]"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['janga la ukimwi unajua sana sana tunavyojua lilitokea uganda sehemu za rakai',\n",
              " 'hali ya mambo baada tu ya kupata mada hii kwamba',\n",
              " 'mamlaka za afya nchini haiti zinathibitisha kuwa kesi mia moja na ishirini zimeripotiwa',\n",
              " 'watu wafuga lakini pigo',\n",
              " 'kwanza kutolewa kwa matokeo ya awali lakini hadi sasa']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KY06jQ-lXfod",
        "outputId": "6caf8ea9-a49d-4858-d45c-65bd3c42c739"
      },
      "source": [
        "'' in clean_txts"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "keV4RInJXkY7",
        "outputId": "7dd3d4b3-cb49-4102-98eb-df47d90c5919"
      },
      "source": [
        "df = pd.DataFrame(clean_txts)\n",
        "df.columns = ['texts']\n",
        "df.head()"
      ],
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>texts</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>janga la ukimwi unajua sana sana tunavyojua li...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>hali ya mambo baada tu ya kupata mada hii kwamba</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>mamlaka za afya nchini haiti zinathibitisha ku...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>watu wafuga lakini pigo</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>kwanza kutolewa kwa matokeo ya awali lakini ha...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               texts\n",
              "0  janga la ukimwi unajua sana sana tunavyojua li...\n",
              "1   hali ya mambo baada tu ya kupata mada hii kwamba\n",
              "2  mamlaka za afya nchini haiti zinathibitisha ku...\n",
              "3                            watu wafuga lakini pigo\n",
              "4  kwanza kutolewa kwa matokeo ya awali lakini ha..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LoqxUW6XXoPi",
        "outputId": "b33c9ada-25ab-4f70-a988-06e17013699c"
      },
      "source": [
        "idxs = df[df['texts'] == ''].index\n",
        "idxs"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Int64Index([43, 158, 240], dtype='int64')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wcn64fCTXrJK"
      },
      "source": [
        "del clean_txts[idxs[-1]]\n",
        "del clean_txts[idxs[-2]]\n",
        "del clean_txts[idxs[-3]]"
      ],
      "execution_count": 98,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZL0HfjOtXtxT",
        "outputId": "4d85a6d2-acf3-4372-d379-22831df53752"
      },
      "source": [
        "'' in clean_txts"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LofBDpPvXxLg"
      },
      "source": [
        "del sample_audios[idxs[-1]]\n",
        "del sample_audios[idxs[-2]]\n",
        "del sample_audios[idxs[-3]]"
      ],
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P3Aa-Wdc7Rnq"
      },
      "source": [
        "## Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "30NG8Jgz1H56"
      },
      "source": [
        "def character_dict():\n",
        "    alphabet = 'a b c d e f g h i j k l m n o p q r s t u v w x y z'\n",
        "    supported = alphabet.split()\n",
        "\n",
        "    char_map = {}\n",
        "    char_map[\"\"] = 0\n",
        "    char_map[\"<SPACE>\"] = 1\n",
        "    idx = 2\n",
        "    for c in supported:\n",
        "        char_map[c] = idx\n",
        "        idx += 1\n",
        "    index_map = {v: k for k, v in char_map.items()}\n",
        "    return char_map, index_map"
      ],
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VfjpMjZYU6nj"
      },
      "source": [
        "char_map, index_map = character_dict()"
      ],
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMCrhqBSU6nk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f0bf1399-6b1f-4b1d-d9ff-c7267f83240c"
      },
      "source": [
        "char_map"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'': 0,\n",
              " '<SPACE>': 1,\n",
              " 'a': 2,\n",
              " 'b': 3,\n",
              " 'c': 4,\n",
              " 'd': 5,\n",
              " 'e': 6,\n",
              " 'f': 7,\n",
              " 'g': 8,\n",
              " 'h': 9,\n",
              " 'i': 10,\n",
              " 'j': 11,\n",
              " 'k': 12,\n",
              " 'l': 13,\n",
              " 'm': 14,\n",
              " 'n': 15,\n",
              " 'o': 16,\n",
              " 'p': 17,\n",
              " 'q': 18,\n",
              " 'r': 19,\n",
              " 's': 20,\n",
              " 't': 21,\n",
              " 'u': 22,\n",
              " 'v': 23,\n",
              " 'w': 24,\n",
              " 'x': 25,\n",
              " 'y': 26,\n",
              " 'z': 27}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C9nJ78TV8X-b"
      },
      "source": [
        "def text_to_int_sequence(text):\n",
        "    \"\"\" Convert text to an integer sequence \"\"\"\n",
        "    int_sequence = []\n",
        "    for c in text:\n",
        "        if c == ' ':\n",
        "            ch = char_map['<SPACE>']\n",
        "        elif c in alphabets:\n",
        "            ch = char_map[c]\n",
        "        else:\n",
        "            print(c)\n",
        "            print('character not found')\n",
        "            break\n",
        "        int_sequence.append(ch)\n",
        "    return np.array(int_sequence)"
      ],
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4y39Q5Bq6frJ"
      },
      "source": [
        "def int_sequence_to_text(int_sequence):\n",
        "    \"\"\" Convert an integer sequence to text \"\"\"\n",
        "    textch = []\n",
        "    for c in int_sequence:\n",
        "        ch = index_map[c]\n",
        "        textch.append(ch)\n",
        "    text = ''.join(textch)\n",
        "    text = text.replace('<SPACE>', ' ')\n",
        "    return text"
      ],
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dmvz_Svx7U7l"
      },
      "source": [
        "class DataGenerator(tf.keras.utils.Sequence):\n",
        "    def __init__(self, audios, texts, batch_size=32):\n",
        "        self.audios = audios\n",
        "        self.texts = texts\n",
        "        self.batch_size = batch_size\n",
        "        self.steps = int(len(self.audios) // self.batch_size)\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.steps\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        self.indexes = np.arange(self.steps*self.batch_size)\n",
        "        # np.random.shuffle(self.indexes)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        indexes = self.indexes[int(index*self.batch_size):int((index+1)*self.batch_size)]\n",
        "    \n",
        "        batch_audios = [self.audios[int(i)] for i in indexes]\n",
        "        batch_texts = [self.texts[int(i)] for i in indexes]\n",
        "        \n",
        "        return  self.data_generation(batch_audios, batch_texts)\n",
        "    \n",
        "    def data_generation(self, batch_audios, batch_texts):\n",
        "\n",
        "        longest_audio = max([len(i) for i in batch_audios])\n",
        "        longest_txt = max([len(i) for i in batch_texts])\n",
        "\n",
        "        audios          = np.zeros([int(self.batch_size), longest_audio], dtype=\"float32\")\n",
        "        txts            = np.zeros([int(self.batch_size), longest_txt], dtype=\"int64\")\n",
        "        audio_length    = np.zeros([int(self.batch_size)], dtype=\"int64\")\n",
        "        txt_length      = np.zeros([int(self.batch_size)], dtype=\"int64\")\n",
        "\n",
        "        i = 0\n",
        "        for audio, txt in zip(batch_audios, batch_texts):\n",
        "\n",
        "            txt_len = len(txt)\n",
        "\n",
        "            txt = text_to_int_sequence(txt)\n",
        "       \n",
        "            txts[i,: txt_len] = txt\n",
        "\n",
        "            audio_len = len(audio)\n",
        "\n",
        "            audios[i, :audio_len] = audio\n",
        "\n",
        "            audio_length[i] = audio_len\n",
        "            txt_length[i] = txt_len\n",
        "\n",
        "            i+=1          \n",
        "            \n",
        "        outputs = {'ctc': np.zeros([self.batch_size])}\n",
        "        inputs = {\n",
        "                    'the_input':    tf.convert_to_tensor(audios), \n",
        "                    'the_labels':   tf.convert_to_tensor(txts), \n",
        "                    'input_length': tf.convert_to_tensor(audio_length), \n",
        "                    'label_length': tf.convert_to_tensor(txt_length)\n",
        "                }\n",
        "        return (inputs, outputs)\n"
      ],
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FMZLj1GeQny1"
      },
      "source": [
        "dg = DataGenerator(sample_audios, clean_txts)"
      ],
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E9yv2uTEBcWp",
        "outputId": "12ce1aa6-bb52-4053-a302-5b62bca05ff2"
      },
      "source": [
        "len(dg)"
      ],
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "34"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8F9PAObyKKNM",
        "outputId": "e39cf2a7-c894-4258-a892-3ea9bc7fdd15"
      },
      "source": [
        "dg[5]"
      ],
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'input_length': <tf.Tensor: shape=(32,), dtype=int64, numpy=\n",
              "  array([ 76292,  49616,  98786,  64826,  52920, 113343,  54906,  50274,\n",
              "          61960,  88644, 128772,  78278,  84671,  69458,  48289,  54686,\n",
              "          58433,  48510,  69017,  50054,  77175,  78938,  60858,  91508,\n",
              "         104741,  90189,  69458, 100769,  69899,  62180,  63284, 106942])>,\n",
              "  'label_length': <tf.Tensor: shape=(32,), dtype=int64, numpy=\n",
              "  array([50, 15, 97, 49, 49,  5, 56, 20, 51, 74, 60, 46, 62, 52, 44, 33, 41,\n",
              "         39, 58, 48, 53, 62, 74, 71, 57, 77, 27, 82, 18, 54, 35, 76])>,\n",
              "  'the_input': <tf.Tensor: shape=(32, 128772), dtype=float32, numpy=\n",
              "  array([[-0.00383717, -0.00409213, -0.0034892 , ...,  0.        ,\n",
              "           0.        ,  0.        ],\n",
              "         [ 0.03015642, -0.02410989, -0.03892324, ...,  0.        ,\n",
              "           0.        ,  0.        ],\n",
              "         [-0.02567921, -0.02798967, -0.02170274, ...,  0.        ,\n",
              "           0.        ,  0.        ],\n",
              "         ...,\n",
              "         [-0.00794118, -0.01024136, -0.00763915, ...,  0.        ,\n",
              "           0.        ,  0.        ],\n",
              "         [ 0.00313044,  0.01564104,  0.02499356, ...,  0.        ,\n",
              "           0.        ,  0.        ],\n",
              "         [-0.02381748, -0.03126577, -0.03101708, ...,  0.        ,\n",
              "           0.        ,  0.        ]], dtype=float32)>,\n",
              "  'the_labels': <tf.Tensor: shape=(32, 97), dtype=int64, numpy=\n",
              "  array([[12, 24,  2, ...,  0,  0,  0],\n",
              "         [24, 24, 24, ...,  0,  0,  0],\n",
              "         [12, 10, 13, ..., 22, 24,  2],\n",
              "         ...,\n",
              "         [21,  2, 15, ...,  0,  0,  0],\n",
              "         [ 9,  2, 10, ...,  0,  0,  0],\n",
              "         [20, 10, 15, ...,  0,  0,  0]])>},\n",
              " {'ctc': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b2V8FZlhWdIt"
      },
      "source": [
        "batch1 = dg[0][0]\n"
      ],
      "execution_count": 119,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EvPWc-V09v-c",
        "outputId": "7d34cfc8-d58c-45e4-ccbe-e04f9b01037a"
      },
      "source": [
        "batch1['the_labels']"
      ],
      "execution_count": 120,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(32, 109), dtype=int64, numpy=\n",
              "array([[11,  2, 15, ...,  0,  0,  0],\n",
              "       [ 9,  2, 13, ...,  0,  0,  0],\n",
              "       [14,  2, 14, ...,  0,  0,  0],\n",
              "       ...,\n",
              "       [12, 24,  2, ...,  0,  0,  0],\n",
              "       [12,  2, 21, ...,  0,  0,  0],\n",
              "       [15, 10,  1, ...,  0,  0,  0]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9qL0X05E9cit",
        "outputId": "3670db90-1533-47c9-9814-e56bd2584097"
      },
      "source": [
        "batch1['the_input']"
      ],
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(32, 117746), dtype=float32, numpy=\n",
              "array([[-1.9447643e-02, -2.3805585e-02, -1.9458510e-02, ...,\n",
              "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00],\n",
              "       [ 6.4895018e-03, -9.7726457e-05, -6.6772741e-03, ...,\n",
              "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00],\n",
              "       [-1.2900349e-02, -1.4687851e-02, -1.2652838e-02, ...,\n",
              "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00],\n",
              "       ...,\n",
              "       [ 3.2423830e-03,  2.7590861e-05, -3.5644281e-03, ...,\n",
              "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00],\n",
              "       [ 8.2196379e-03,  9.0677310e-03,  7.8771934e-03, ...,\n",
              "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00],\n",
              "       [-1.0445359e-04, -1.1000659e-03, -1.9130670e-03, ...,\n",
              "         0.0000000e+00,  0.0000000e+00,  0.0000000e+00]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6C5AAUQvbiyk"
      },
      "source": [
        "### ctc\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IQIvrmyvjW13"
      },
      "source": [
        "def ctc_lambda_func(args):\n",
        "    y_pred, labels, input_length, label_length = args\n",
        "    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)"
      ],
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xnn6WhlZ4HJr"
      },
      "source": [
        "def input_lengths_lambda_func(args):\n",
        "    input_length = args\n",
        "    return tf.cast(tf.math.floor(input_length/hop_size)-1, dtype=\"float32\")"
      ],
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kdD6eacRjLtx"
      },
      "source": [
        "def add_ctc_loss(model_builder):\n",
        "    the_labels      = Input(name='the_labels',      shape=(None,), dtype='float32')\n",
        "    input_lengths   = Input(name='input_length',    shape=(1,), dtype='float32')\n",
        "    label_lengths   = Input(name='label_length',    shape=(1,), dtype='float32')\n",
        "\n",
        "    input_lengths2 = Lambda(input_lengths_lambda_func)(input_lengths)\n",
        "    if model_builder.output_length:\n",
        "         output_lengths  = Lambda(model_builder.output_length)(input_lengths2)\n",
        "    else:\n",
        "         output_lengths  = input_lengths2\n",
        "    \n",
        "    # CTC loss is implemented in a lambda layer\n",
        "    loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([model_builder.output, the_labels, output_lengths, label_lengths])\n",
        "    model = Model( inputs=[model_builder.input, the_labels, input_lengths, label_lengths],  outputs=loss_out)\n",
        "    return model"
      ],
      "execution_count": 115,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vdGaA4MTU6nx"
      },
      "source": [
        "## LogMelSpectrogram"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p0lSFc5roaSe"
      },
      "source": [
        "class LogMelSpectrogram(tf.keras.layers.Layer):\n",
        "    \"\"\"Compute log-magnitude mel-scaled spectrograms.\"\"\"\n",
        "\n",
        "    def __init__(self, sample_rate, fft_size, hop_size, n_mels,\n",
        "                 f_min=0.0, f_max=None, **kwargs):\n",
        "        super(LogMelSpectrogram, self).__init__(**kwargs)\n",
        "        self.sample_rate = sample_rate\n",
        "        self.fft_size = fft_size\n",
        "        self.hop_size = hop_size\n",
        "        self.n_mels = n_mels\n",
        "        self.f_min = f_min\n",
        "        self.f_max = f_max if f_max else sample_rate / 2\n",
        "        self.mel_filterbank = tf.signal.linear_to_mel_weight_matrix(\n",
        "            num_mel_bins=self.n_mels,\n",
        "            num_spectrogram_bins=fft_size // 2 + 1,\n",
        "            sample_rate=self.sample_rate,\n",
        "            lower_edge_hertz=self.f_min,\n",
        "            upper_edge_hertz=self.f_max)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        self.non_trainable_weights.append(self.mel_filterbank)\n",
        "        super(LogMelSpectrogram, self).build(input_shape)\n",
        "\n",
        "    def call(self, waveforms):\n",
        "        \"\"\"Forward pass.\n",
        "        Parameters\n",
        "        ----------\n",
        "        waveforms : tf.Tensor, shape = (None, n_samples)\n",
        "            A Batch of mono waveforms.\n",
        "        Returns\n",
        "        -------\n",
        "        log_mel_spectrograms : (tf.Tensor), shape = (None, time, freq, ch)\n",
        "            The corresponding batch of log-mel-spectrograms\n",
        "        \"\"\"\n",
        "        def _tf_log10(x):\n",
        "            numerator = tf.math.log(x)\n",
        "            denominator = tf.math.log(tf.constant(10, dtype=numerator.dtype))\n",
        "            return numerator / denominator\n",
        "\n",
        "        def power_to_db(magnitude, amin=1e-16, top_db=80.0):\n",
        "            \"\"\"\n",
        "            https://librosa.github.io/librosa/generated/librosa.core.power_to_db.html\n",
        "            \"\"\"\n",
        "            ref_value = tf.reduce_max(magnitude)\n",
        "            log_spec = 10.0 * _tf_log10(tf.maximum(amin, magnitude))\n",
        "            log_spec -= 10.0 * _tf_log10(tf.maximum(amin, ref_value))\n",
        "            log_spec = tf.maximum(log_spec, tf.reduce_max(log_spec) - top_db)\n",
        "\n",
        "            return log_spec\n",
        "\n",
        "        spectrograms = tf.signal.stft(waveforms,\n",
        "                                      frame_length=self.fft_size,\n",
        "                                      frame_step=self.hop_size,\n",
        "                                      pad_end=False)\n",
        "\n",
        "        magnitude_spectrograms = tf.abs(spectrograms)\n",
        "\n",
        "        mel_spectrograms = tf.matmul(tf.square(magnitude_spectrograms),\n",
        "                                     self.mel_filterbank)\n",
        "\n",
        "        log_mel_spectrograms = power_to_db(mel_spectrograms)\n",
        "\n",
        "        # add channel dimension\n",
        "        log_mel_spectrograms = tf.expand_dims(log_mel_spectrograms, 3)\n",
        "\n",
        "        return log_mel_spectrograms\n",
        "\n",
        "    def get_config(self):\n",
        "        config = {\n",
        "            'fft_size': self.fft_size,\n",
        "            'hop_size': self.hop_size,\n",
        "            'n_mels': self.n_mels,\n",
        "            'sample_rate': self.sample_rate,\n",
        "            'f_min': self.f_min,\n",
        "            'f_max': self.f_max,\n",
        "        }\n",
        "        config.update(super(LogMelSpectrogram, self).get_config())\n",
        "\n",
        "        return config"
      ],
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qxIVBU1NkAX5"
      },
      "source": [
        "## Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "38567d3a"
      },
      "source": [
        "def preprocessin_model(sample_rate, fft_size, frame_step, n_mels, mfcc=False):\n",
        "\n",
        "    input_data = Input(name='input', shape=(None,), dtype=\"float32\")\n",
        "    featLayer = LogMelSpectrogram(\n",
        "        fft_size=fft_size,\n",
        "        hop_size=frame_step,\n",
        "        n_mels=n_mels,\n",
        "        \n",
        "        sample_rate=sample_rate,\n",
        "        f_min=0.0,\n",
        "        \n",
        "        f_max=int(sample_rate / 2)\n",
        "    )(input_data)\n",
        "    \n",
        "    x = BatchNormalization()(featLayer)\n",
        "    model = Model(inputs=input_data, outputs=x, name=\"preprocessin_model\")\n",
        "\n",
        "    return model"
      ],
      "execution_count": 123,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fa7b082"
      },
      "source": [
        "def simple_rnn_model(input_dim, output_dim=224):\n",
        "\n",
        "    input_data = Input(name='the_input', shape=(None, input_dim))\n",
        "    simp_rnn = GRU(output_dim, return_sequences=True,\n",
        "                   implementation=2, name='rnn')(input_data)\n",
        "    y_pred = Activation('softmax', name='softmax')(simp_rnn)\n",
        "    model = Model(inputs=input_data, outputs=y_pred, name=\"simple_rnn_model\")\n",
        "    model.output_length = lambda x: x\n",
        "    return model\n"
      ],
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wBW9WjFwDWxW"
      },
      "source": [
        "def BidirectionalRNN(input_dim, rnn_layers=2, units=400, drop_out=0.5, act='tanh', output_dim=224):\n",
        "\n",
        "    input_data = Input(name='the_input', shape=(\n",
        "        None, input_dim))\n",
        "\n",
        "    x = Bidirectional(LSTM(units,  activation=act,\n",
        "                      return_sequences=True, implementation=2))(input_data)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = Dropout(drop_out)(x)\n",
        "\n",
        "    for i in range(rnn_layers - 2):\n",
        "        x = Bidirectional(\n",
        "            LSTM(units, activation=act, return_sequences=True))(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Dropout(drop_out)(x)\n",
        "\n",
        "    x = Bidirectional(LSTM(units,  activation=act,\n",
        "                      return_sequences=True, implementation=2))(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = Dropout(drop_out)(x)\n",
        "\n",
        "    time_dense = TimeDistributed(Dense(output_dim))(x)\n",
        "\n",
        "    y_pred = Activation('softmax', name='softmax')(time_dense)\n",
        "\n",
        "    model = Model(inputs=input_data, outputs=y_pred, name=\"BidirectionalRNN\")\n",
        "\n",
        "    return model"
      ],
      "execution_count": 125,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XWc7n6yF5QKe"
      },
      "source": [
        "def conv_rnn(n_mels, output_dim=224, rnn_layers=4, units=400, drop_out=0.5, act='tanh'):\n",
        "\n",
        "    input_data = Input(name='the_input', shape=(None, n_mels, 1))\n",
        "\n",
        "    y = Conv2D(32, (3, 3), padding='same')(input_data)  # was 32\n",
        "    y = Activation('relu')(y)\n",
        "    y = BatchNormalization()(y)\n",
        "\n",
        "    x = MaxPooling2D((1, 2))(y)\n",
        "\n",
        "    x = Conv2D(64, (3, 3), padding='same')(y)  # was 32\n",
        "    x = Activation('relu')(x)\n",
        "    y = BatchNormalization()(y)\n",
        "\n",
        "    x = MaxPooling2D((1, 2))(x)\n",
        "\n",
        "    x = Conv2D(128, (3, 3), padding='same')(x)\n",
        "    x = Activation('relu')(x)\n",
        "    y = BatchNormalization()(y)\n",
        "\n",
        "    x = MaxPooling2D((1, 2))(x)\n",
        "\n",
        "    x = Dense(128)(x)\n",
        "    x = Dense(64)(x)\n",
        "    x = Dense(32)(x)\n",
        "\n",
        "    x = Reshape((-1, x.shape[-1] * x.shape[-2]))(x)\n",
        "\n",
        "    for i in range(rnn_layers):\n",
        "        x = Bidirectional(\n",
        "            LSTM(units, activation=act, return_sequences=True))(x)\n",
        "        x = BatchNormalization()(x)\n",
        "        x = Dropout(drop_out)(x)\n",
        "\n",
        "    bn_rnn = BatchNormalization()(x)\n",
        "\n",
        "    time_dense = TimeDistributed(Dense(output_dim))(bn_rnn)\n",
        "\n",
        "    y_pred = Activation('softmax', name='softmax')(time_dense)\n",
        "\n",
        "    model = Model(inputs=input_data, outputs=y_pred, name=\"custom_model\")\n",
        "\n",
        "    return model"
      ],
      "execution_count": 126,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ae84266"
      },
      "source": [
        "def train(model_builder, \n",
        "          data_gen,\n",
        "          epochs, \n",
        "          verbose=1,\n",
        "          optimizer=SGD(learning_rate=0.002, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5),\n",
        "          ):    \n",
        "              \n",
        "    model = add_ctc_loss(model_builder)\n",
        "\n",
        "    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer)\n",
        "    print(model.summary())\n",
        "\n",
        "    # add checkpointer\n",
        "    checkpointer = ModelCheckpoint(filepath='/content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5', verbose=1)\n",
        "    early_stopping = EarlyStopping( monitor=\"val_loss\", patience=10, restore_best_weights=True)\n",
        "\n",
        "    hist = model.fit_generator(generator=data_gen,\n",
        "                               epochs=epochs,\n",
        "                               callbacks=[checkpointer, early_stopping], \n",
        "                               verbose=verbose, \n",
        "                               use_multiprocessing=False)\n",
        "    \n",
        "    # save model loss\n",
        "    with open('/content/drive/MyDrive/structured_data/train/models/cnn_rnn.pickle', 'wb') as f:\n",
        "        pickle.dump(hist.history, f)"
      ],
      "execution_count": 127,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GEC6FFW_kayX"
      },
      "source": [
        "### Trainig"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fa980211"
      },
      "source": [
        "\n",
        "sample_rate = 22050\n",
        "fft_size = 1024\n",
        "frame_step = 512\n",
        "n_mels = 128\n",
        "\n",
        "batch_size = 32\n",
        "epochs = 100\n",
        "data_len = len(clean_txts)\n",
        "output_dim = len(char_map) + 2\n"
      ],
      "execution_count": 128,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a97ffc77"
      },
      "source": [
        "dg = DataGenerator(sample_audios, clean_txts, batch_size)"
      ],
      "execution_count": 129,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "68e9985a",
        "outputId": "783dc184-7e55-460c-9ff2-e92b877e9147"
      },
      "source": [
        "preprocess_model = preprocessin_model(sample_rate, fft_size, frame_step, n_mels)\n",
        "preprocess_model.summary()\n"
      ],
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"preprocessin_model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input (InputLayer)           [(None, None)]            0         \n",
            "_________________________________________________________________\n",
            "log_mel_spectrogram_3 (LogMe (None, None, 128, 1)      0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_27 (Batc (None, None, 128, 1)      4         \n",
            "=================================================================\n",
            "Total params: 4\n",
            "Trainable params: 2\n",
            "Non-trainable params: 2\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2b8711e1",
        "outputId": "48100620-5244-4eea-ed6b-922411be61fe"
      },
      "source": [
        "speech_model = conv_rnn(n_mels, output_dim = output_dim)\n",
        "speech_model.summary()\n"
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"custom_model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "the_input (InputLayer)       [(None, None, 128, 1)]    0         \n",
            "_________________________________________________________________\n",
            "conv2d_9 (Conv2D)            (None, None, 128, 32)     320       \n",
            "_________________________________________________________________\n",
            "activation_9 (Activation)    (None, None, 128, 32)     0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_28 (Batc (None, None, 128, 32)     128       \n",
            "_________________________________________________________________\n",
            "conv2d_10 (Conv2D)           (None, None, 128, 64)     18496     \n",
            "_________________________________________________________________\n",
            "activation_10 (Activation)   (None, None, 128, 64)     0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_10 (MaxPooling (None, None, 64, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv2d_11 (Conv2D)           (None, None, 64, 128)     73856     \n",
            "_________________________________________________________________\n",
            "activation_11 (Activation)   (None, None, 64, 128)     0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_11 (MaxPooling (None, None, 32, 128)     0         \n",
            "_________________________________________________________________\n",
            "dense_12 (Dense)             (None, None, 32, 128)     16512     \n",
            "_________________________________________________________________\n",
            "dense_13 (Dense)             (None, None, 32, 64)      8256      \n",
            "_________________________________________________________________\n",
            "dense_14 (Dense)             (None, None, 32, 32)      2080      \n",
            "_________________________________________________________________\n",
            "reshape_3 (Reshape)          (None, None, 1024)        0         \n",
            "_________________________________________________________________\n",
            "bidirectional_12 (Bidirectio (None, None, 800)         4560000   \n",
            "_________________________________________________________________\n",
            "batch_normalization_31 (Batc (None, None, 800)         3200      \n",
            "_________________________________________________________________\n",
            "dropout_12 (Dropout)         (None, None, 800)         0         \n",
            "_________________________________________________________________\n",
            "bidirectional_13 (Bidirectio (None, None, 800)         3843200   \n",
            "_________________________________________________________________\n",
            "batch_normalization_32 (Batc (None, None, 800)         3200      \n",
            "_________________________________________________________________\n",
            "dropout_13 (Dropout)         (None, None, 800)         0         \n",
            "_________________________________________________________________\n",
            "bidirectional_14 (Bidirectio (None, None, 800)         3843200   \n",
            "_________________________________________________________________\n",
            "batch_normalization_33 (Batc (None, None, 800)         3200      \n",
            "_________________________________________________________________\n",
            "dropout_14 (Dropout)         (None, None, 800)         0         \n",
            "_________________________________________________________________\n",
            "bidirectional_15 (Bidirectio (None, None, 800)         3843200   \n",
            "_________________________________________________________________\n",
            "batch_normalization_34 (Batc (None, None, 800)         3200      \n",
            "_________________________________________________________________\n",
            "dropout_15 (Dropout)         (None, None, 800)         0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_35 (Batc (None, None, 800)         3200      \n",
            "_________________________________________________________________\n",
            "time_distributed_3 (TimeDist (None, None, 30)          24030     \n",
            "_________________________________________________________________\n",
            "softmax (Activation)         (None, None, 30)          0         \n",
            "=================================================================\n",
            "Total params: 16,249,278\n",
            "Trainable params: 16,241,214\n",
            "Non-trainable params: 8,064\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "62f39784"
      },
      "source": [
        "def build_model(output_dim, custom_model, preprocess_model, calc=None):\n",
        "\n",
        "    input_audios = Input(name='the_input', shape=(None,))\n",
        "    pre = preprocess_model(input_audios)\n",
        "    pre = tf.squeeze(pre, [3])\n",
        "\n",
        "    y_pred = custom_model(pre)\n",
        "    model = Model(inputs=input_audios, outputs=y_pred, name=\"model_builder\")\n",
        "    model.output_length = calc\n",
        "\n",
        "    return model"
      ],
      "execution_count": 132,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "35985dd6",
        "outputId": "c75ae365-20ac-4248-c5cb-0f5babe1dd7e"
      },
      "source": [
        "model = build_model(output_dim, speech_model, preprocess_model)\n",
        "model.summary()\n"
      ],
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_builder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "the_input (InputLayer)       [(None, None)]            0         \n",
            "_________________________________________________________________\n",
            "preprocessin_model (Function (None, None, 128, 1)      4         \n",
            "_________________________________________________________________\n",
            "tf.compat.v1.squeeze_3 (TFOp (None, None, 128)         0         \n",
            "_________________________________________________________________\n",
            "custom_model (Functional)    (None, None, 30)          16249278  \n",
            "=================================================================\n",
            "Total params: 16,249,282\n",
            "Trainable params: 16,241,216\n",
            "Non-trainable params: 8,066\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kvrVqeu8FvgQ"
      },
      "source": [
        "model.load_weights('/content/drive/MyDrive/structured_data/train/models/cnn_rnn3.h5')"
      ],
      "execution_count": 134,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N38b7Grd3zXj"
      },
      "source": [
        "# !pip install mlflow --quiet"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b6b2a7e2"
      },
      "source": [
        "# import mlflow"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ea406859",
        "outputId": "244e6368-540c-4401-8d3d-8f7abb61a907"
      },
      "source": [
        "# mlflow.set_experiment('Speech Model-RNN-baseline')\n",
        "# mlflow.tensorflow.autolog()\n",
        "hop_size = 512\n",
        "train(model, dg, epochs=100)\n"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_7\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "the_input (InputLayer)          [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "preprocessin_model (Functional) (None, None, 128, 1) 4           the_input[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "tf.compat.v1.squeeze_3 (TFOpLam (None, None, 128)    0           preprocessin_model[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "input_length (InputLayer)       [(None, 1)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "custom_model (Functional)       (None, None, 30)     16249278    tf.compat.v1.squeeze_3[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "the_labels (InputLayer)         [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lambda_7 (Lambda)               (None, 1)            0           input_length[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "label_length (InputLayer)       [(None, 1)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "ctc (Lambda)                    (None, 1)            0           custom_model[0][0]               \n",
            "                                                                 the_labels[0][0]                 \n",
            "                                                                 lambda_7[0][0]                   \n",
            "                                                                 label_length[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 16,249,282\n",
            "Trainable params: 16,241,216\n",
            "Non-trainable params: 8,066\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "Epoch 1/100\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:1940: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
            "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "34/34 [==============================] - 27s 567ms/step - loss: 48.5397\n",
            "\n",
            "Epoch 00001: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 2/100\n",
            "34/34 [==============================] - 20s 582ms/step - loss: 46.7798\n",
            "\n",
            "Epoch 00002: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 3/100\n",
            "34/34 [==============================] - 20s 580ms/step - loss: 45.7685\n",
            "\n",
            "Epoch 00003: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 4/100\n",
            "34/34 [==============================] - 20s 571ms/step - loss: 44.1256\n",
            "\n",
            "Epoch 00004: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 5/100\n",
            "34/34 [==============================] - 19s 565ms/step - loss: 42.7439\n",
            "\n",
            "Epoch 00005: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 6/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 41.3259\n",
            "\n",
            "Epoch 00006: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 7/100\n",
            "34/34 [==============================] - 20s 581ms/step - loss: 40.1269\n",
            "\n",
            "Epoch 00007: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 8/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 39.2883\n",
            "\n",
            "Epoch 00008: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 9/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 37.5927\n",
            "\n",
            "Epoch 00009: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 10/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 36.5857\n",
            "\n",
            "Epoch 00010: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 11/100\n",
            "34/34 [==============================] - 20s 571ms/step - loss: 35.7804\n",
            "\n",
            "Epoch 00011: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 12/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 34.8992\n",
            "\n",
            "Epoch 00012: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 13/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 33.5699\n",
            "\n",
            "Epoch 00013: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 14/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 33.0343\n",
            "\n",
            "Epoch 00014: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 15/100\n",
            "34/34 [==============================] - 20s 575ms/step - loss: 31.9176\n",
            "\n",
            "Epoch 00015: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 16/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 30.7577\n",
            "\n",
            "Epoch 00016: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 17/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 29.7100\n",
            "\n",
            "Epoch 00017: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 18/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 29.2900\n",
            "\n",
            "Epoch 00018: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 19/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 28.3538\n",
            "\n",
            "Epoch 00019: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 20/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 27.5446\n",
            "\n",
            "Epoch 00020: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 21/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 26.6095\n",
            "\n",
            "Epoch 00021: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 22/100\n",
            "34/34 [==============================] - 19s 571ms/step - loss: 26.2184\n",
            "\n",
            "Epoch 00022: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 23/100\n",
            "34/34 [==============================] - 20s 576ms/step - loss: 25.8485\n",
            "\n",
            "Epoch 00023: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 24/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 25.0630\n",
            "\n",
            "Epoch 00024: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 25/100\n",
            "34/34 [==============================] - 20s 570ms/step - loss: 24.2988\n",
            "\n",
            "Epoch 00025: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 26/100\n",
            "34/34 [==============================] - 20s 571ms/step - loss: 23.7840\n",
            "\n",
            "Epoch 00026: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 27/100\n",
            "34/34 [==============================] - 19s 572ms/step - loss: 23.0746\n",
            "\n",
            "Epoch 00027: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 28/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 22.8091\n",
            "\n",
            "Epoch 00028: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 29/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 22.1586\n",
            "\n",
            "Epoch 00029: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 30/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 21.6893\n",
            "\n",
            "Epoch 00030: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 31/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 21.2147\n",
            "\n",
            "Epoch 00031: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 32/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 20.5552\n",
            "\n",
            "Epoch 00032: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 33/100\n",
            "34/34 [==============================] - 20s 576ms/step - loss: 20.2954\n",
            "\n",
            "Epoch 00033: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 34/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 19.7116\n",
            "\n",
            "Epoch 00034: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 35/100\n",
            "34/34 [==============================] - 19s 566ms/step - loss: 19.2656\n",
            "\n",
            "Epoch 00035: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 36/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 19.0924\n",
            "\n",
            "Epoch 00036: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 37/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 18.4083\n",
            "\n",
            "Epoch 00037: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 38/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 17.9969\n",
            "\n",
            "Epoch 00038: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 39/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 17.8372\n",
            "\n",
            "Epoch 00039: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 40/100\n",
            "34/34 [==============================] - 19s 564ms/step - loss: 17.1260\n",
            "\n",
            "Epoch 00040: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 41/100\n",
            "34/34 [==============================] - 20s 583ms/step - loss: 17.0283\n",
            "\n",
            "Epoch 00041: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 42/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 16.6456\n",
            "\n",
            "Epoch 00042: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 43/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 16.4503\n",
            "\n",
            "Epoch 00043: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 44/100\n",
            "34/34 [==============================] - 19s 573ms/step - loss: 15.8909\n",
            "\n",
            "Epoch 00044: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 45/100\n",
            "34/34 [==============================] - 20s 570ms/step - loss: 15.8254\n",
            "\n",
            "Epoch 00045: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 46/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 15.0997\n",
            "\n",
            "Epoch 00046: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 47/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 14.8243\n",
            "\n",
            "Epoch 00047: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 48/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 14.5989\n",
            "\n",
            "Epoch 00048: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 49/100\n",
            "34/34 [==============================] - 20s 575ms/step - loss: 14.4338\n",
            "\n",
            "Epoch 00049: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 50/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 14.1483\n",
            "\n",
            "Epoch 00050: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 51/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 14.0081\n",
            "\n",
            "Epoch 00051: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 52/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 14.1976\n",
            "\n",
            "Epoch 00052: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 53/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 13.5791\n",
            "\n",
            "Epoch 00053: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 54/100\n",
            "34/34 [==============================] - 19s 565ms/step - loss: 13.1339\n",
            "\n",
            "Epoch 00054: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 55/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 13.0239\n",
            "\n",
            "Epoch 00055: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 56/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 12.7983\n",
            "\n",
            "Epoch 00056: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 57/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 12.5079\n",
            "\n",
            "Epoch 00057: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 58/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 12.2203\n",
            "\n",
            "Epoch 00058: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 59/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 11.9382\n",
            "\n",
            "Epoch 00059: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 60/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 11.9083\n",
            "\n",
            "Epoch 00060: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 61/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 11.6169\n",
            "\n",
            "Epoch 00061: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 62/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 11.6063\n",
            "\n",
            "Epoch 00062: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 63/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 11.1925\n",
            "\n",
            "Epoch 00063: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 64/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 11.2169\n",
            "\n",
            "Epoch 00064: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 65/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 11.0159\n",
            "\n",
            "Epoch 00065: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 66/100\n",
            "34/34 [==============================] - 19s 572ms/step - loss: 10.6053\n",
            "\n",
            "Epoch 00066: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 67/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 10.7639\n",
            "\n",
            "Epoch 00067: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 68/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 10.4338\n",
            "\n",
            "Epoch 00068: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 69/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 10.1445\n",
            "\n",
            "Epoch 00069: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 70/100\n",
            "34/34 [==============================] - 19s 566ms/step - loss: 10.2466\n",
            "\n",
            "Epoch 00070: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 71/100\n",
            "34/34 [==============================] - 19s 572ms/step - loss: 10.0303\n",
            "\n",
            "Epoch 00071: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 72/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 10.0191\n",
            "\n",
            "Epoch 00072: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 73/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 9.4472\n",
            "\n",
            "Epoch 00073: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 74/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 9.4314\n",
            "\n",
            "Epoch 00074: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 75/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 9.4117\n",
            "\n",
            "Epoch 00075: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 76/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 9.0642\n",
            "\n",
            "Epoch 00076: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 77/100\n",
            "34/34 [==============================] - 19s 571ms/step - loss: 8.8870\n",
            "\n",
            "Epoch 00077: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 78/100\n",
            "34/34 [==============================] - 20s 575ms/step - loss: 8.9282\n",
            "\n",
            "Epoch 00078: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 79/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 8.9739\n",
            "\n",
            "Epoch 00079: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 80/100\n",
            "34/34 [==============================] - 20s 579ms/step - loss: 8.6647\n",
            "\n",
            "Epoch 00080: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 81/100\n",
            "34/34 [==============================] - 20s 570ms/step - loss: 8.4257\n",
            "\n",
            "Epoch 00081: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 82/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 8.5937\n",
            "\n",
            "Epoch 00082: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 83/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 8.3279\n",
            "\n",
            "Epoch 00083: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 84/100\n",
            "34/34 [==============================] - 20s 572ms/step - loss: 8.0158\n",
            "\n",
            "Epoch 00084: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 85/100\n",
            "34/34 [==============================] - 19s 564ms/step - loss: 8.1327\n",
            "\n",
            "Epoch 00085: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 86/100\n",
            "34/34 [==============================] - 20s 582ms/step - loss: 7.8640\n",
            "\n",
            "Epoch 00086: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 87/100\n",
            "34/34 [==============================] - 20s 576ms/step - loss: 7.8188\n",
            "\n",
            "Epoch 00087: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 88/100\n",
            "34/34 [==============================] - 19s 568ms/step - loss: 7.7013\n",
            "\n",
            "Epoch 00088: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 89/100\n",
            "34/34 [==============================] - 19s 569ms/step - loss: 7.5034\n",
            "\n",
            "Epoch 00089: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 90/100\n",
            "34/34 [==============================] - 20s 576ms/step - loss: 7.4524\n",
            "\n",
            "Epoch 00090: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 91/100\n",
            "34/34 [==============================] - 19s 570ms/step - loss: 7.3499\n",
            "\n",
            "Epoch 00091: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 92/100\n",
            "34/34 [==============================] - 20s 581ms/step - loss: 7.3828\n",
            "\n",
            "Epoch 00092: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 93/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 7.3912\n",
            "\n",
            "Epoch 00093: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 94/100\n",
            "34/34 [==============================] - 19s 565ms/step - loss: 6.8932\n",
            "\n",
            "Epoch 00094: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 95/100\n",
            "34/34 [==============================] - 19s 567ms/step - loss: 6.9892\n",
            "\n",
            "Epoch 00095: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 96/100\n",
            "34/34 [==============================] - 20s 574ms/step - loss: 6.9707\n",
            "\n",
            "Epoch 00096: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 97/100\n",
            "34/34 [==============================] - 20s 573ms/step - loss: 6.7217\n",
            "\n",
            "Epoch 00097: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 98/100\n",
            "34/34 [==============================] - 19s 566ms/step - loss: 6.8565\n",
            "\n",
            "Epoch 00098: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 99/100\n",
            "34/34 [==============================] - 19s 571ms/step - loss: 6.6752\n",
            "\n",
            "Epoch 00099: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n",
            "Epoch 100/100\n",
            "34/34 [==============================] - 20s 575ms/step - loss: 6.6256\n",
            "\n",
            "Epoch 00100: saving model to /content/drive/MyDrive/structured_data/train/models/cnn_rnn.h5\n",
            "WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f5ZKNcVjcpB5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ac11b1bb-147f-4a1e-c372-baa94f033de8"
      },
      "source": [
        "!pip install jiwer --quiet"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |██████▌                         | 10 kB 35.0 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 20 kB 25.7 MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 30 kB 17.8 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 40 kB 15.7 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 50 kB 4.2 MB/s \n",
            "\u001b[?25h  Building wheel for python-Levenshtein (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1-1frHIJYv5k"
      },
      "source": [
        "from jiwer import wer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Al0dtmT1YK0p"
      },
      "source": [
        "def predict(data_gen,  num_elem=1, index=0):\n",
        "    \n",
        "    pred_data = data_gen.__getitem__(index)\n",
        "\n",
        "    pred_audios = pred_data[0][\"the_input\"]\n",
        "    pred_labels = pred_data[0][\"the_labels\"]\n",
        "    \n",
        "    y_pred = model.predict(pred_audios)\n",
        "\n",
        "    input_shape = tf.keras.backend.shape(y_pred)\n",
        "    input_length = tf.ones(shape=input_shape[0]) * tf.keras.backend.cast(input_shape[1], 'float32')\n",
        "    prediction = tf.keras.backend.ctc_decode(y_pred, input_length, greedy=False)[0][0]\n",
        "    \n",
        "\n",
        "    for i in range(0, num_elem):  # only on clean data\n",
        "        \n",
        "        pred = K.eval(prediction[i]).flatten().tolist()\n",
        "        pred = [i for i in pred if i != -1]\n",
        "\n",
        "\n",
        "\n",
        "        ground_truth = int_sequence_to_text(pred_labels[i].numpy())\n",
        "        hypothesis   = ''.join(int_sequence_to_text(pred))\n",
        "        # error        = wer(ground_truth, hypothesis)\n",
        "                \n",
        "        print('-'*48 + ' ' + str(i) + ' ' + '-'*48)\n",
        "        print('True transcription:\\n' + '\\n' + ground_truth)\n",
        "        print('-'*100)\n",
        "        print('Predicted transcription:\\n' + '\\n' + hypothesis)\n",
        "        # print('-'*100)\n",
        "        # print('Word Error Rate:' + str(error))\n",
        "        print('\\n')"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QzC37XxWZH4o"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fJnS0XSnZSt_",
        "outputId": "93a3bc63-05af-4bd5-bda8-0ba1f6c9b25a"
      },
      "source": [
        "rows1 = []\n",
        "parent_dir1 = \"./SWH-05-20101107\"\n",
        "files1 = os.listdir(parent_dir1)\n",
        "for f in files1:\n",
        "    audio, fs = librosa.load(f\"{parent_dir1}/{f}\")\n",
        "    filename = f.split('.')[0]\n",
        "    row = {'filename': filename, 'audio': audio}\n",
        "    rows1.append(row)\n",
        "rows1[:5]"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'audio': array([0.0043117 , 0.00573289, 0.00458423, ..., 0.01505196, 0.02441092,\n",
              "         0.01784338], dtype=float32),\n",
              "  'filename': 'SWH-05-20101107_16k-emission_swahili_05h30_-_06h00_tu_20101107_part1'},\n",
              " {'audio': array([ 0.00038796,  0.0003749 ,  0.00038821, ..., -0.13299914,\n",
              "         -0.15665275, -0.09959406], dtype=float32),\n",
              "  'filename': 'SWH-05-20101107_16k-emission_swahili_05h30_-_06h00_tu_20101107_part10'},\n",
              " {'audio': array([-0.00805496, -0.00836756, -0.00562658, ...,  0.00168179,\n",
              "         -0.0017211 ,  0.        ], dtype=float32),\n",
              "  'filename': 'SWH-05-20101107_16k-emission_swahili_05h30_-_06h00_tu_20101107_part100'},\n",
              " {'audio': array([0.01148875, 0.01478087, 0.01499198, ..., 0.07277071, 0.03534885,\n",
              "         0.        ], dtype=float32),\n",
              "  'filename': 'SWH-05-20101107_16k-emission_swahili_05h30_-_06h00_tu_20101107_part101'},\n",
              " {'audio': array([-0.0029805 , -0.00687017, -0.00985629, ...,  0.0279939 ,\n",
              "          0.0324325 ,  0.        ], dtype=float32),\n",
              "  'filename': 'SWH-05-20101107_16k-emission_swahili_05h30_-_06h00_tu_20101107_part102'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nm5CFcjsZSuH",
        "outputId": "71d1db44-c8ca-4641-8081-dcd4fe06dd91"
      },
      "source": [
        "sample_audios1 = []\n",
        "for row in rows1:\n",
        "    audio = row['audio']\n",
        "    sample_audios1.append(audio)\n",
        "sample_audios1[:5]"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([0.0043117 , 0.00573289, 0.00458423, ..., 0.01505196, 0.02441092,\n",
              "        0.01784338], dtype=float32),\n",
              " array([ 0.00038796,  0.0003749 ,  0.00038821, ..., -0.13299914,\n",
              "        -0.15665275, -0.09959406], dtype=float32),\n",
              " array([-0.00805496, -0.00836756, -0.00562658, ...,  0.00168179,\n",
              "        -0.0017211 ,  0.        ], dtype=float32),\n",
              " array([0.01148875, 0.01478087, 0.01499198, ..., 0.07277071, 0.03534885,\n",
              "        0.        ], dtype=float32),\n",
              " array([-0.0029805 , -0.00687017, -0.00985629, ...,  0.0279939 ,\n",
              "         0.0324325 ,  0.        ], dtype=float32)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L8BBrDjBZSuJ"
      },
      "source": [
        "meta_df = pd.read_csv('/content/drive/MyDrive/metadata.csv')"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xx8JWPEwZSuO",
        "outputId": "d2fe69ce-f210-42a4-9015-23f06a2da3bc"
      },
      "source": [
        "txts1 = []\n",
        "for row in rows1:\n",
        "    filename = row['filename']\n",
        "    filter = meta_df[meta_df['filename'] == filename]\n",
        "    txt = filter[['transcription']].values\n",
        "    txts1.append(txt)\n",
        "\n",
        "txts1[:5]"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([['karibu katika matangazo yetu ya asubuhi ya idhaa ya kiswahili']],\n",
              "       dtype=object),\n",
              " array([['kwenda na kuondoka jarkata indonesia']], dtype=object),\n",
              " array([['lazima uwe na outside support mataifa ya nje yakuunge mkono']],\n",
              "       dtype=object),\n",
              " array([['nne lazima uwe na military support eeh']], dtype=object),\n",
              " array([['lazima akuwe mshindi tu hapo unaona je kwa nini wasiwe washindi']],\n",
              "       dtype=object)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ft5U3nwAZSuR"
      },
      "source": [
        "txts1 = np.array(txts1).reshape(-1)"
      ],
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TvjcHFl8ZSuT",
        "outputId": "30459a85-3044-4f0a-8f70-481a10fb9537"
      },
      "source": [
        "txts1[:5]"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['karibu katika matangazo yetu ya asubuhi ya idhaa ya kiswahili',\n",
              "       'kwenda na kuondoka jarkata indonesia',\n",
              "       'lazima uwe na outside support mataifa ya nje yakuunge mkono',\n",
              "       'nne lazima uwe na military support eeh',\n",
              "       'lazima akuwe mshindi tu hapo unaona je kwa nini wasiwe washindi'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kFCM_27ZSuV"
      },
      "source": [
        "clean_txts1 = []\n",
        "alphabets = 'a b c d e f g h i j k l m n o p q r s t u v w x y z'.split()\n",
        "for txt in txts1:\n",
        "    clean_txt = []\n",
        "    for c in txt:\n",
        "        if c not in alphabets and c != ' ':\n",
        "            continue\n",
        "        clean_txt.append(c)\n",
        "    clean_txt = ''.join(clean_txt)\n",
        "    clean_txts1.append(clean_txt)"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YfIJMLzzZSuX",
        "outputId": "043ec233-4277-4755-b03e-1050eabefae6"
      },
      "source": [
        "clean_txts1[:5]"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['karibu katika matangazo yetu ya asubuhi ya idhaa ya kiswahili',\n",
              " 'kwenda na kuondoka jarkata indonesia',\n",
              " 'lazima uwe na outside support mataifa ya nje yakuunge mkono',\n",
              " 'nne lazima uwe na military support eeh',\n",
              " 'lazima akuwe mshindi tu hapo unaona je kwa nini wasiwe washindi']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fRJv5vTLZSuZ",
        "outputId": "d406bcc5-c597-4a82-8d65-6092f91f5c04"
      },
      "source": [
        "'' in clean_txts1"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q4JYiuMYaejv"
      },
      "source": [
        "dg1 = DataGenerator(sample_audios1[:5], clean_txts1[:5], 5)"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7F6q_hiEanxN",
        "outputId": "d3b5929a-7e62-4b95-b43e-c9be08efbba6"
      },
      "source": [
        "predict(dg1, 4)"
      ],
      "execution_count": 140,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "------------------------------------------------ 0 ------------------------------------------------\n",
            "True transcription:\n",
            "\n",
            "karibu katika matangazo yetu ya asubuhi ya idhaa ya kiswahili\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Predicted transcription:\n",
            "\n",
            "karibu kaia natanazo yeti ya subuia idha ya kiswahili \n",
            "\n",
            "\n",
            "------------------------------------------------ 1 ------------------------------------------------\n",
            "True transcription:\n",
            "\n",
            "kwenda na kuondoka jarkata indonesia\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Predicted transcription:\n",
            "\n",
            "kuwenda kuna kundoka jijya kata indoneziaoanfuyuhyhya b\n",
            "\n",
            "\n",
            "------------------------------------------------ 2 ------------------------------------------------\n",
            "True transcription:\n",
            "\n",
            "lazima uwe na outside support mataifa ya nje yakuunge mkono\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Predicted transcription:\n",
            "\n",
            "nazimago na hausaiu sako matafa yania ya kuingie mko\n",
            "\n",
            "\n",
            "------------------------------------------------ 3 ------------------------------------------------\n",
            "True transcription:\n",
            "\n",
            "nne lazima uwe na military support eeh\n",
            "----------------------------------------------------------------------------------------------------\n",
            "Predicted transcription:\n",
            "\n",
            "nie amzmau na mbili tane sapouteafufuyhyab\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}