Chatbot_Aksara-Jawa_using_LSTM.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "6xoKYBBO6xaV"
      },
      "source": [
        "# **Chatbot Java writing using Seq2Seq LSTM models**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "2SOmE99B7Is0"
      },
      "source": [
        "This project is to create conversational chatbot using Sequence to sequence LSTM models. \n",
        "Sequence to sequence learning is about training models to convert from one domain to sequences another domain. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mVuZTAV08qWY"
      },
      "source": [
        "# Import all the packages "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#Preprocessing Data\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import re\n",
        "from nltk.stem.porter import PorterStemmer \n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "#Extract Csv\n",
        "import os\n",
        "import csv\n",
        "\n",
        "#NLP\n",
        "import tensorflow as tf\n",
        "import pickle\n",
        "from tensorflow.keras import layers, activations, models, utils, preprocessing\n",
        "\n",
        "from gensim.models import Word2Vec\n",
        "import re"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "l4kJp6uO-fQE"
      },
      "source": [
        "# Preprocessing the data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Create List for Question and Answers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "dataframe = pd.read_csv(\"Corpus 2 Terbaru Gabungan - Sheet1.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def text_clean(corpus):\n",
        "    '''\n",
        "    Purpose : Function to keep only alphabets, digits and certain words (punctuations, qmarks, tabs etc. removed)\n",
        "    \n",
        "    Input : Takes a text corpus, 'corpus' to be cleaned along with a list of words, 'keep_list', which have to be retained\n",
        "            even after the cleaning process\n",
        "    \n",
        "    Output : Returns the cleaned text corpus\n",
        "    \n",
        "    '''\n",
        "    cleaned_corpus = pd.Series()\n",
        "    for row in corpus:\n",
        "        qs = []\n",
        "        for word in row.split():\n",
        "            p1 = re.sub(pattern='[^a-zA-Z]',repl=' ',string=word)\n",
        "            p1 = p1.lower()\n",
        "            qs.append(p1)\n",
        "        cleaned_corpus = cleaned_corpus.append(pd.Series(' '.join(qs)))\n",
        "    return cleaned_corpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def stopwords_removal(corpus):\n",
        "    wh_words = ['who', 'what', 'when', 'why', 'how', 'which', 'where', 'whom']\n",
        "    stop = set(stopwords.words('english'))\n",
        "    for word in wh_words:\n",
        "        stop.remove(word)\n",
        "    corpus = [[x for x in x.split() if x not in stop] for x in corpus]\n",
        "    return corpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def lemmatize(corpus):\n",
        "    lem = WordNetLemmatizer()\n",
        "    corpus = [[lem.lemmatize(x, pos = 'v') for x in x] for x in corpus]\n",
        "    return corpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def stem(corpus, stem_type = None):\n",
        "    if stem_type == 'snowball':\n",
        "        stemmer = SnowballStemmer(language = 'english')\n",
        "        corpus = [[stemmer.stem(x) for x in x] for x in corpus]\n",
        "    else :\n",
        "        stemmer = PorterStemmer()\n",
        "        corpus = [[stemmer.stem(x) for x in x] for x in corpus]\n",
        "    return corpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def preprocess(corpus, cleaning = False, stemming = False, stem_type = None, lemmatization = False, remove_stopwords = False):\n",
        "    \n",
        "    '''\n",
        "    Purpose : Function to perform all pre-processing tasks (cleaning, stemming, lemmatization, stopwords removal etc.)\n",
        "    \n",
        "    Input : \n",
        "    'corpus' - Text corpus on which pre-processing tasks will be performed\n",
        "    \n",
        "    'cleaning', 'stemming', 'lemmatization', 'remove_stopwords' - Boolean variables indicating whether a particular task should \n",
        "                                                                  be performed or not\n",
        "    'stem_type' - Choose between Porter stemmer or Snowball(Porter2) stemmer. Default is \"None\", which corresponds to Porter\n",
        "                  Stemmer. 'snowball' corresponds to Snowball Stemmer\n",
        "    \n",
        "    Note : Either stemming or lemmatization should be used. There's no benefit of using both of them together\n",
        "    \n",
        "    Output : Returns the processed text corpus\n",
        "    \n",
        "    '''\n",
        "    if cleaning == True:\n",
        "        corpus = text_clean(corpus)\n",
        "    \n",
        "    if remove_stopwords == True:\n",
        "        corpus = stopwords_removal(corpus)\n",
        "    else :\n",
        "        corpus = [[x for x in x.split()] for x in corpus]\n",
        "    \n",
        "    if lemmatization == True:\n",
        "        corpus = lemmatize(corpus)\n",
        "        \n",
        "        \n",
        "    if stemming == True:\n",
        "        corpus = stem(corpus, stem_type)\n",
        "    \n",
        "    corpus = [' '.join(x) for x in corpus]\n",
        "        \n",
        "\n",
        "    return corpus"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#Fast 15x cleaning row\n",
        "slice = set(range(dataframe.shape[0])) - set([0,1001,1002]) #set(range[500-750]) for specific target\n",
        "#dataframe2 = dataframe.take(list(slice))\n",
        "\n",
        "DcolQ = [0,2,3,4,5,6,7,8] # 0 = Indo Question\n",
        "dfQ = dataframe.drop(dataframe.columns[DcolQ], axis=1).take(list(slice))\n",
        "\n",
        "DcolA = [0,1,2,3,4,6,7,8] # 5 = Jawa Answer\n",
        "dfA = dataframe.drop(dataframe.columns[DcolA], axis=1).take(list(slice))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "dfC = dfA.rename(columns={'Unnamed: 5':'Answer'}) #Unnamed -> Answer because the columns name is empty\n",
        "\n",
        "corpusQ = pd.Series(dfQ.INDONESIA.tolist()).astype(str)\n",
        "corpusA = pd.Series(dfC.Answer.tolist()).astype(str)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "Quest = preprocess(corpusQ, lemmatization=True, stemming=False, remove_stopwords=True, cleaning= False)\n",
        "Answ = preprocess(corpusA, lemmatization=True, stemming=False, remove_stopwords=True, cleaning= False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Start Here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "-bRvbQ00Coy5",
        "outputId": "ef129a1c-3071-4d10-e6b2-7ac5a6ff2bac"
      },
      "outputs": [],
      "source": [
        "questions = Quest\n",
        "answers = Answ\n",
        "\n",
        "answers_with_tags = list()\n",
        "for i in range( len( answers ) ):\n",
        "    if type( answers[i] ) == str:\n",
        "        answers_with_tags.append( answers[i] )\n",
        "    else:\n",
        "        questions.pop( i )\n",
        "\n",
        "answers = list()\n",
        "for i in range( len( answers_with_tags ) ) :\n",
        "    answers.append( '<START> ' + answers_with_tags[i] + ' <END>' )\n",
        "\n",
        "tokenizer = preprocessing.text.Tokenizer()\n",
        "tokenizer.fit_on_texts( questions + answers )\n",
        "VOCAB_SIZE = len( tokenizer.word_index )+1\n",
        "print( 'VOCAB SIZE : {}'.format( VOCAB_SIZE ))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "WMPqb8LxIeGI"
      },
      "source": [
        "### b) Preparing data for Seq2Seq model\n",
        "\n",
        "This model requires 3 arrays encoder_input_data, decoder_input_data and decoder_output_data.\n",
        "\n",
        "For encoder_input_data:\n",
        "Tokensize the Questions and Pad them to their maximum Length.\n",
        "\n",
        "For decoder_input_data:\n",
        "Tokensize the Answers and Pad them to their maximum Length.\n",
        "\n",
        "For decoder_output_data:\n",
        "Tokensize the Answers and Remove the 1st element from all the tokenized_answers. This is the <START> element which was added earlier."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "QqYoDsbSCo4f"
      },
      "outputs": [],
      "source": [
        "vocab = []\n",
        "for word in tokenizer.word_index:\n",
        "  vocab.append(word)\n",
        "\n",
        "def tokenize(sentences):\n",
        "  tokens_list = []\n",
        "  vocabulary = []\n",
        "  for sentence in sentences:\n",
        "    sentence = sentence.lower()\n",
        "    sentence = re.sub('[^a-zA-Z]', ' ', sentence)\n",
        "    tokens = sentence.split()\n",
        "    vocabulary += tokens\n",
        "    tokens_list.append(tokens)\n",
        "  return tokens_list, vocabulary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "9vKhieIwCo7J",
        "outputId": "e97b4a74-7384-478c-d4ae-082513257107"
      },
      "outputs": [],
      "source": [
        "#encoder_input_data\n",
        "tokenized_questions = tokenizer.texts_to_sequences( questions )\n",
        "maxlen_questions = max( [len(x) for x in tokenized_questions ] )\n",
        "padded_questions = preprocessing.sequence.pad_sequences( tokenized_questions, maxlen = maxlen_questions, padding = 'post')\n",
        "encoder_input_data = np.array(padded_questions)\n",
        "print(encoder_input_data.shape, maxlen_questions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "AJo7WPjLCo-q",
        "outputId": "28b5e209-5389-4313-f3cd-f5451fa8c519"
      },
      "outputs": [],
      "source": [
        "# decoder_input_data\n",
        "tokenized_answers = tokenizer.texts_to_sequences( answers )\n",
        "maxlen_answers = max( [ len(x) for x in tokenized_answers ] )\n",
        "padded_answers = preprocessing.sequence.pad_sequences( tokenized_answers , maxlen=maxlen_answers , padding='post' )\n",
        "decoder_input_data = np.array( padded_answers )\n",
        "print( decoder_input_data.shape , maxlen_answers )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "colab_type": "code",
        "id": "ccY0wWdRCpCa",
        "outputId": "07877cda-7e07-42b2-bb7e-772d118a3cf0"
      },
      "outputs": [],
      "source": [
        "# decoder_output_data\n",
        "tokenized_answers = tokenizer.texts_to_sequences( answers )\n",
        "for i in range(len(tokenized_answers)) :\n",
        "    tokenized_answers[i] = tokenized_answers[i][1:]\n",
        "padded_answers = preprocessing.sequence.pad_sequences( tokenized_answers , maxlen=maxlen_answers , padding='post' )\n",
        "onehot_answers = utils.to_categorical( padded_answers , VOCAB_SIZE )\n",
        "decoder_output_data = np.array( onehot_answers )\n",
        "print( decoder_output_data.shape )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "-D53pyucPCnk"
      },
      "source": [
        "# Defining Encoder Decoder Model\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 440
        },
        "colab_type": "code",
        "id": "W3YjCFDwPRVN",
        "outputId": "7bc112a0-6945-4100-e8d9-3bc5691797e5"
      },
      "outputs": [],
      "source": [
        "encoder_inputs = tf.keras.layers.Input(shape=( maxlen_questions , ))\n",
        "encoder_embedding = tf.keras.layers.Embedding( VOCAB_SIZE, 200 , mask_zero=True ) (encoder_inputs)\n",
        "encoder_outputs , state_h , state_c = tf.keras.layers.LSTM( 200 , return_state=True )( encoder_embedding )\n",
        "encoder_states = [ state_h , state_c ]\n",
        "\n",
        "decoder_inputs = tf.keras.layers.Input(shape=( maxlen_answers ,  ))\n",
        "decoder_embedding = tf.keras.layers.Embedding( VOCAB_SIZE, 200 , mask_zero=True) (decoder_inputs)\n",
        "decoder_lstm = tf.keras.layers.LSTM( 200 , return_state=True , return_sequences=True )\n",
        "decoder_outputs , _ , _ = decoder_lstm ( decoder_embedding , initial_state=encoder_states )\n",
        "decoder_dense = tf.keras.layers.Dense( VOCAB_SIZE , activation=tf.keras.activations.softmax ) \n",
        "output = decoder_dense ( decoder_outputs )\n",
        "\n",
        "model = tf.keras.models.Model([encoder_inputs, decoder_inputs], output )\n",
        "model.compile(optimizer=tf.keras.optimizers.RMSprop(), loss='categorical_crossentropy')\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wVfSormAPb3w"
      },
      "source": [
        "# Training the Model\n",
        "\n",
        "We train the model for a number of epochs with RMSprop optimizer and categorical_crossentropy loss function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "colab_type": "code",
        "id": "OHlqQq64PYTH",
        "outputId": "c477c08f-55e4-41ac-b8b7-1973aa38f48a"
      },
      "outputs": [],
      "source": [
        "model.fit([encoder_input_data , decoder_input_data], decoder_output_data, batch_size=50, epochs=300 ) #epoch 200/loss 0.03"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "F1MIy1j9aVTo"
      },
      "source": [
        "# Defining Inference Models\n",
        "\n",
        "Encoder Inference Model: Takes questions as input and outputs LSTM states (h and c)\n",
        "\n",
        "Decoder Inference Model: Takes in 2 inputs one are the LSTM states, second are the answer input sequences. it will o/p the answers for questions which fed to the encoder model and it's state values."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "MpLowS27cn8X"
      },
      "outputs": [],
      "source": [
        "def make_inference_models():\n",
        "    \n",
        "    encoder_model = tf.keras.models.Model(encoder_inputs, encoder_states)\n",
        "    \n",
        "    decoder_state_input_h = tf.keras.layers.Input(shape=( 200 ,))\n",
        "    decoder_state_input_c = tf.keras.layers.Input(shape=( 200 ,))\n",
        "    \n",
        "    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
        "    \n",
        "    decoder_outputs, state_h, state_c = decoder_lstm(\n",
        "        decoder_embedding , initial_state=decoder_states_inputs)\n",
        "    \n",
        "    decoder_states = [state_h, state_c]\n",
        "\n",
        "    decoder_outputs = decoder_dense(decoder_outputs)\n",
        "    \n",
        "    decoder_model = tf.keras.models.Model(\n",
        "        [decoder_inputs] + decoder_states_inputs,\n",
        "        [decoder_outputs] + decoder_states)\n",
        "    \n",
        "    return encoder_model , decoder_model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "EwoYVsBTeYra"
      },
      "source": [
        "# Talking with the Chatbot\n",
        "\n",
        "define a method str_to_tokens which converts str questions to Integer tokens with padding.\n",
        "\n",
        "1. First, we take a question as input and predict the state values using enc_model.\n",
        "2. We set the state values in the decoder's LSTM.\n",
        "3. Then, we generate a sequence which contains the <start> element.\n",
        "4. We input this sequence in the dec_model.\n",
        "5. We replace the <start> element with the element which was predicted by the dec_model and update the state values.\n",
        "6. We carry out the above steps iteratively till we hit the <end> tag or the maximum answer length.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "oA7Yx45Li3wo"
      },
      "outputs": [],
      "source": [
        "def str_to_tokens( sentence : str ):\n",
        "\n",
        "    words = sentence.lower().split()\n",
        "    tokens_list = list()\n",
        "  \n",
        "    for word in words:\n",
        "        tokens_list.append( tokenizer.word_index[ word ] ) \n",
        "    return preprocessing.sequence.pad_sequences( [tokens_list] , maxlen=maxlen_questions , padding='post')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#Java Writing function\n",
        "#!/usr/bin/env python\n",
        "#-*- coding: utf-8 -*-\n",
        "\n",
        "\"\"\"\n",
        "__version__     =   \"0.0.1\"\n",
        "__author__      =   \"@lantip\"\n",
        "__date__        =   \"2019/04/02\"\n",
        "__description__ =   \"Latin to Javanese Transliterator\"\n",
        "\"\"\" \n",
        "\n",
        "import sys\n",
        "\n",
        "HURUF = {\n",
        "    'h': 'ꦲ',\n",
        "    'n': 'ꦤ',\n",
        "    'c': 'ꦕ',\n",
        "    'r': 'ꦫ',\n",
        "    'k': 'ꦏ',\n",
        "    'd': 'ꦢ',\n",
        "    't': 'ꦠ',\n",
        "    's': 'ꦱ',\n",
        "    'w': 'ꦮ',\n",
        "    'l': 'ꦭ',\n",
        "    'p': 'ꦥ',\n",
        "    'dh': 'ꦝ',\n",
        "    'j': 'ꦗ',\n",
        "    'y': 'ꦪ',\n",
        "    'ny': 'ꦚ',\n",
        "    'm': 'ꦩ',\n",
        "    'g': 'ꦒ',\n",
        "    'b': 'ꦧ',\n",
        "    'th': 'ꦛ',\n",
        "    'ng': 'ꦔ',\n",
        "    ',': '꧈',\n",
        "    '.': '꧉'\n",
        "}\n",
        "\n",
        "PASANGAN = {\n",
        "    'h': '꧀ꦲ',\n",
        "    'n': '꧀ꦤ',\n",
        "    'c': '꧀ꦕ',\n",
        "    'r': '꧀ꦫ',\n",
        "    'k': '꧀ꦏ',\n",
        "    'd': '꧀ꦢ',\n",
        "    't': '꧀ꦠ',\n",
        "    's': '꧀ꦱ',\n",
        "    'w': '꧀ꦮ',\n",
        "    'l': '꧀ꦭ',\n",
        "    'p': '꧀ꦥ',\n",
        "    'dh': '꧀ꦓ',\n",
        "    'j': '꧀ꦗ',\n",
        "    'y': '꧀ꦪ',\n",
        "    'ny': '꧀ꦚ',\n",
        "    'm': '꧀ꦩ',\n",
        "    'g': '꧀ꦒ',\n",
        "    'b': '꧀ꦧ',\n",
        "    'th': '꧀ꦛ',\n",
        "    'ng': '꧀ꦔ'\n",
        "}\n",
        "\n",
        "SANDHANGAN = {\n",
        "    'wulu': 'ꦶ',\n",
        "    'suku': 'ꦸ',\n",
        "    'pepet': 'ꦼ',\n",
        "    'taling': 'ꦺ',\n",
        "    'taling-tarung': 'ꦺꦴ',\n",
        "    'cecak': 'ꦁ',\n",
        "    'wignyan': 'ꦃ',\n",
        "    'layar': 'ꦂ',\n",
        "    'cakra': 'ꦿ',\n",
        "    'keret': 'ꦽ',\n",
        "    'pengkal': 'ꦾ',\n",
        "    'pangkon': '꧀'\n",
        "}\n",
        "\n",
        "def transliterate(hrf, isend, prv, nxt):\n",
        "    ltr = ''\n",
        "    dobel = ['th', 'dh', 'ny']\n",
        "    iskeret = False\n",
        "    if hrf.find('ng') == 0:\n",
        "        if len(hrf) == 2:\n",
        "            ltr += SANDHANGAN['cecak']\n",
        "        else:\n",
        "            ltr += HURUF['ng']\n",
        "        if len(hrf) > 3:\n",
        "            if hrf[2] == 'l':\n",
        "                ltr += PASANGAN['l']\n",
        "            elif hrf[2] == 'y':\n",
        "                ltr += SANDHANGAN['pengkal']\n",
        "            elif hrf[2] == 'r':\n",
        "                if hrf[3] == 'e':\n",
        "                    ltr += SANDHANGAN['keret']\n",
        "                    iskeret = True\n",
        "                else:\n",
        "                    ltr += SANDHANGAN['cakra']\n",
        "    elif hrf.find('ny') == 0:\n",
        "        if prv:\n",
        "            if len(prv) == 1:\n",
        "                ltr += PASANGAN['ny']\n",
        "            else:\n",
        "                ltr += HURUF['ny']\n",
        "        else:\n",
        "            ltr += HURUF['ny']\n",
        "        if len(hrf) > 3:\n",
        "            if hrf[2] == 'l':\n",
        "                ltr += PASANGAN['l']\n",
        "            elif hrf[2] == 'y':\n",
        "                ltr += SANDHANGAN['pengkal']\n",
        "            elif hrf[2] == 'r':\n",
        "                if hrf[3] == 'e':\n",
        "                    ltr += SANDHANGAN['keret']\n",
        "                    iskeret = True\n",
        "                else:\n",
        "                    ltr += SANDHANGAN['cakra']\n",
        "    elif hrf.find('th') == 0:\n",
        "        if prv:\n",
        "            if len(prv) == 1:\n",
        "                ltr += PASANGAN['th']\n",
        "            else:\n",
        "                ltr += HURUF['th']\n",
        "        else:\n",
        "            ltr += HURUF['th']\n",
        "        if len(hrf) > 3:\n",
        "            if hrf[2] == 'l':\n",
        "                ltr += PASANGAN['l']\n",
        "            elif hrf[2] == 'y':\n",
        "                ltr += SANDHANGAN['pengkal']\n",
        "            elif hrf[2] == 'r':\n",
        "                if hrf[3] == 'e':\n",
        "                    ltr += SANDHANGAN['keret']\n",
        "                    iskeret = True\n",
        "                else:\n",
        "                    ltr += SANDHANGAN['cakra']\n",
        "    elif hrf.find('dh') == 0:\n",
        "        if prv:\n",
        "            if len(prv) == 1:\n",
        "                ltr += PASANGAN['dh']\n",
        "            else:\n",
        "                ltr += HURUF['dh']\n",
        "        else:\n",
        "            ltr += HURUF['dh']\n",
        "        if len(hrf) > 3:\n",
        "            if hrf[2] == 'l':\n",
        "                ltr += PASANGAN['l']\n",
        "            elif hrf[2] == 'y':\n",
        "                ltr += SANDHANGAN['pengkal']\n",
        "            elif hrf[2] == 'r':\n",
        "                if hrf[4] == 'e':\n",
        "                    ltr += SANDHANGAN['keret']\n",
        "                    iskeret = True\n",
        "                else:\n",
        "                    ltr += SANDHANGAN['cakra']\n",
        "    if len(hrf) == 2:\n",
        "        if hrf == 'ng':\n",
        "            ltr += SANDHANGAN['cecak']\n",
        "        else:\n",
        "            if prv:\n",
        "                if len(prv) == 1:\n",
        "                    if prv not in  ['h', 'r', 'y']:\n",
        "                        ltr += PASANGAN[hrf[0]]\n",
        "                    else:\n",
        "                        ltr += HURUF[hrf[0]]\n",
        "                else:\n",
        "                    ltr += HURUF[hrf[0]]\n",
        "            else:\n",
        "                ltr += HURUF[hrf[0]]\n",
        "    elif len(hrf) == 1:\n",
        "        if hrf[0] not in [',', '.']:\n",
        "            if hrf == 'r':\n",
        "                ltr += SANDHANGAN['layar']\n",
        "            elif hrf == 'h':\n",
        "                ltr += SANDHANGAN['wignyan']\n",
        "            elif hrf == ',':\n",
        "                pass\n",
        "            else:\n",
        "                if isend:\n",
        "                    ltr += HURUF[hrf[0]]\n",
        "                    ltr += SANDHANGAN['pangkon']\n",
        "                else:\n",
        "                    ltr += HURUF[hrf[0]]\n",
        "\n",
        "    elif len(hrf) > 2:\n",
        "        if hrf[1] == 'l':\n",
        "            ltr += HURUF[hrf[0]]\n",
        "            ltr += PASANGAN['l']\n",
        "        elif hrf[1] == 'y' and hrf[0] != 'n':\n",
        "            ltr += HURUF[hrf[0]]\n",
        "            ltr += SANDHANGAN['pengkal']\n",
        "        elif hrf[1] == 'r':\n",
        "            if prv:\n",
        "                if len(prv) == 1:\n",
        "                    if prv not in  ['h', 'r', 'y']:\n",
        "                        ltr += PASANGAN[hrf[0]]\n",
        "                        ltr += SANDHANGAN['cakra']\n",
        "                    else:\n",
        "                        ltr += HURUF[hrf[0]]    \n",
        "                        ltr += SANDHANGAN['cakra']\n",
        "                else:\n",
        "                    ltr += HURUF[hrf[0]]\n",
        "                    ltr += SANDHANGAN['cakra']\n",
        "            else:\n",
        "                ltr += HURUF[hrf[0]]\n",
        "                ltr += SANDHANGAN['cakra']\n",
        "    if hrf.find('u') == (len(hrf) - 1):\n",
        "        ltr += SANDHANGAN['suku']\n",
        "    \n",
        "    if 'é' in hrf or 'è' in hrf:\n",
        "        if prv:\n",
        "            ltr += SANDHANGAN['taling']\n",
        "        else:\n",
        "            ltr += SANDHANGAN['taling']\n",
        "    if hrf.find('e') == (len(hrf) - 1):\n",
        "        if not iskeret:\n",
        "            ltr += SANDHANGAN['pepet']\n",
        "    if hrf.find('i') == (len(hrf) - 1):\n",
        "        ltr += SANDHANGAN['wulu']\n",
        "    if 'o' in hrf:\n",
        "        ltr += SANDHANGAN['taling-tarung']\n",
        "    if nxt == '.':\n",
        "        ltr += HURUF[nxt]\n",
        "    return ltr\n",
        "\n",
        "\n",
        "def translate(word):\n",
        "    ltr = []\n",
        "    start = 0\n",
        "    consonant = ['c','k','s','w','l','p','j','m','b']\n",
        "    specials = ['t','d']\n",
        "    dobel = ['th', 'dh', 'ny', 'ng']\n",
        "    insrt = [ 'h','y','g','n']\n",
        "    vowels = \"AaEeÈèÉéIiOoUuÊêĚěĔĕṚṛXxôâāīūō\"\n",
        "    for dob in dobel:\n",
        "        if word.find(dob) == 0:\n",
        "            if len(word) >= 3:\n",
        "                if word[2] in vowels:\n",
        "                    ltr.append(dob+word[2])\n",
        "                    start = 3\n",
        "            elif len(word) >= 4:\n",
        "                if word[2] == 'r':\n",
        "                    if word[3] in vowels:\n",
        "                        ltr.append(dob+'r'+word[3])\n",
        "                        start = 4\n",
        "    for ins in insrt:\n",
        "        if word.find(ins) == 0:\n",
        "            if len(word) >=2:\n",
        "                if word[1] in vowels:\n",
        "                    ltr.append(ins+word[1])\n",
        "                    start = 2\n",
        "                elif word[1] in ['l', 'r', 'y']:\n",
        "                    if word[2] in vowels:\n",
        "                        ltr.append(ins+word[1]+word[2])\n",
        "                        start = 3\n",
        "    if word[0] in vowels:\n",
        "        ltr.append('h'+word[0])\n",
        "        start = 1\n",
        "    for i in range(start,len(word)):\n",
        "        if word[i] in consonant:\n",
        "            try:\n",
        "                if len(word[i:]) > 1:\n",
        "                    if word[i+1] in vowels and word[i] != 'l':\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "                    else:\n",
        "                        if word[i+1] in ['l', 'r','y']:\n",
        "                            if len(word[i:]) > 2:\n",
        "                                if word[i+2] in vowels:\n",
        "                                    ltr.append(word[i]+word[i+1]+word[i+2])\n",
        "                                    i = i + 3\n",
        "                                else:\n",
        "                                    ltr.append(word[i]+word[i+1])\n",
        "                                    i = i + 2\n",
        "                            else:\n",
        "                                if (i-2) >= 0:\n",
        "                                    if len(word[i:]) > 1:\n",
        "                                        if word[i] not in word[i-2]+word[i-1]:\n",
        "                                            ltr.append(word[i]+word[i+1])\n",
        "                                            i = i + 2\n",
        "                        else:\n",
        "                            if word[i] != 'l':\n",
        "                                ltr.append(word[i])\n",
        "                                i = i + 1\n",
        "                            else:\n",
        "                                if len(word[i:]) > 1:\n",
        "                                    if word[i+1] in vowels:\n",
        "                                        if len(ltr) > 0:\n",
        "                                            if not word[i]+word[i+1] in ltr[len(ltr)-1]:\n",
        "                                                ltr.append(word[i]+word[i+1])\n",
        "                                                i = i + 2\n",
        "                                        else:\n",
        "                                            ltr.append(word[i]+word[i+1])\n",
        "                                            i = i + 2\n",
        "                else:\n",
        "                    ltr.append(word[i])\n",
        "                    i = i + 1\n",
        "            except:\n",
        "                ltr.append(word[i])\n",
        "                i = i + 1\n",
        "        elif word[i] in specials:\n",
        "            try:\n",
        "                if len(word[i:]) >=2:\n",
        "                    if word[i+1] == 'h' and word[i+2] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1]+word[i+2])\n",
        "                        i = i + 3\n",
        "                    elif word[i+1] in ['l', 'r']:\n",
        "                        if len(word[i:]) > 2:\n",
        "                            if word[i+2] in vowels:\n",
        "                                ltr.append(word[i]+word[i+1]+word[i+2])\n",
        "                                i = i + 3\n",
        "                            else:\n",
        "                                ltr.append(word[i]+word[i+1])\n",
        "                                i = i + 2\n",
        "                        else:\n",
        "                            ltr.append(word[i]+word[i+1])\n",
        "                            i = i + 2\n",
        "                    elif word[i+1] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "                elif len(word[i:]) == 1:\n",
        "                    if word[i+1] == 'h':\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "                    elif word[i+1] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "            except:\n",
        "                ltr.append(word[i])\n",
        "                i = i + 1\n",
        "        elif word[i] == 'n':\n",
        "            if len(word[i:]) > 2:\n",
        "                if word[i+1] in ['g','y'] and word[i+2] in vowels:\n",
        "                    ltr.append(word[i]+word[i+1]+word[i+2])\n",
        "                    i = i + 3\n",
        "                elif word[i+1] in ['g','y'] and word[i+2] not in vowels:\n",
        "                    ltr.append(word[i]+word[i+1])\n",
        "                    i = i + 2\n",
        "                else:\n",
        "                    if word[i+1] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "                    else:\n",
        "                        ltr.append(word[i])\n",
        "                        i = i + 1\n",
        "            else:\n",
        "                try:\n",
        "                    nxt = word[i+1]\n",
        "                except:\n",
        "                    nxt = None\n",
        "                if nxt:\n",
        "                    if nxt in vowels:\n",
        "                        ltr.append(word[i]+nxt)\n",
        "                        i = i + 2\n",
        "                    elif nxt == 'g':\n",
        "                        ltr.append(word[i]+nxt)\n",
        "                        i = i + 2\n",
        "                    else:\n",
        "                        ltr.append(word[i])\n",
        "                        i = i + 1\n",
        "                else:\n",
        "                    ltr.append(word[i])\n",
        "                    i = i + 1\n",
        "        elif word[i] in ['r','y']:\n",
        "            if i == 0:\n",
        "                if len(word[i:]) > 1:\n",
        "                    if word[i+1] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "            else:\n",
        "                if len(word[i:]) > 1:\n",
        "                    if word[i+1] in vowels:\n",
        "                        if word[i-1] not in vowels:\n",
        "                            if (i-2) >=0:\n",
        "                                if (word[i-2]+word[i-1]) in dobel:\n",
        "                                    ltr.append(word[i-2]+word[i-1]+word[i]+word[i+1])\n",
        "                                    i = i + 2\n",
        "                                else:        \n",
        "                                    if not (word[i]+word[i+1]) in ltr[len(ltr)-1]:\n",
        "                                        ltr[len(ltr)-1] = ltr[len(ltr)-1] + word[i] + word[i+1]\n",
        "                                        i = i + 1\n",
        "                        else:\n",
        "                            ltr.append(word[i]+word[i+1])\n",
        "                            i = i + 2\n",
        "                    else:\n",
        "                        ltr.append(word[i])\n",
        "                        i = i + 1\n",
        "                else:\n",
        "                    ltr.append(word[i])\n",
        "                    i = i + 1\n",
        "        elif word[i] == 'g':\n",
        "            if 'g' in ltr[len(ltr)-1] and len(ltr[len(ltr)-1]) >= 2:\n",
        "                pass\n",
        "            else:\n",
        "                if len(word[i:]) > 1:\n",
        "                    if word[i+1] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "                    else:\n",
        "                        if (i-2) > 0:\n",
        "                            if (word[i-2]+word[i-1]) == 'ng':\n",
        "                                pass\n",
        "                        else:\n",
        "                            if ltr[len(ltr) - 1] != 'ng':\n",
        "                                ltr.append(word[i])\n",
        "                                i = i + 1\n",
        "                            else:\n",
        "                                i = i + 1\n",
        "                else:\n",
        "                    if (i-2) > 0:\n",
        "                        if (word[i-2] + word[i-1]) == 'ng':\n",
        "                            pass\n",
        "                        elif (word[i-1] + word[i]) == 'ng':\n",
        "                            pass\n",
        "                        else:\n",
        "                            ltr.append(word[i])\n",
        "                            i = i + 1\n",
        "                    else:\n",
        "                        ltr.append(word[i])\n",
        "                        i = i + 1\n",
        "        elif word[i] == 'h':\n",
        "            if 'h' in ltr[len(ltr)-1] and len(ltr[len(ltr)-1]) > 2:\n",
        "                pass\n",
        "            else:\n",
        "                if len(word[i:]) > 1:\n",
        "                    if word[i+1] in vowels:\n",
        "                        ltr.append(word[i]+word[i+1])\n",
        "                        i = i + 2\n",
        "                    else:\n",
        "                        ltr.append(word[i])\n",
        "                        i = i + 1\n",
        "                else:\n",
        "                    ltr.append(word[i])\n",
        "                    i = i + 1\n",
        "    return ltr\n",
        "\n",
        "\n",
        "def translatethis(text):\n",
        "    if ',' in text:\n",
        "        trslt = translate(text.replace(',','')) + [',']\n",
        "    elif '.' in text:\n",
        "        trslt = translate(text.replace(',','')) + ['.']\n",
        "    else:\n",
        "        trslt = translate(text)\n",
        "    return trslt\n",
        "\n",
        "def dotranslate(word):\n",
        "    trslt = []\n",
        "    for wrds in word.split():\n",
        "        if '-' in wrds:\n",
        "            for wrd in wrds.split('-'):\n",
        "                trslt = trslt + translatethis(wrd.lower())\n",
        "        else:\n",
        "            trslt = trslt + translatethis(wrds.lower())\n",
        "\n",
        "    return trslt\n",
        "\n",
        "\n",
        "def dotransliterate(word):\n",
        "    litr = ''\n",
        "    if '.' in word:\n",
        "        for ijk,wrd in enumerate(word.split('.')):\n",
        "            if ',' in wrd:\n",
        "                for idx,wr in enumerate(wrd.split(',')):\n",
        "                    ltr = dotranslate(wr)\n",
        "                    isend = False\n",
        "                    for index, lt in enumerate(ltr):\n",
        "                        if index == len(ltr) - 1:\n",
        "                            isend = True\n",
        "                            nxt = None\n",
        "                        else:\n",
        "                            nxt = ltr[index+1]\n",
        "                        if (index - 1) >= 0:\n",
        "                            prv = ltr[index-1]\n",
        "                        else:\n",
        "                            prv = None\n",
        "\n",
        "                        litr += transliterate(lt, isend, prv, nxt)\n",
        "                    if idx < (len(wrd.split(',')) - 1):\n",
        "                        litr += HURUF[',']\n",
        "            else:\n",
        "                ltr = dotranslate(wrd)\n",
        "                isend = False\n",
        "                for index, lt in enumerate(ltr):\n",
        "                    if index == len(ltr) - 1:\n",
        "                        isend = True\n",
        "                        nxt = None\n",
        "                    else:\n",
        "                        nxt = ltr[index+1]\n",
        "                    if (index - 1) >= 0:\n",
        "                        prv = ltr[index-1]\n",
        "                    else:\n",
        "                        prv = None\n",
        "\n",
        "                    litr += transliterate(lt, isend, prv, nxt)    \n",
        "            if ijk  < (len(word.split('.')) - 1):      \n",
        "                litr += HURUF['.']\n",
        "    else:\n",
        "        ltr = dotranslate(word)\n",
        "        isend = False\n",
        "        for index, lt in enumerate(ltr):\n",
        "            if index == len(ltr) - 1:\n",
        "                isend = True\n",
        "                nxt = None\n",
        "            else:\n",
        "                nxt = ltr[index+1]\n",
        "            if (index - 1) >= 0:\n",
        "                prv = ltr[index-1]\n",
        "            else:\n",
        "                prv = None\n",
        "\n",
        "            litr += transliterate(lt, isend, prv, nxt)\n",
        "    #print(ltr) show per huruf\n",
        "    return litr"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def text_to_javanese(text):\n",
        "    converted_text = ''\n",
        "    for char in text:\n",
        "            converted_text = [dotransliterate(text).lower()]\n",
        "    return converted_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 650
        },
        "colab_type": "code",
        "id": "eUr4SQDveVb0",
        "outputId": "abafd831-63b1-4eee-e35a-323ce20e9fba"
      },
      "outputs": [],
      "source": [
        "enc_model , dec_model = make_inference_models()\n",
        "\n",
        "for _ in range(10):\n",
        "    states_values = enc_model.predict( str_to_tokens( input( 'Enter question : ' ) ) )\n",
        "    empty_target_seq = np.zeros( ( 1 , 1 ) )\n",
        "    empty_target_seq[0, 0] = tokenizer.word_index['start']\n",
        "    stop_condition = False\n",
        "    decoded_translation = ''\n",
        "    while not stop_condition :\n",
        "        dec_outputs , h , c = dec_model.predict([ empty_target_seq ] + states_values )\n",
        "        sampled_word_index = np.argmax( dec_outputs[0, -1, :] )\n",
        "        sampled_word = None\n",
        "        for word , index in tokenizer.word_index.items() :\n",
        "            if sampled_word_index == index :\n",
        "                decoded_translation += ' {}'.format( word )\n",
        "                sampled_word = word\n",
        "        \n",
        "        if sampled_word == 'end' or len(decoded_translation.split()) > maxlen_answers:\n",
        "            stop_condition = True\n",
        "            \n",
        "        empty_target_seq = np.zeros( ( 1 , 1 ) )  \n",
        "        empty_target_seq[ 0 , 0 ] = sampled_word_index\n",
        "        states_values = [ h , c ] \n",
        "\n",
        "    output_aksara_jawa = text_to_javanese(decoded_translation)\n",
        "\n",
        "    print(output_aksara_jawa)\n",
        "    print(decoded_translation)\n",
        "    \n",
        "\n",
        "\"\"\"\n",
        "if __name__ == '__main__':\n",
        "    jawa = input('> ')\n",
        "    while (jawa != 'quit'):\n",
        "        try:\n",
        "            print (dotransliterate(jawa).lower())\n",
        "            question = input('> ')\n",
        "        except:\n",
        "            sys.exit(1)\n",
        "\"\"\""
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "Chatbot_using_LSTM.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.18"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}