seq2seq.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from __future__ import unicode_literals, print_function, division\n",
        "from io import open\n",
        "import unicodedata\n",
        "import string\n",
        "import re\n",
        "import random\n",
        "import sys\n",
        "import os\n",
        "sys.path.insert(0, os.path.abspath('..'))\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch import optim\n",
        "import torch.nn.functional as F\n",
        "import sequence_generator\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from function import FunctionTerm, Function\n",
        "import sequence_generator\n",
        "\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "MAX_LENGTH = 22\n",
        "SOS_token = 10001\n",
        "EOS_token = 10002\n",
        "\n",
        "NTERMS = 2\n",
        "USE_INTERACTION = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "# def load_data(nterms, use_interaction = False, train_data=True):\n",
        "#     # return a list [sequence, mask] elements\n",
        "#     if use_interaction:\n",
        "#         if train_data:\n",
        "#             df = pd.read_csv(f'data/train/{nterms}/{nterms}_int.csv', names=['prompt', 'completion'], delimiter='],', engine='python')\n",
        "#         else:\n",
        "#             df = pd.read_csv(f'data/test/{nterms}/{nterms}_int.csv', names=['prompt', 'completion'], delimiter='],', engine='python')\n",
        "#     else:\n",
        "#         if train_data:\n",
        "#             df = pd.read_csv(f'data/train/{nterms}/{nterms}.csv', names=['prompt', 'completion'], delimiter='],', engine='python')\n",
        "#         else:\n",
        "#             df = pd.read_csv(f'data/test/{nterms}/{nterms}.csv', names=['prompt', 'completion'], delimiter='],', engine='python')\n",
        "    \n",
        "#     data = []\n",
        "#     for i in range(len(df)):\n",
        "#         seq = df.loc[i, 'prompt']\n",
        "#         seq = seq.replace('[','').replace(']','')\n",
        "#         seq = seq.split(',')\n",
        "#         seq = [int(s) for s in seq]\n",
        "\n",
        "#         mask =df.loc[i, 'completion']\n",
        "#         mask = mask.replace('[','').replace(']','')\n",
        "#         mask = mask.split(',')\n",
        "#         mask = [eval(s) for s in mask]\n",
        "\n",
        "#         data.append([seq, mask])\n",
        "    \n",
        "#     return data\n",
        "\n",
        "# def process_data(data):\n",
        "#     processed_sequences = []\n",
        "#     for in_sequence, out_sequence in data:\n",
        "#         in_sequence = torch.tensor(in_sequence)\n",
        "#         in_sequence = in_sequence.long()\n",
        "#         in_sequence = in_sequence[:, None]\n",
        "#         in_sequence = in_sequence.to(device)\n",
        "\n",
        "#         out_sequence = torch.tensor(out_sequence)\n",
        "#         out_sequence = out_sequence.long()\n",
        "#         out_sequence = out_sequence[:, None]\n",
        "#         out_sequence = out_sequence.to(device)\n",
        "#         processed_sequences.append((in_sequence, out_sequence))\n",
        "\n",
        "#     return processed_sequences\n",
        "\n",
        "# train_sequences = process_data(load_data(NTERMS, True))\n",
        "# test_sequences = process_data(load_data(NTERMS, False))\n",
        "\n",
        "# print('train set size: ', len(train_sequences), 'test set size', len(test_sequences))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "22\n",
            "train set size:  400 test set size 100\n"
          ]
        }
      ],
      "source": [
        "sequences = sequence_generator.make_n_random_functions(n=500, nterms=NTERMS, use_interaction=USE_INTERACTION, torchify=True)\n",
        "# sequences = sequence_generator.make_n_random_functions(n=1000, torchify=True)\n",
        "\n",
        "\n",
        "processed_sequences = []\n",
        "\n",
        "for in_sequence, out_sequence in sequences:\n",
        "    in_sequence = in_sequence.long()\n",
        "    in_sequence = in_sequence[:, None]\n",
        "    in_sequence = in_sequence.to(device)\n",
        "\n",
        "    out_sequence = torch.tensor(out_sequence)\n",
        "    out_sequence = out_sequence.long()\n",
        "    out_sequence = out_sequence[:, None]\n",
        "    out_sequence = out_sequence.to(device)\n",
        "    processed_sequences.append((in_sequence, out_sequence))\n",
        "\n",
        "# 80-20 train test split\n",
        "train_sequences = processed_sequences[:int(len(processed_sequences) * 0.8)]\n",
        "test_sequences = processed_sequences[int(len(processed_sequences) * 0.8):]\n",
        "del processed_sequences\n",
        "\n",
        "print('train set size: ', len(train_sequences), 'test set size', len(test_sequences))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "class EncoderRNN(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size):\n",
        "        super(EncoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "\n",
        "    def forward(self, input, hidden):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        output = embedded\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "        return output, hidden\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "class DecoderRNN(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size):\n",
        "        super(DecoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "        self.out = nn.Linear(hidden_size, output_size)\n",
        "        self.softmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    def forward(self, input, hidden):\n",
        "        output = self.embedding(input).view(1, 1, -1)\n",
        "        output = F.relu(output)\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "        output = self.softmax(self.out(output[0]))\n",
        "        return output, hidden\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "class AttnDecoderRNN(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):\n",
        "        super(AttnDecoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        self.dropout_p = dropout_p\n",
        "        self.max_length = max_length\n",
        "\n",
        "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
        "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
        "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
        "        self.dropout = nn.Dropout(self.dropout_p)\n",
        "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
        "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
        "\n",
        "    def forward(self, input, hidden, encoder_outputs):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        embedded = self.dropout(embedded)\n",
        "\n",
        "        attn_weights = F.softmax(\n",
        "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n",
        "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
        "                                 encoder_outputs.unsqueeze(0))\n",
        "\n",
        "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
        "        output = self.attn_combine(output).unsqueeze(0)\n",
        "\n",
        "        output = F.relu(output)\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "\n",
        "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
        "        return output, hidden, attn_weights\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "teacher_forcing_ratio = 0.5\n",
        "\n",
        "\n",
        "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):\n",
        "    encoder_hidden = encoder.initHidden()\n",
        "\n",
        "    encoder_optimizer.zero_grad()\n",
        "    decoder_optimizer.zero_grad()\n",
        "\n",
        "    input_length = input_tensor.size(0)\n",
        "    target_length = target_tensor.size(0)\n",
        "\n",
        "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "    loss = 0\n",
        "\n",
        "    for ei in range(input_length):\n",
        "        encoder_output, encoder_hidden = encoder(\n",
        "            input_tensor[ei], encoder_hidden)\n",
        "        encoder_outputs[ei] = encoder_output[0, 0]\n",
        "\n",
        "    decoder_input = torch.tensor(np.array([[SOS_token]]), device=device, dtype=torch.long)\n",
        "\n",
        "    decoder_hidden = encoder_hidden\n",
        "\n",
        "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
        "\n",
        "    if use_teacher_forcing:\n",
        "        # Teacher forcing: Feed the target as the next input\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            decoder_input = target_tensor[di]  # Teacher forcing\n",
        "\n",
        "    else:\n",
        "        # Without teacher forcing: use its own predictions as the next input\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            topv, topi = decoder_output.topk(1)\n",
        "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
        "\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            if decoder_input.item() == EOS_token:\n",
        "                break\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    encoder_optimizer.step()\n",
        "    decoder_optimizer.step()\n",
        "\n",
        "    return loss.item() / target_length"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import math\n",
        "\n",
        "\n",
        "def asMinutes(s):\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    return '%dm %ds' % (m, s)\n",
        "\n",
        "\n",
        "def timeSince(since, percent):\n",
        "    now = time.time()\n",
        "    s = now - since\n",
        "    es = s / (percent)\n",
        "    rs = es - s\n",
        "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):\n",
        "    start = time.time()\n",
        "    plot_losses = []\n",
        "    print_loss_total = 0  # Reset every print_every\n",
        "    plot_loss_total = 0  # Reset every plot_every\n",
        "\n",
        "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
        "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
        "    training_pairs = [random.choice(train_sequences)\n",
        "                      for i in range(n_iters)]\n",
        "    criterion = nn.NLLLoss()\n",
        "\n",
        "    for iter in range(1, n_iters + 1):\n",
        "        training_pair = training_pairs[iter - 1]\n",
        "        input_tensor = training_pair[0]\n",
        "        target_tensor = training_pair[1]\n",
        "\n",
        "        loss = train(input_tensor, target_tensor, encoder,\n",
        "                     decoder, encoder_optimizer, decoder_optimizer, criterion)\n",
        "        print_loss_total += loss\n",
        "        plot_loss_total += loss\n",
        "\n",
        "        if (iter + 1) % print_every == 0:\n",
        "            evaluateRandomly(encoder, decoder)\n",
        "            print_loss_avg = print_loss_total / print_every\n",
        "            print_loss_total = 0\n",
        "            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),\n",
        "                                         iter, iter / n_iters * 100, print_loss_avg))\n",
        "\n",
        "        if iter % plot_every == 0:\n",
        "            plot_loss_avg = plot_loss_total / plot_every\n",
        "            plot_losses.append(plot_loss_avg)\n",
        "            plot_loss_total = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "def evaluate(encoder, decoder, input_tensor, max_length=MAX_LENGTH):\n",
        "    with torch.no_grad():\n",
        "        input_length = input_tensor.size()[0]\n",
        "        encoder_hidden = encoder.initHidden()\n",
        "\n",
        "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "        for ei in range(input_length):\n",
        "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
        "                                                     encoder_hidden)\n",
        "            encoder_outputs[ei] += encoder_output[0, 0]\n",
        "\n",
        "        decoder_input = torch.tensor(np.array([[SOS_token]]), device=device, dtype=torch.long)  # SOS\n",
        "\n",
        "        decoder_hidden = encoder_hidden\n",
        "\n",
        "        decoded_words = []\n",
        "        decoder_attentions = torch.zeros(max_length, max_length)\n",
        "\n",
        "        for di in range(max_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            decoder_attentions[di] = decoder_attention.data\n",
        "            topv, topi = decoder_output.data.topk(1)\n",
        "            if topi.item() == EOS_token:\n",
        "                decoded_words.append('<EOS>')\n",
        "                break\n",
        "            else:\n",
        "                # decoded_words.append(output_lang.index2word[topi.item()])\n",
        "                decoded_words.append(topi.item())\n",
        "\n",
        "            decoder_input = topi.squeeze().detach()\n",
        "\n",
        "        return decoded_words, decoder_attentions[:di + 1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [],
      "source": [
        "def grid_search(sequence, terms, upper_bound = 5, lower_bound = -5):\n",
        "    # sequence: target sequence\n",
        "    # terms: list of all FunctionTerm objects proposed by MCTS\n",
        "\n",
        "    if len(terms) == 0:\n",
        "        return np.inf\n",
        "\n",
        "    coeff = sorted(list(range(1, upper_bound+1))+list(range(lower_bound,0)), key=lambda x: abs(x))\n",
        "    base = len(coeff)\n",
        "    digit_to_coeff = {i: coeff[i] for i in range(base)}\n",
        "\n",
        "\n",
        "    def int_to_base_helper(num, base):\n",
        "        ret = []\n",
        "        while num > 0:\n",
        "            ret.append(num % base)\n",
        "            num = num // base\n",
        "        return ret\n",
        "    \n",
        "    penalty = np.inf\n",
        "    \n",
        "    for i in range(base**len(terms)):\n",
        "        term_coeffs = [digit_to_coeff[c] for c in int_to_base_helper(i, base)+[0]*50]\n",
        "        f = Function()\n",
        "        for j, term in enumerate(terms):\n",
        "            term.updateCoeff(term_coeffs[j])\n",
        "            f.addTerm(term)\n",
        "        \n",
        "        if f.startIndex() > len(sequence):\n",
        "            continue\n",
        "        \n",
        "        targets = np.array(sequence[f.startIndex()-1:])\n",
        "        predictions = [f.evaluate(sequence, n) for n in range(f.startIndex(), len(sequence)+1)]\n",
        "        \n",
        "        if None in predictions:\n",
        "            raise Exception(f'None in prediction! Current f is f[n] = {f} and prediction is {predictions}')\n",
        "        \n",
        "        predictions = np.array(predictions)\n",
        "        \n",
        "        rmse_loss = np.sqrt(np.mean((predictions-targets)**2))\n",
        "        if rmse_loss < penalty:\n",
        "            penalty = rmse_loss\n",
        "        \n",
        "        del f\n",
        "        \n",
        "        if penalty == 0:\n",
        "            return penalty  \n",
        "        \n",
        "    return penalty\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "def generate_term_to_id_map():\n",
        "    i = 0\n",
        "    id_to_func_term = {}\n",
        "    term_str_to_func_term = {}\n",
        "    term_str_to_id = {}\n",
        "    \n",
        "    POSSIBLE_TERMS = sequence_generator.make_possible_terms(False)\n",
        "    for t in POSSIBLE_TERMS:\n",
        "        temp_f = Function()\n",
        "        temp_f.addTerm(t)\n",
        "\n",
        "        id_to_func_term[i] = t\n",
        "        \n",
        "        term_str = str(t).replace('0*','').replace('0','1')\n",
        "        term_str_to_id[term_str] = i\n",
        "        \n",
        "        term_str_to_func_term[term_str] = t\n",
        "        \n",
        "        i += 1\n",
        "            \n",
        "    term_str_to_id['<ROOT>'] = i\n",
        "    i += 1\n",
        "    term_str_to_id['<EOS>'] = i\n",
        "    return id_to_func_term, term_str_to_func_term, term_str_to_id\n",
        "\n",
        "id_to_func_term, term_str_to_func_term, term_str_to_id = generate_term_to_id_map()\n",
        "\n",
        "def make_functionterms_from_sentence(indices):\n",
        "    indices = np.where(indices)[0]\n",
        "    out = []\n",
        "    for i in indices:\n",
        "        if i < len(id_to_func_term):\n",
        "            out.append(id_to_func_term[i])\n",
        "    return out\n",
        "\n",
        "\n",
        "best_totally_correct = 0\n",
        "best_avg_rmse = np.inf\n",
        "\n",
        "def evaluateRandomly(encoder, decoder):\n",
        "    # print('starting evaluation')\n",
        "    num_totally_correct = 0\n",
        "\n",
        "    correct_tokens = 0\n",
        "    total_tokens = 0\n",
        "\n",
        "    rmses = []\n",
        "    min_rmse = np.inf\n",
        "    for pair in test_sequences:\n",
        "        output_words, attentions = evaluate(encoder, decoder, pair[0])\n",
        "        output_sentence = output_words\n",
        "\n",
        "        sequence = pair[0][:, 0].detach().cpu().numpy()\n",
        "        rmse = grid_search(sequence, make_functionterms_from_sentence(output_sentence[:len(id_to_func_term)])[:NTERMS]) # cut the thing short\n",
        "        min_rmse = rmse if rmse < min_rmse else min_rmse\n",
        "\n",
        "        if (not np.isinf(rmse)) and (not np.isnan(rmse)):\n",
        "            rmses.append(rmse)\n",
        "\n",
        "        for idx, gt_element in enumerate(pair[0][:, 0]):\n",
        "            total_tokens += 1\n",
        "            if output_words[idx] == gt_element:\n",
        "                correct_tokens += 1\n",
        "\n",
        "\n",
        "        if pair[1][:, 0].tolist() == output_sentence:\n",
        "            num_totally_correct += 1\n",
        "\n",
        "    avg_rmse = np.mean(rmses)\n",
        "    print()\n",
        "    print('num correct:', num_totally_correct)\n",
        "    print('num tokens correct:', correct_tokens)\n",
        "    print('num total tokens', total_tokens)\n",
        "    print('avg rmse', avg_rmse)\n",
        "\n",
        "    global best_totally_correct\n",
        "    best_totally_correct = num_totally_correct if num_totally_correct > best_totally_correct else best_totally_correct\n",
        "\n",
        "    global best_avg_rmse\n",
        "    best_avg_rmse = avg_rmse if avg_rmse < best_avg_rmse else best_avg_rmse\n",
        "    # # making sure that the model is updating on each iteration\n",
        "    # global last_attentions\n",
        "    # print('same attention?', last_attentions == attentions)\n",
        "    # last_attentions = attentions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/home/longyuxi/miniconda3/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3419: RuntimeWarning: Mean of empty slice.\n",
            "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
            "/home/longyuxi/miniconda3/lib/python3.8/site-packages/numpy/core/_methods.py:188: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "num correct: 0\n",
            "num tokens correct: 0\n",
            "num total tokens 700\n",
            "avg rmse nan\n",
            "0m 18s (- 11m 53s) (499 2%) 0.3855\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 0\n",
            "num total tokens 700\n",
            "avg rmse nan\n",
            "0m 35s (- 11m 24s) (999 4%) 0.3076\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 3\n",
            "num total tokens 700\n",
            "avg rmse 1150.0122534714083\n",
            "0m 54s (- 11m 10s) (1499 7%) 0.3002\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 3\n",
            "num total tokens 700\n",
            "avg rmse 1083.4728800784455\n",
            "1m 12s (- 10m 52s) (1999 9%) 0.2938\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 0\n",
            "num total tokens 700\n",
            "avg rmse 492.2465990595712\n",
            "1m 30s (- 10m 31s) (2499 12%) 0.2826\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 575.9733274441187\n",
            "1m 47s (- 10m 9s) (2999 14%) 0.2756\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 10\n",
            "num total tokens 700\n",
            "avg rmse 462.49851272074847\n",
            "2m 5s (- 9m 53s) (3499 17%) 0.3604\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 16\n",
            "num total tokens 700\n",
            "avg rmse 465.0355146788604\n",
            "2m 23s (- 9m 34s) (3999 19%) 0.4632\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 6\n",
            "num total tokens 700\n",
            "avg rmse 743.4984362654204\n",
            "2m 41s (- 9m 15s) (4499 22%) 0.3392\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 4\n",
            "num total tokens 700\n",
            "avg rmse 246.50289494734784\n",
            "2m 58s (- 8m 56s) (4999 24%) 0.3038\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 4\n",
            "num total tokens 700\n",
            "avg rmse 79.62604947718053\n",
            "3m 16s (- 8m 39s) (5499 27%) 0.2890\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 261.6266648583401\n",
            "3m 34s (- 8m 20s) (5999 29%) 0.2824\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 12\n",
            "num total tokens 700\n",
            "avg rmse 571.3783901636457\n",
            "3m 52s (- 8m 2s) (6499 32%) 0.2693\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 11\n",
            "num total tokens 700\n",
            "avg rmse 463.86421825052884\n",
            "4m 10s (- 7m 45s) (6999 34%) 0.2619\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 13\n",
            "num total tokens 700\n",
            "avg rmse 507.40590731393576\n",
            "4m 28s (- 7m 27s) (7499 37%) 0.2621\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 493.8415556041817\n",
            "4m 45s (- 7m 8s) (7999 39%) 0.2590\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 11\n",
            "num total tokens 700\n",
            "avg rmse 648.391619946372\n",
            "5m 3s (- 6m 51s) (8499 42%) 0.2518\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 418.3719486008135\n",
            "5m 21s (- 6m 33s) (8999 44%) 0.2467\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 12\n",
            "num total tokens 700\n",
            "avg rmse 706.2187903487328\n",
            "5m 39s (- 6m 15s) (9499 47%) 0.2422\n",
            "\n",
            "num correct: 1\n",
            "num tokens correct: 10\n",
            "num total tokens 700\n",
            "avg rmse 560.7790047620882\n",
            "5m 57s (- 5m 57s) (9999 49%) 0.2334\n",
            "\n",
            "num correct: 1\n",
            "num tokens correct: 11\n",
            "num total tokens 700\n",
            "avg rmse 623.6763816513127\n",
            "6m 15s (- 5m 40s) (10499 52%) 0.2272\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 276.9780049215908\n",
            "6m 33s (- 5m 22s) (10999 54%) 0.2221\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 7\n",
            "num total tokens 700\n",
            "avg rmse 206.03074557634542\n",
            "6m 51s (- 5m 4s) (11499 57%) 0.2181\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 7\n",
            "num total tokens 700\n",
            "avg rmse 519.0966881044588\n",
            "7m 8s (- 4m 45s) (11999 59%) 0.2207\n",
            "\n",
            "num correct: 1\n",
            "num tokens correct: 12\n",
            "num total tokens 700\n",
            "avg rmse 531.1663089271199\n",
            "7m 26s (- 4m 28s) (12499 62%) 0.2153\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 7\n",
            "num total tokens 700\n",
            "avg rmse 1263.7549336222678\n",
            "7m 44s (- 4m 10s) (12999 64%) 0.2177\n",
            "\n",
            "num correct: 1\n",
            "num tokens correct: 9\n",
            "num total tokens 700\n",
            "avg rmse 706.4811827377007\n",
            "8m 2s (- 3m 52s) (13499 67%) 0.2104\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 14\n",
            "num total tokens 700\n",
            "avg rmse 521.5535685235991\n",
            "8m 20s (- 3m 34s) (13999 69%) 0.2367\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 11\n",
            "num total tokens 700\n",
            "avg rmse 702.8913527980682\n",
            "8m 38s (- 3m 16s) (14499 72%) 0.2069\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 1313.11532353517\n",
            "8m 56s (- 2m 58s) (14999 74%) 0.2052\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 9\n",
            "num total tokens 700\n",
            "avg rmse 472.20798332446236\n",
            "9m 14s (- 2m 41s) (15499 77%) 0.2033\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 13\n",
            "num total tokens 700\n",
            "avg rmse 925.3331770693164\n",
            "9m 32s (- 2m 23s) (15999 79%) 0.1914\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 19\n",
            "num total tokens 700\n",
            "avg rmse 780.6383861130257\n",
            "9m 49s (- 2m 5s) (16499 82%) 0.1922\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 12\n",
            "num total tokens 700\n",
            "avg rmse 749.1963585172983\n",
            "10m 7s (- 1m 47s) (16999 84%) 0.1876\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 14\n",
            "num total tokens 700\n",
            "avg rmse 787.1584369057215\n",
            "10m 26s (- 1m 29s) (17499 87%) 0.1764\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 9\n",
            "num total tokens 700\n",
            "avg rmse 1123.638508077967\n",
            "10m 44s (- 1m 11s) (17999 89%) 0.1742\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 7\n",
            "num total tokens 700\n",
            "avg rmse 1783.80430382094\n",
            "11m 2s (- 0m 53s) (18499 92%) 0.1741\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 9\n",
            "num total tokens 700\n",
            "avg rmse 975.8259832680842\n",
            "11m 20s (- 0m 35s) (18999 94%) 0.1682\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 9\n",
            "num total tokens 700\n",
            "avg rmse 1575.5800224323566\n",
            "11m 38s (- 0m 17s) (19499 97%) 0.1754\n",
            "\n",
            "num correct: 0\n",
            "num tokens correct: 8\n",
            "num total tokens 700\n",
            "avg rmse 1028.5740470269777\n",
            "11m 56s (- 0m 0s) (19999 99%) 0.1646\n",
            "==============================\n",
            "most number of totally correct functions: 1\n",
            "best avg rmse 79.62604947718053\n"
          ]
        }
      ],
      "source": [
        "n_words = 10005\n",
        "hidden_size = 256\n",
        "encoder1 = EncoderRNN(n_words, hidden_size).to(device)\n",
        "attn_decoder1 = AttnDecoderRNN(hidden_size, n_words, dropout_p=0.1).to(device)\n",
        "\n",
        "trainIters(encoder1, attn_decoder1, 20000, print_every=500)\n",
        "\n",
        "print('='*30)\n",
        "print('most number of totally correct functions:', best_totally_correct)\n",
        "print('best avg rmse', best_avg_rmse)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.8.5 ('base')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "vscode": {
      "interpreter": {
        "hash": "6bbe29e2308ccd271c4ed360a09e65ae469839e520c3df2570e8ecd9e52abb09"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}