Packed Padded Sequences, Masking and Inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torchtext\n",
    "from torchtext.datasets import TranslationDataset, Multi30k\n",
    "from torchtext.data import Field, BucketIterator\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "import spacy\n",
    "\n",
    "import random\n",
    "import math\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 1\n",
    "\n",
    "random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_de = spacy.load('de')\n",
    "spacy_en = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_de(text):\n",
    "    \"\"\"\n",
    "    Tokenizes German text from a string into a list of strings\n",
    "    \"\"\"\n",
    "    return [tok.text for tok in spacy_de.tokenizer(text)]\n",
    "\n",
    "def tokenize_en(text):\n",
    "    \"\"\"\n",
    "    Tokenizes English text from a string into a list of strings\n",
    "    \"\"\"\n",
    "    return [tok.text for tok in spacy_en.tokenizer(text)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- include lengths in source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "SRC = Field(tokenize=tokenize_de, init_token='<sos>', eos_token='<eos>', lower=True, include_lengths=True)\n",
    "TRG = Field(tokenize=tokenize_en, init_token='<sos>', eos_token='<eos>', lower=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, valid_data, test_data = Multi30k.splits(exts=('.de', '.en'), fields=(SRC, TRG))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "SRC.build_vocab(train_data, min_freq=2)\n",
    "TRG.build_vocab(train_data, min_freq=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- all this stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "     batch_size=BATCH_SIZE,\n",
    "     sort_key = lambda x : len(x.src),\n",
    "     sort_within_batch=True,\n",
    "     device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- encoder needs to accept lengths\n",
    "- pack sequence before going into rnn\n",
    "- unpack output before returning it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.input_dim = input_dim\n",
    "        self.emb_dim = emb_dim\n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        self.dropout = dropout\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU(emb_dim, enc_hid_dim, bidirectional=True)\n",
    "        \n",
    "        self.fc = nn.Linear(enc_hid_dim * 2, dec_hid_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src, src_len):\n",
    "        \n",
    "        #src = [src sent len, batch size]\n",
    "        #src_len = [src sent len]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(src))\n",
    "        \n",
    "        #embedded = [src sent len, batch size, emb dim]\n",
    "        \n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, src_len)\n",
    "        \n",
    "        packed_outputs, hidden = self.rnn(packed_embedded)\n",
    "               \n",
    "        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs) \n",
    "            \n",
    "        #outputs = [sent len, batch size, hid dim * num directions]\n",
    "        #hidden = [n layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        #hidden is stacked [forward_1, backward_1, forward_2, backward_2, ...]\n",
    "        #outputs are always from the last layer\n",
    "        \n",
    "        #hidden [-2, :, : ] is the last of the forwards RNN \n",
    "        #hidden [-1, :, : ] is the last of the backwards RNN\n",
    "        \n",
    "        #initial decoder hidden is final hidden state of the forwards and backwards encoder RNNs fed through a linear layer\n",
    "        hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1)))\n",
    "        \n",
    "        #outputs = [sent len, batch size, enc hid dim * 2]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        \n",
    "        return outputs, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- unchanged as mask is applied in decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    def __init__(self, enc_hid_dim, dec_hid_dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        \n",
    "        self.attn = nn.Linear((enc_hid_dim * 2) + dec_hid_dim, dec_hid_dim)\n",
    "        self.v = nn.Parameter(torch.rand(dec_hid_dim))\n",
    "        \n",
    "    def forward(self, hidden, encoder_outputs):\n",
    "        \n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        \n",
    "        batch_size = encoder_outputs.shape[1]\n",
    "        src_len = encoder_outputs.shape[0]\n",
    "        \n",
    "        #repeat encoder hidden state src_len times\n",
    "        hidden = hidden.unsqueeze(1).repeat(1, src_len, 1)\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #hidden = [batch size, src sent len, dec hid dim]\n",
    "        #encoder_outputs = [batch size, src sent len, enc hid dim * 2]\n",
    "        \n",
    "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim=2))) \n",
    "        \n",
    "        #energy = [batch size, src sent len, dec hid dim]\n",
    "        \n",
    "        energy = energy.permute(0, 2, 1)\n",
    "        \n",
    "        #energy = [batch size, dec hid dim, src sent len]\n",
    "        \n",
    "        #v = [dec hid dim]\n",
    "        \n",
    "        v = self.v.repeat(batch_size, 1).unsqueeze(1)\n",
    "        \n",
    "        #v = [batch size, 1, dec hid dim]\n",
    "                \n",
    "        attention = torch.bmm(v, energy).squeeze(1)\n",
    "        \n",
    "        #attention = [batch size, src sent len]\n",
    "        \n",
    "        return F.softmax(attention, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- applies mask to attention output by decoder\n",
    "- renormalizes attention so each row sums to 1 still\n",
    "- returns attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):\n",
    "        super().__init__()\n",
    "\n",
    "        self.emb_dim = emb_dim\n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.dropout = dropout\n",
    "        self.attention = attention\n",
    "        \n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU((enc_hid_dim * 2) + emb_dim, dec_hid_dim)\n",
    "        \n",
    "        self.out = nn.Linear((enc_hid_dim * 2) + dec_hid_dim + emb_dim, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, input, hidden, encoder_outputs, mask):\n",
    "             \n",
    "        #input = [batch size]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        #mask = [batch size, src sent len]\n",
    "        \n",
    "        input = input.unsqueeze(0)\n",
    "        \n",
    "        #input = [1, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "        \n",
    "        #embedded = [1, batch size, emb dim]\n",
    "        \n",
    "        a = self.attention(hidden, encoder_outputs)\n",
    "                \n",
    "        #a = [batch size, src sent len]\n",
    "        \n",
    "        #apply mask\n",
    "        a = a * mask\n",
    "        \n",
    "        #re-normalize attention\n",
    "        _sums = a.sum(1, keepdim=True)  #keepdim keeps tensor [batch size, 1] instead of making it [batch size]\n",
    "        a = a.div(_sums) \n",
    "        \n",
    "        a = a.unsqueeze(1)\n",
    "        \n",
    "        #a = [batch size, 1, src sent len]\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #encoder_outputs = [batch size, src sent len, enc hid dim * 2]\n",
    "        \n",
    "        weighted = torch.bmm(a, encoder_outputs)\n",
    "        \n",
    "        #weighted = [batch size, 1, enc hid dim * 2]\n",
    "        \n",
    "        weighted = weighted.permute(1, 0, 2)\n",
    "        \n",
    "        #weighted = [1, batch size, enc hid dim * 2]\n",
    "        \n",
    "        rnn_input = torch.cat((embedded, weighted), dim=2)\n",
    "        \n",
    "        #rnn_input = [1, batch size, (enc hid dim * 2) + emb dim]\n",
    "            \n",
    "        output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0))\n",
    "        \n",
    "        #output = [sent len, batch size, dec hid dim * n directions]\n",
    "        #hidden = [n layers * n directions, batch size, dec hid dim]\n",
    "        \n",
    "        #sent len, n layers and n directions will always be 1 in this decoder, therefore:\n",
    "        #output = [1, batch size, dec hid dim]\n",
    "        #hidden = [1, batch size, dec hid dim]\n",
    "        #this also means that output == hidden\n",
    "        assert (output == hidden).all(), print(output.shape, hidden.shape, output[0,0,:25], hidden[0,0,:25])\n",
    "        \n",
    "        embedded = embedded.squeeze(0)\n",
    "        output = output.squeeze(0)\n",
    "        weighted = weighted.squeeze(0)\n",
    "        \n",
    "        output = self.out(torch.cat((output, weighted, embedded), dim=1))\n",
    "        \n",
    "        #output = [bsz, output dim]\n",
    "        \n",
    "        return output, hidden.squeeze(0), a.squeeze(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- seq2seq needs to take in src_len, creates mask\n",
    "- pass mask to decoder to be used in attn calculation\n",
    "- receive attention from decoder and store in tensor and return it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, encoder, decoder, sos_idx, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.sos_idx = sos_idx\n",
    "        self.device = device\n",
    "        \n",
    "    def create_mask(self, src_len):\n",
    "        max_len = src_len.max()\n",
    "        idxs = torch.arange(0,max_len).to(src_len.device)\n",
    "        mask = (idxs<src_len.unsqueeze(1)).float()\n",
    "        return mask\n",
    "        \n",
    "    def forward(self, src, src_len, trg, teacher_forcing_ratio=0.5):\n",
    "        \n",
    "        #src = [src sent len, batch size]\n",
    "        #src_len = [batch size]\n",
    "        #trg = [trg sent len, batch size]\n",
    "        #teacher_forcing_ratio is probability to use teacher forcing\n",
    "        #e.g. if teacher_forcing_ratio is 0.75 we use teacher forcing 75% of the time\n",
    "        \n",
    "        if trg is None:\n",
    "            assert teacher_forcing_ratio == 0, \"Must be zero during inference\"\n",
    "            trg = torch.zeros((20, src.shape[1]), dtype=torch.long).fill_(self.sos_idx).to(src.device)\n",
    "            \n",
    "        batch_size = src.shape[1]\n",
    "        max_len = trg.shape[0]\n",
    "        trg_vocab_size = self.decoder.output_dim\n",
    "        \n",
    "        #tensor to store decoder outputs\n",
    "        outputs = torch.zeros(max_len, batch_size, trg_vocab_size).to(self.device)\n",
    "        \n",
    "        #tensor to store attention\n",
    "        attentions = torch.zeros(max_len, batch_size, src.shape[0]).to(self.device)\n",
    "        \n",
    "        #encoder_outputs is all hidden states of the input sequence, back and forwards\n",
    "        #hidden is the final forward and backward hidden states, passed through a linear layer\n",
    "        encoder_outputs, hidden = self.encoder(src, src_len)\n",
    "                \n",
    "        #first input to the decoder is the <sos> tokens\n",
    "        output = trg[0,:]\n",
    "        \n",
    "        mask = self.create_mask(src_len)\n",
    "                \n",
    "        #mask = [batch size, src sent len]\n",
    "                \n",
    "        for t in range(1, max_len):\n",
    "            output, hidden, attention = self.decoder(output, hidden, encoder_outputs, mask)\n",
    "            outputs[t] = output\n",
    "            attentions[t] = attention\n",
    "            teacher_force = random.random() < teacher_forcing_ratio\n",
    "            top1 = output.max(1)[1]\n",
    "            output = (trg[t] if teacher_force else top1)\n",
    "\n",
    "        return outputs, attentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "INPUT_DIM = len(SRC.vocab)\n",
    "OUTPUT_DIM = len(TRG.vocab)\n",
    "ENC_EMB_DIM = 256\n",
    "DEC_EMB_DIM = 256\n",
    "ENC_HID_DIM = 512\n",
    "DEC_HID_DIM = 512\n",
    "ENC_DROPOUT = 0.5\n",
    "DEC_DROPOUT = 0.5\n",
    "SOS_IDX = TRG.vocab.stoi['<sos>']\n",
    "\n",
    "attn = Attention(ENC_HID_DIM, DEC_HID_DIM)\n",
    "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT)\n",
    "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn)\n",
    "\n",
    "model = Seq2Seq(enc, dec, SOS_IDX, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_idx = TRG.vocab.stoi['<pad>']\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=pad_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, clip):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    for i, batch in enumerate(iterator):\n",
    "        \n",
    "        src, src_len = batch.src\n",
    "        trg = batch.trg\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        output, _ = model(src, src_len, trg)\n",
    "        \n",
    "        loss = criterion(output[1:].view(-1, output.shape[2]), trg[1:].view(-1))\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in enumerate(iterator):\n",
    "\n",
    "            src, src_len = batch.src\n",
    "            trg = batch.trg\n",
    "\n",
    "            output, _ = model(src, src_len, trg, 0) #turn off teacher forcing\n",
    "\n",
    "            loss = criterion(output[1:].view(-1, output.shape[2]), trg[1:].view(-1))\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Epoch: 001 | Train Loss: 4.239 | Train PPL:  69.318 | Val. Loss: 3.720 | Val. PPL:  41.279 |\n",
      "| Epoch: 002 | Train Loss: 3.088 | Train PPL:  21.938 | Val. Loss: 3.315 | Val. PPL:  27.523 |\n",
      "| Epoch: 003 | Train Loss: 2.657 | Train PPL:  14.255 | Val. Loss: 3.290 | Val. PPL:  26.837 |\n",
      "| Epoch: 004 | Train Loss: 2.366 | Train PPL:  10.651 | Val. Loss: 3.196 | Val. PPL:  24.442 |\n",
      "| Epoch: 005 | Train Loss: 2.151 | Train PPL:   8.592 | Val. Loss: 3.264 | Val. PPL:  26.144 |\n",
      "| Epoch: 006 | Train Loss: 1.998 | Train PPL:   7.373 | Val. Loss: 3.328 | Val. PPL:  27.875 |\n",
      "| Epoch: 007 | Train Loss: 1.861 | Train PPL:   6.431 | Val. Loss: 3.361 | Val. PPL:  28.817 |\n",
      "| Epoch: 008 | Train Loss: 1.777 | Train PPL:   5.914 | Val. Loss: 3.421 | Val. PPL:  30.606 |\n",
      "| Epoch: 009 | Train Loss: 1.698 | Train PPL:   5.462 | Val. Loss: 3.360 | Val. PPL:  28.783 |\n",
      "| Epoch: 010 | Train Loss: 1.613 | Train PPL:   5.019 | Val. Loss: 3.452 | Val. PPL:  31.571 |\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 10\n",
    "CLIP = 10\n",
    "SAVE_DIR = 'models'\n",
    "MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'tut4_model.pt')\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "if not os.path.isdir(f'{SAVE_DIR}'):\n",
    "    os.makedirs(f'{SAVE_DIR}')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    \n",
    "    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)\n",
    "    valid_loss = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), MODEL_SAVE_PATH)\n",
    "    \n",
    "    print(f'| Epoch: {epoch+1:03} | Train Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f} | Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f} |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Test Loss: 3.225 | Test PPL:  25.149 |\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(MODEL_SAVE_PATH))\n",
    "\n",
    "test_loss = evaluate(model, test_iterator, criterion)\n",
    "\n",
    "print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_sentence(sentence):\n",
    "    tokenized = tokenize_de(sentence) #tokenize sentence\n",
    "    tokenized = ['<sos>'] + [t.lower() for t in tokenized] + ['<eos>'] #add <sos> and <eos> tokens and lowercase\n",
    "    numericalized = [SRC.vocab.stoi[t] for t in tokenized] #convert tokens into indexes\n",
    "    sentence_length = torch.LongTensor([len(numericalized)]).to(device) #need sentence length for masking\n",
    "    tensor = torch.LongTensor(numericalized).unsqueeze(1).to(device) #convert to tensor and add batch dimension\n",
    "    translation_tensor_probs, attention = model(tensor, sentence_length, None, 0) #pass through model to get translation probabilities\n",
    "    translation_tensor = torch.argmax(translation_tensor_probs.squeeze(1), 1) #get translation from highest probabilities\n",
    "    translation = [TRG.vocab.itos[t] for t in translation_tensor][1:] #ignore the first token, just like we do in the training loop\n",
    "    return translation, attention[1:] #ignore first attention array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_attention(candidate, translation, attention):\n",
    "    # Set up figure with colorbar\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    translation = translation[:translation.index('<eos>')] #cut translation after first <eos> token\n",
    "    attention = attention[:len(translation)].squeeze(1).cpu().detach().numpy() #cut attention to same length as translation\n",
    "    cax = ax.matshow(attention, cmap='bone')\n",
    "    fig.colorbar(cax)\n",
    "\n",
    "    # Set up axes\n",
    "    ax.set_xticklabels([''] + ['<sos>'] + [t.lower for t in tokenize_de(candidate)] + ['<eos>'], rotation=90)\n",
    "    ax.set_yticklabels([''] + translation)\n",
    "\n",
    "    # Show label at every tick\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zwei junge weiße männer sind im freien in der nähe vieler büsche .\n",
      "two young , white males are outside near many bushes .\n"
     ]
    }
   ],
   "source": [
    "candidate = ' '.join(vars(train_data.examples[0])['src'])\n",
    "candidate_translation = ' '.join(vars(train_data.examples[0])['trg'])\n",
    "\n",
    "print(candidate)\n",
    "print(candidate_translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['two', 'young', 'white', 'males', 'are', 'outside', 'near', 'near', 'bushes', '.', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEICAYAAABVv+9nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XucXFWZ7vHfQwgXAUGFgxdAlEEYRLkkgiAoijLoKHgBEVFEkYyj6CgHPYw6yqDOeBsYcVCJXETEYRBFGA4a1EEIIJIEwiUIB+QiiLcooAgSSD/nj7ULKk13unbV7q5K9fPlU5/UZdeq1ST91tprr/W+sk1ERAyf1frdgYiImBwJ8BERQyoBPiJiSCXAR0QMqQT4iIghlQAfETGkEuAjIoZUAnxExJBKgJ/GJM2Q9I1+9yMiJkcC/DRmezmwkaQ1+t2XiGje6v3uQPTd7cBlks4D/tx60vaxfetRRDQiAT7urm6rAev1uS+xipH0TGBL2z+UtDawuu0/9btfUSjJxgJA0jq2/zzxkRGFpMOAOcCTbW8haUvgK7b37HPXopI5+GlO0i6SbgB+Vj3eTtKX+tytWDW8B3gR8EcA2zcD/6uvPYoVJMDHvwN/A/wewPY1wIv72qNYVTxke1nrgaTVgb5OCUiaKWmxpBf0sx+DYtoEeElPkHSvpJw+jmL7zlFPLe9LR2JVc7GkDwNrS3oF8C3gv/vcp32BNYDD+tyPgTBtAjzwRmAJ8M5+d2TA3ClpV8CS1pB0JNV0TcQEjgJ+B1wH/B1wAfDRvvYIDgXeAewh6Ql97kvfTZuLrJIuofwj/A6wq+17+tylgSBpQ+ALwMsBARcC/2D7933tWERNkjYFvmt7lqTPAj+zfWq/+9VP0yLAS9oaOMn2bpI+Btxj+4t96ouAg4Bn2z5G0mbAU21fWbOd5wBfBja2va2k5wP72P5k871e9VRnJZvTthTY9tf71qEhJOlFwNHAMyn/nwXY9rP71J+PAffZ/oKkvwa+anu3fvRlUEyXAP854EbbJ1cB9bu2d+xTX74MjAAvs/3Xkp4EXGi71kUhSRcDHwROtL1D9dz1tret2c5GlPnKzVkxGL6jTjuDRNLpwBbAYh67nmDb7+tfrwaDpNWAa+v+OxmnrRuBDwCLaLtu04+zv2rgdBPwQtt/qJ77CXCI7Zumuj+DYug3OkmaCbwBeB6A7V9I+r2k2bYX9qFLO9veUdLVVX/u6TJVwBNsX1n+XT/qkS7aOReYD/yQ4bm4OhvYxtNh9FKT7RFJ10jazPYvemzuPtvfa6RjvVsPeH8ruFfeTTmrmLaGPsADM4HXj9rE8076F8weljSDajlZNYIe6aKdpZK2aGtnP+BXXbTzBNv/p4v3DbLrgafS3f+P6eBpwBJJV7Jieop9OnmzpNbZ70XV2fF3gIfa2rmqwb52xPYfKRd5gUfPVH5ePT9tTYspmnbVlMimtq/t0+cfBBwA7AicBuwHfNT2t2q282xgLrArcA9wG/AW27fXbOeTwOW2L5jw4FWEpIuA7YErWTHwdBTABpmk3SipAU6tBgfr2r6tZhsvGet52xd3+P6LVvKybb+sTn+aIumbwLsog7dFwPrAsbY/14/+DIJpEeAl/RjYh3LGspiytOti20f0qT9bA3tSTh9/ZLvrZYmS1gFW6zb/h6Q/AetQAuHDPHah7Ind9qnfeg1gg0rSxynTT1vZfo6kpwPfsv2iLtpqzyHzBGDGqp5DRtJi29tXg6hZwP8BFtl+fp+71jfTZR38+tWp2uuBU23PoiwLnHKSngz8FvhP4JvAb6rrBHXaeK6kp1UPZwBnSbpD0gF1+2N7Pdur2V7b9hOrx6tscIdHA/ntwMzq/gJgyqcNJsHrKAOVPwPYvpsuEsRVOWTOBk6snnoG8N0u2vkXSRu0PX5SdUbYLzOr36XXAufafpg+76ztt+kS4FevAuIbgfPrvFHSy6o/Xz/WrYu+XEU5g/h/wM3V/dskXSVpVgf9ORb4NNBaEXIq8ANgZ+DDXfQHSc+QtKukF7du3bQzKJoKYANoWXXhuHXdZZ0u22kqh8wrbd/belDtLXlVl31qwomUL/Z1gEuqs5RpPQc/HS6yAhwDzAMus72gmr++ucP3vgT4H+A1Y7xmygWmOr4PnGN7HoCkvYC9gbOAL1EC9cq81PYOkk6X9EvK/PmxVVu1RyuSPkO5JnADbUsKgUvqtjVA3gPsBPwUSgCTNAxJsM6SdCKwQfUl9g7gq12085DtZa0VWD3kkJkhaU3bD1XtrA2s2UU7jbB9PHB821N3SHppv/ozCKbFHPwgkbTQ9uyxnmvNIU7w/muA19q+TdKTWjtyq1UDh9k+cWXvH6O9m4Dnt35J66rm8Mf9R9SP6R5JP7W9s6Srqy/D1YGrhmEutsr5shflWsk82z/ooo3PAvcCBwPvpSwnvMH2R2q28yHKlNGplH8D7wDOs/3Zun1qgqT1gY/zWLK8i4FjbN/Xj/4MgmkR4CVtAnyRclpq4FLKdvy7arSxMfAvwNNtv1LSNsAutk+u2ZcLgR8BZ1ZPHQC8gjKKXzDRBixJrwJOAJ4y+iW6uDgq6XvA/rbvr/O+Mdo5Bvg1cHrVl4OA9frxy95UABtW1WDgUNq+KCg7vbs5A9ybtjQXrTPTfpD0bcoS2dOqp94KbGe7m6nUoTBdAvwPKBc0T6+eegtwkO1X1Gjje5SRykdsb1eNCq+2/byafdmQMsrYjfJLcSnwz8B9wGa2b6nTXq+qX4rtKF867UsKa+36bI2aJ3puKjQZwAZJdc3nM5T5ctHnFU/VNYAHq81TWwFbAd+rLm72oz+POwPu5Kx4mE2XAN/zX7ykBbZf0Drt76aNJkja2vaNbZtNVlB3k4mkt43TzmljPb+Sdi6nnFmcSTlLOhB4j+1d67QzbKpNbZ+2/cEG2roFeE23y2olXcfKp9NqTWFJWgTsDjwJuAJYCDxg+6Bu+tcrldQEH7R9afX4RcDnbe/Sj/4MgulykXWppLdQliZCCT5182X8WdJTeGwFwwspo+5aVJKEHcnjc790ujnkCEqZtH8b4zUDtTaZ1A3kK/FmSlbKL1T9uKx6bspIOsv2G8cLZP2Yg7e9XNIsSWrgDOI3veyZAF7d4+ePJtsPSDoU+KLtz6pKwdEnfw+cVs3FQ9kAeEj/utN/02UEvxnwH0Drm/wyyhz8HTXamEW5Qr8tZZ5vI2C/ujtiq4ukX+HxCZoW1WmnKSp1NP8V2AZYq60/fckI2AtJT7P9q2p53OPU+ftukqR/A7akFMRoTw3Q0QqstuW4L6GkYPguK06n1V3J1Uix7CqYvxs4DjjU9hJJ19WdtmyapCfCo+kLJtXee+/tpUuXdnTsokWL5tnee5K7tIJpMYJ3SarU0zZ124tUdkhuRZn7vKnLucZHbH+5l74AqOw+PIIybz+nCtRb2e5onb+kS11SqZ5KuSZwHPBS4O10kaBJA5CV0vavqj/vGCuATVU/xvBkyhlj+9lVnSW27Ut0H6BcW+imHeDRfQJzqn5tAWxCGXTUrXb2fuAfKct+l1TLj1eWxmBSNbUQoo6lS5eycGFnOQur629TarqM4D8LfBJ4kLIOfTtK5rlv1GhjPmVt+HzKevpuUwMcTdnJeg4rjsL+MN57xmnnvyhnAQe75INfG/hJp9cEJD3R9h8lLXIpkPDoyEvSfNu71+zP5ZT/N6PPTL5dp50mtAcw21tUX35fsZ1yjZRrR1T7BNquJ/V95N2rphZC1DF79mwvWLCgo2NXW221RaOXSE+2aTGCB/ay/SFJrwPuAvanjDQ6DvDA2ygrX94AfE7SQ8B82x+o2ZfWRc32i24G6k6JbGH7AEkHAth+UFKdkfc3KXOyD1Xvu1nS4cAv6W5XYyNZKSU9i7K0cXNWPBOocwY2UBud1FBxFklrUVYHPZcVp9PqniU1stFJJenYWNc6+pJsDNjQ9lmS/rHqxyOSJjVrrIHlI90kg50a0yXAt3K9vAr4T9t/qBcLwfatkh4EllW3lwJ/Xbcjtp9V9z3jWFaN2lsXfbeg7Yygg360Lrh9AFiXkvrgE5Sf6+Au+nO+pFe596yU3wVOphRv7vY3p6mdmk35KlVxFgDb16pkPuwowEuaa3sOZZnvjcDfUHZnH0R39XMv1orFst9NjWLZrYvZlMUCLWtRBj/d1CRoSiMLIeoxHuB0N9MlwP+3SvWZB4F3V/PFf6nTgKSfA0spI9+Tgffarh2AJI0ZPF2/nNzHKdNNm0o6g7KJ65C6/aFMp3ydUnat9UX4VaDuipN/AD5cndn0kpXyLy5bznvRUwCbBL0WZzm2+vOvbO8vaV/bp1VfEt1sLDqKcibQXiz7pBrv/wcYc2HAZSqVxvrlCOA8YAtJl1EthJjUTzSMDG58nx4B3vZRKjlX/lgtW3sA2LdmM8dTpmgOBHagBJFLbP+8ZjvtpfnWolzYuooSZOs4GPi/lKRat1JWBXV2OX9FZ1BGl9fR/YgZ27WzGo7jCyppcS+k+yISvQawFaj3+q69Fmc5hpIor3VR/15J21J2Dm9eox2gVHWifIl3k8eG6n2vVsmM2rIaJZXxU7tss2e2r2poIUTdz53sj+ja0Af4arXJlravaXv6KdSs6GT7C5Tgsy5lpcnRlNUHM2q2895R/Vufx3bY1nEq5QvnFZT5+8XVF84XarbzO9vndfH5QPMbryilFd9KWXHS+sLpeH1/tbHoNNtvofsA1t7emPVdqfeF/B5KcZatVRLE3UaZXunUP1R/nqxSsOajlJHqusA/ddpIg/sEWn1f1NbOI5RMjod22p8mjfo9X1I9t5mk5bZ/OVmfa2BkgAP80K+iUckPfSMlodafq+cuBD7sGjVZJX2esmtvXeAnlBUj823f2kD/rrVdez6/CmYvoMybv4uybXzrmm3sSTkrGZ2qoNM12nOrZZrty+Me/UdV94JbNZX2fNvL6rxvVBvzKDs+u26jra2f0WV9V0mjC8qsTRnptvK5H/u4N628vTUp89yb89h0mm0f0+H7W/sEjqBUu7qz/fW6+wSqa0Dvpgw0TPmd+LLtWtOfTWjq97yuHWfN8vzLL+/o2HXXWiuraJpm+2FJ51CSep2isulpoy7+0o8A3mr7jNYTkuZSluN1TNJ/81gAnEHZYHRWzb4g6UeUvNetL5sX2P5t3XYoZyNbUwJG+4i5owBfXfyDskrk+9XSy3+ilCT8RBf9uQbYgLKUtFu3U+aDz2PFjUW1Amqll/qurWmrrShfxOdSpg7eSnfpmM+lXDRcRI0L6i2tfQJVv04E/kBJLXG27d900Z/TKPnWW9dMDqScje7fRVs9afD3vO7nZhXNADiJcrp+CmXu+tQu2rgdmCNpi7YRUzffxp9nxdPaO7o8hbyWUpZsW8ov/b2SfmL7wZrtbNfQOuGPVkvUWtNG/0YJ+nWTjW0M3ChpATXrqUo63fZbKb/kx1FGy71eG9gQuEGlQHWt/tj+56pfFwI7tvZOVHshatXgrWziBnZCVv3652q55gGU60l32a5b5Wwr29u1Pb5IZad2vzTxe17bIM+CTIsAX80Rt9YjH0g5pazrXsoF0eOrUfhb6rxZj+0cPZ8S4FtLKqxSqOMPwOdsf6mT9lrr79uuCZxKGWnWLbhwhaRtbN9Q832jtean/5ayqejcKpDV9fEe+jBLZQfrLyjpoZtwdANtbEZZWtuyjC4ujgKXS3qe7esa6BOUs6RfU3bZdrNP4GpJL7R9BYCknSlpQPqiod/z+p+bZZID4WTKN/y1ropk1CTbj1CWWR5CSfP7pE7fXAX3cVebVOt3L6dUdZq4M2VT0u6UUfwdlFHL/E7702Y34G2SbqOMUFvLG+suk/ylSrWhlwOfqeaLa5eEdG+Fsb9CWTr6LEpmwxbR3Waypgp1nw5cWU0hmFJbtZskb7sBh/T6dyXp7ykj940oq7AOq/MF33aRdiZwsKRfVI+fSakM1jNJT7X96y7e2uvveS3lIutkf0r3hv4ia0t1lf1XwBts/7CL9/+d26olqSQfe08XuwhX9hlPa5snnejYD1LmcRdVXzzdfmYjibmq/797A9dVO0efBjzP9oUdvv9S27vp8RWiaq+nl/Rl239fp/+T2Z+qvR0pX8gAl9iunXWxwb+rTwNn2l5ctw8r60e3/RnnM/6v7b/t4n09/Z7Xtf2OO/pHl3R2OWXD9dab8ous0ybAR0Q0bfsddvCFF3d2krfx+utnFU1ExKrC5CJrRMTQGuSNTrUvgq3qJNVatz6d2hmkvqSdqWlnkPoyiO10wnZHt36YdgGemhuTplk7g9SXtDM17QxSXwaxnQm44//6IVM0ERFdcrJJTp1qw1DPx82aNWvCNjbbbDNmz5690nYWLeqszGqn/Z6KdgapL2lnatoZpL5McTtLbW/U6+eMJFXBqqXTGosTUc2iIhExpXperz/o2SQT4CMiepBlkhERw8jOCD4iYlgN8gh+ypZJStpA0run6vMiIiabgeV2R7d+mMp18BtQqr9ERAyNbHQqPk2pdr5Y0qmS9gGQdI6kU6r7h0r6ZHX/CEnXV7f3T2E/IyI6lgBfHAX83Pb2wDweS536DErZOij5rudXqXjfTqkG9ELgMEk7TGFfIyIm5Ooiaye3fuhXqoL5wO6StqEUCPhNlT98F0rRi92Ac2z/2fb9lPqgu4/VkKQ5khZKmtTaixERYxnkEXxfVtHY/qWkJ1EKRFwCPBl4I3C/7T+pxg4h23OBudDcDriIiE5lFU3xJ1YsgPwT4P2UAD8fOJLHSs5dArxW0hMkrUMpcdZNObqIiElTVtGMdHTrhykbwdv+vaTLJF0PfI8SsPeyfYukOyij+PnVsVdJ+hpwZfX2k7opcRYRMdmSbKxi+82jnjq5ev5hYJ1Rxx4LHDtFXYuIqK+P8+udyE7WiIgupWRfRMQQSy6aiIghlRH8KqapPO5N/cUnr3zEYLLN8hT8iIgYTv2qt9qJBPiIiB4M8jLJfqUqiIhY5bVW0TSRqkDS3pJuknSLpKPGeH0zSRdJulrStZJeNVGbCfARET1oIsBLmgGcALySknzxwCpXV7uPAmfZ3gF4E/ClifqWKZqIiG41d5F1J+AW27cCSDoT2JeSjPHRTwOeWN1fH7h7okYT4CMiutTgRqdnAHe2Pb6Lki693dHAhZLeS9n5//KJGs0UTURED2rkg9+wldq8us1pa2astdCjvzkOBL5mexPgVcDpklYaw7sawUv6BLDU9heqx58CfgtsQplDMvBJ2/8laQ/gSNuvro79D2Ch7a9Juh04DXgNMBPY3/aNkjYCvgk8BVhASSs8y/bSbvobETFZaiyTXGp79jiv3QVs2vZ4Ex4/BXMoJRZi+yeS1gI2pMTeMXU7gj8ZeBtA9Q3ypqqD2wPbUU4dPlcV8ZjIUts7Al+mpAwG+DjwP9Xz5wCbddnPiIhJZXd2m8ACYEtJz5K0BiWmnjfqmF8AewJI+mtgLeB3K2u0qxG87dsl/b4qo7cxcDWlCtN/2l5OqdB0MfAC4I8TNPed6s9FwOur+7tRcsBj+/uS7hnvzdVpzpzxXo+ImCymmVw0th+RdDilnOkM4BTbSyQdQ5nxOA/438BXJX2g+uhDPMEFgF4usp4EHAI8FTgF2Guc4x5hxTOFtUa9/lD15/K2/qSiU0QMvgZTFdi+ALhg1HMfa7t/A/CiOm32cpH1HMp80Aso3zqXAAdImlHNob+YUrDjDmAbSWtKWp/qFGMCl1JK+CFpL+BJPfQzImJSNLnRaTJ0PYK3vUzSRcC9tpdLOodSNPsays/9Idu/BpB0FnAtcDNlOmci/wz8p6QDgIuBX1FK/kVEDJShzCZZXVx9IbA/QDUX9MHqtgLbHwI+NMbzm7fdXwjsUT28D/ibal5qF+Clth8a/f6IiH4bunzw1Rba84FzbN/cbJeAsmrmrOpLZBlw2CR8RkREjzx82SSryf5nN9yX9vZvBnaYrPYjIprQ4RLIvkmqgoiIHqTgxzQ1SJWhUhUqonlNrYOfLAnwERE9GMpVNBER014f17h3IgE+IqIXCfAREcNpZHkCfETE0CnLJAc3wE9JwQ9J94/z/LskHVzdP0TS06eiPxERTRnKXDRNsP2VtoeHANfTQZ3BiIjBMA0uskr6EPAX28dLOg7YzvbLJO0JvL065lPAq4EHgX1t/0bS0cD9wO3AbOAMSQ9SkpZtAxwLrAsspeQ+/lUT/Y2IaIpHBjfANzVFcwmwe3V/NrCupJmUwh3zKQVir7C9XXXsCrllbJ8NLAQOsr09JYf8F4H9bM+i5Jv/1FgfLGlOq8ZhQz9LRERHWnPwwz5FswiYJWk9SgGPqyiBfnfgfZSEYee3HfuKCdrbCtgW+EG1A3MGJWXw46TgR0T0k4c9VYHth6sC2m8HLqfkfn8psAXwM+DhttJS7ZWbxiNgie1dmuhfRMRkGeAp+EZX0VxCKZp9CWVa5l3A4olqBrb5E7Bedf8mYKMqFzySZkp6boN9jYjonY1HOrv1Q5MBfj7wNOAntn8D/KV6rlNfA74iaTFlSmY/4DOSrgEWA7s22NeIiEZMhzl4bP8ImNn2+Dlt99dtu382cHZ1/+i2578NfLutycWUuq4REQOpVZN1UGUna0REDxLgIyKGkY2XD/kqmoiI6Soj+OjJ1y++pOc2dt99/wZ6AvPnf6uRdiKGxQDH9wT4iIhu5SJrRMSwGvB0wQnwERFdMyO5yBoRMZwGeQQ/JQU/xiJpD0nnT3xkRMRgmi7ZJCMipqdhHcFL2lzSjZJOknS9pDMkvVzSZZJulrRTdbtc0tXVn1uN0c46kk6RtKA6bt/q+edKulLSYknXStqyl/5GRDTNI53d+qGJEfxfAfsDc4AFwJsphT72AT4MHAy82PYjkl4O/AvwhlFtfAT4H9vvkLQBcKWkH1IyUn7B9hmS1qAkIYuIGBiDPAffRIC/zfZ1AJKWAD+ybUnXAZsD6wOnVaNv05aQrM1ewD6SjqwerwVsBvwE+IikTYDv2L559BslzaF8uURETC2bkQEu+NHERdaH2u6PtD0eoXyBfAK4yPa2wGsowXs0AW+wvX1128z2z2x/k3Im8CAwT9LLRr/R9lzbs23PbuBniYjoWGujUxMXWSXtLekmSbdIOmqcY94o6QZJSyR9c6I2p2IVzfrAL6v7h4xzzDzgvarq80naofrz2cCtto8HzgOeP7ldjYiowTRS8EPSDOAE4JXANsCBkrYZdcyWwD8CL7L9XOD9E3VvKgL8Z4F/lXQZ48+hf4IydXOtpOurxwAHANdXRUC2Br4+2Z2NiKilrJWc+LZyOwG32L7V9jLgTGDfUcccBpxg+57ysf7tRI32NAdv+3ZKcezW40PGee05bW/7p+r1HwM/ru4/CPzdGO3/K/CvvfQxImLyNLbG/RnAnW2P7wJ2HnXMcwDaBstH2/7+yhrNOviIiB6MdF5vdUNJC9sez7U9t7qvMY4f3fDqwJbAHsAmwHxJ29q+d7wPTICPiOiSqzn4Di1dyWKQu4BN2x5vAtw9xjFX2H4YuE3STZSAv2C8D+xbqoKIiGHQ0CqaBcCWkp5V7fl5E2VhSbvvAi8FkLQhZcrm1pU1mhF8REQPmpiDrzaCHk5ZUTgDOMX2EknHAAttn1e9tpekG4DlwAdt/35l7SbArwLeseeePbex/wFHTnxQB475j2YWMn3s8IMbaSeiv5pLJGb7AuCCUc99rO2+gSOqW0cS4CMiupWCHxERw8mAlyfAR0QMpYzgIyKGUR+LeXQiAT4iogc11sFPuYEM8JJm2F7e735ERExkkEfwfdnoJOm7khZVKS/nVM/dL+kYST8FdpE0S9LF1XHzJD2tH32NiBhPk+mCJ0O/RvDvsP0HSWsDCyR9G1gHuN72xyTNBC4G9rX9O0kHAJ8C3tGn/kZEPJ6NB7jgR78C/Pskva66vykln8Jy4NvVc1tRMlH+oEoRPwP41VgNpaJTRPRTv+qtdmLKA7ykPYCXA7vYfkDSjylVnv7SNu8uYIntXSZqr8rGNrdqe3AnwyJiKGUOfkXrA/dUwX1r4IVjHHMTsJGkXQAkzZT03KnsZETEhDzYc/D9CPDfB1aXdC2lctMVow+oKprsB3xG0jXAYmDXKe1lRMQEcpF1FNsPUeoOjrbuqOMWAy+ekk5FRHTFjCwf3En4gVwHHxGxSkiysYiIIZYAHxExnAY4vifArwqWL3+k5zbOPeeEBnoCHzjm6kba+eQaazXSzjrrbNBIO/fc8+tG2onppXWRdVAlwEdEdKte0e0plwAfEdE1M5JUBRERwylTNBERwyoBPiJi+HjA5+AbT1Ug6RBJT5/gmHdJOniM5zeXdH3TfYqImCx2Z7d+mIwR/CHA9cDd4x1g+yuT8LkREVNsCGqySjqCx4ptnAR8Fzjf9rbV60dScslcD8wGzpD0ILAL8HFgH+AR4ELbR0o6Grjf9uclzQJOAR4ALm37zBnAp4E9gDWBE2yf2NNPGxHRJLNqr6KpAvDbgZ0pedp/Sqm29Di2z5Z0OHCk7YWSngy8DtjatiWNtSvlVOC9ti+W9Lm25w8F7rP9AklrApdJutD2bbV+woiISWJW/Tn43YBzbP/Z9v3Ad4DdO2z/j8BfgJMkvZ4ySn+UpPWBDWy3vjBOb3t5L+BgSYspXypPoVR+WoGkOZIWSlrYYZ8iIhqzqqcL1hjPbcCKXw5j7ju3/YiknYA9gTcBhwMvG9X2eD+5KCP7eSvrXCo6RUT/9PEKagc6GcFfArxW0hMkrUOZcvke8L8kPaWaPnl12/F/AtYDkLQusL7tC4D3A9u3N2z7XuA+SbtVTx3U9vI84O+rAtxIek71+RERg2HAKzpNOIK3fZWkrwFXVk+dZHuBpGMoUye3ATe2veVrwFeqi6yvBM6VtBZlRP6BMT7i7cApkh6gBPWWk4DNgatUKm//Dnht5z9aRMTkG1k+uCP4jlbR2D4WOHbUc8cDx49x7LeBb7c9tdMYxxzddn8RsF3by0dXz48AH65uEREDJ9kkIyKGVSo6RUQMqyHY6BQREWNLgI++e/DB+xtp552vO7SRdjbYYONG2jnquOMaaeeIg97QSDsvcAnWAAAKG0lEQVTjr/qtZ+bMNRtp55FHHu65jXI5LMazqm90ioiIMbSySXZym4ikvSXdJOkWSUet5Lj9JFnS7InaTICPiOhBE+vgq9xbJ1CWlm8DHChpmzGOWw94H2WJ+oQS4CMiutZZcO9gnn4n4Bbbt9peBpwJ7DvGcZ8APktJATOhBPiIiG41N0XzDODOtsd3Vc89StIOwKa2z++0e7nIGhHRgxqraDYclRRxbpVLC8bO+fVow5JWA46j1NvoWAJ8RESXau5kXWp7vAujdwGbtj3ehBWLJq0HbAv8uGRu4anAeZL2sT1uJt1VKsBLWt32I/3uR0REYdxMwY8FwJaSngX8kpJ9982Pfop9H7Bh67GkH1PV3VhZo5M2B1/VV/2ZpK9KWiLpQklrS9pC0vclLZI0X9LW1fGvkfRTSVdL+qGkjavnj5Y0V9KFwNcnq78REbUZPNLZbaXNlIHr4ZSEiz8DzrK9RNIxkvbptnuTPYLfEjjQ9mGSzgLeQMke+S7bN0vaGfgSJUf8pcALq8pP7wQ+BPzvqp1ZwG62H5zk/kZE1NLUTtYqrfoFo5772DjH7tFJm5Md4G+zvbi6v4iS/ndX4FvVPBKUeqtQ5pz+S9LTgDUoaYhbzhsvuEuaA8xpuN8RER2ZzqkKHmq7vxzYGLjX9vZjHPtF4Fjb50nagyptcOXP431AKjpFRL8MerrgqV4H/0fgNkn7A6ho5YJfn3JxAeBtU9yviIj6bEaWj3R064d+bHQ6CDhU0jXAEh7brXU0ZepmPrC0D/2KiKjP7uzWB5M2RWP7dsq6zdbjz7e9vPcYx58LnDvG80dPQvciIhrhhjKIToZVah18RMQgcSo6RUQMKw90vvwE+IiIHmQEHwOgmX+ES5Zc2kg7q68+s5F2LvtOM/1p25fRYzvNrFt4y9v/sZF25p3/jZ7buPvuWxroyfAaaSZVwaRIgI+I6FLJ9Z4AHxExnDJFExExnLJMMiJiSOUia0TEUDIjI8v73YlxJcBHRHQpG50alIpOETFopmWAl7Q58D1KIY9dKZki9wWeDpwAbAQ8ABxm+0ZJrwE+SskF/3vgINu/kXR09Z7NKUnI3kxExIAY5AA/2dkktwROsP1c4F5KRae5wHttzwKOpFR0gscqOu0AnEmp6NQyC9jXdoJ7RAyQDjNJDls2yUoqOkXEUDPTd6NTKjpFxNCyBztVQSo6RUR0zVW6golv/ZCKThERPbBHOrr1Qyo6RUT0YJBX0axS6+AjIgZNAnxExDDq4xLITiTAR0R0ycCIk4smhkRTiZWWLWvmotO8eac20k5Tp9lNtfPMbTdvpJ1Zd/9Nz23cfffPG+gJNFVVbLD0b4VMJxLgIyJ6kAAfETGkEuAjIoZQucY6uDtZE+AjIrpmPMCpChLgIyJ6MMg1WTtKVSBpc0nX9/JBkvaQdH4vbUREDJpBzkWTEXxERNc80HPwdZKNrS7pNEnXSjpb0hMk3S5pQwBJsyX9uLr/EkmLq9vVktar2li3eu+Nks5QlRRe0ixJF0taJGlelRMeSe+TdEP1mWc2+HNHRPSsVZO1iRG8pL0l3STpFklHjfH6EW3x8EeSnjlRm3UC/FbAXNvPp6T9ffdKjj0SeE+V9313oFWsYwfg/cA2wLOBF0maSckFv19V5ekU4FPV8UcBO1Sf+a4afY2ImBJNBHhJMyilTF9JiY8HStpm1GFXA7OreHg28NmJ+lYnwN9p+7Lq/jeA3VZy7GXAsZLeB2zQVij7Stt3uZzTLKZUeNqKknXyB5IWU+qyblIdfy1whqS3AGMW25Y0R9JCSQtr/CwREY0YGRnp6DaBnYBbbN9qexmlbOm+7QfYvsj2A9XDK3gsTo6rToAf/RVkStBttbFWW0c+DbwTWBu4QtLW1UujKzytDghYYnv76vY823tVx/wt5VttFrBI0uOuGdiea3u27dk1fpaIiAYYPNLZbeWeAdzZ9viu6rnxHAp8b6JG6wT4zSTtUt0/kFIk+3ZK8IVSUBsASVvYvs72Z4CFwNaM7yZgo1bbkmZKeq6k1YBNbV9EKcC9AbBujf5GREw6d/gfsGFrtqG6tdeS1phNj6Ga0ZgNfG6ivtVZRfMz4G2STgRuBr4MXAmcLOnDwE/bjn2/pJdSRuk3UL5pdmEMtpdJ2g84XtL6VZ/+Hfh/wDeq5wQcZ/veGv2NiJhUrYusHVq6kpmGu4BN2x5vAtw9+iBJLwc+ArzE9kOjXx+towBfVWcaPeEPMB94zhjHv3eMY39c3VrHHN52fzHw4jHes7J5/oiIvmtojfsCYEtJz6LUpn4T8Ob2AyTtAJwI7G37t500mnXwERFda2YdvO1HJB0OzANmAKfYXiLpGGCh7fMoUzLrUmpXA/zC9j4razcBPiKiBx2skOmI7QuAC0Y997G2+y+v22YCfEREl2rOwU+5BPhYpS1fPub2iFXegnlXNNLO7L127rmNeReu2UBPYNmyvzTSzmBJTdaIiKFlBjcXTQJ8REQPMkUTETGU3NhF1smQAB8R0aWU7IuIGGKZoomIGFIJ8BERQynLJCMihtYgF91e5QN8lXJzzoQHRkQ0zIaRkeX97sa4VvkAb3suMBdA0uB+lUbEEOqs3mq/rPIBPiKinxLgIyKG1CAH+Dol+/pK0gWSnt7vfkREtLNHOrr1wyozgrf9qn73ISJiBc4yyYiIoWRgJKkKIiKGU3LRREQMpcFeJqlB7lxdkn4H3DHBYRsCSxv4uGFsZ5D6knampp1B6stUt/NM2xv18iEzZ67hDTbYuKNjly69a5Ht2b18Xl1DNYLv5C9L0sIm/icPYzuD1Je0MzXtDFJfBrGdiaQma0TE0DJOqoKIiOGUZGODZW7amdQ20s6q1c4g9WUQ25nQIE/RDNVF1oiIqbT66jO97rpP6ujY++77XS6yRkSsKmxnHXxExLAa5FmQBPiIiB6MjGQEHxExnDKCj4gYRsZkBB8RMXSykzUiYoglwEdEDKkE+IiIoWRGkosmImL4DPoc/CpTdDsiYiC16rJOdJuApL0l3STpFklHjfH6mpL+q3r9p5I2n6jNBPiIiK654/9WRtIM4ATglcA2wIGSthl12KHAPbb/CjgO+MxEvUuAj4jogT3S0W0COwG32L7V9jLgTGDfUcfsC5xW3T8b2FOSVtZo5uAjInrQUKqCZwB3tj2+C9h5vGNsPyLpPuAprKQ0YQJ8RET35lHqv3ZiLUkL2x7Ptd3KWz/WSHz0vE4nx6wgAT4ioku2926oqbuATdsebwLcPc4xd0laHVgf+MPKGs0cfERE/y0AtpT0LElrAG8Czht1zHnA26r7+wH/4wnWaGYEHxHRZ9Wc+uGUKZ8ZwCm2l0g6Blho+zzgZOB0SbdQRu5vmqjdlOyLiBhSmaKJiBhSCfAREUMqAT4iYkglwEdEDKkE+IiIIZUAHxExpBLgIyKGVAJ8RMSQ+v8ibkjXnBw7OQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translation, attention = translate_sentence(candidate)\n",
    "\n",
    "print(translation)\n",
    "\n",
    "display_attention(candidate, translation, attention)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eine gruppe von männern lädt baumwolle auf einen lastwagen\n",
      "a group of men are loading cotton onto a truck\n"
     ]
    }
   ],
   "source": [
    "candidate = ' '.join(vars(valid_data.examples[0])['src'])\n",
    "candidate_translation = ' '.join(vars(valid_data.examples[0])['trg'])\n",
    "\n",
    "print(candidate)\n",
    "print(candidate_translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'group', 'of', 'men', 'loading', 'loading', 'a', 'truck', '.', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAErCAYAAADKTqhMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XmcXHWd7vHPk7CEXYfEjV1FGER2WQQFFBhQAVkEEWYEFUYdxWXUcZmrwIhXZdA7KjoGRNQr4AYYHQQF2RFI2Ndckc2IAkEEZElI+rl/nFOk0nS6q7rP6dqeN696ddWpc77nR7r627/+rbJNRET0jymdLkBERFQriT0ios8ksUdE9Jkk9oiIPpPEHhHRZ5LYIyL6TBJ7RESfSWKPiOgzSewREX0miT2iTZJeKOnbkn5Zvt5E0rs6Xa6IhiT2iPadBpwPvKR8/f+AD3WsNBHDJLFHtG+67R8BQwC2FwGLO1ukiCWW63QBInrQE5LWBAwgaXvg0c4WqftIeg2wPk15xvb3OlagAZLEHtG+jwCzgJdJugKYARzY2SJ1F0nfB14G3MCSv2YMJLFPAmXZ3oj2SVoO2AgQMNf2Mx0uUleRdDuwiZNgOiI19ogWSdp/GW+9QhK2z5rUAnW3W4AXAX/qdEEGURJ7ROv2HuU9A0nsS0wHbpN0DbCgcdD2Pp0r0uBIU0xEVE7SziMdt33JZJdlECWxR0dIWgtYj6VHTFzauRKNTdJHRnvf9pcnqyy9QNJ6wIa2L5C0MjDV9uOdLtcgSFNMTDpJXwQOBm5j6RETXZ3YgdU6XYBeIelI4Cjg7yhGx6wF/Dfwhk6Wa1Ckxh6TTtJcYDPbC8Y8OXqSpBuAbYGrbW9ZHrvZ9qs6W7LBkJmn0Ql3Act3uhDjJWltSWdLelDSA5J+KmntTperyyywvbDxohwemlrkJElTTHTCk8ANki5k6RETR3euSG35DnA68Nby9WHlsd07VqLuc4mkTwErSdodeB/w8w6XaWCkKSYmnaR3jHTc9ncnuyzjIekG21uMdWyQSZoCvAvYg2IS1/nAKZmwNDlSY49JJWkqsLvtwzpdlgmYL+kw4Izy9SHAwx0sT9exPQScXD5ikiWxx6SyvVjSDEkrNLfB9ph3Al8HvkLRbnxleSxKkm7muW3qjwJzgM/Zzi/CGqUpJiadpG8BW1EspPVE43ivjAOXNM32050uRzeT9CWKoaynl4feVn59DNjJ9mizeGOCUmOPTri/fEyhN8eG3yLpAeAyirH3V9jOsr1L29H2jk2vb5Z0he0dy2asqFESe4+R9Argm8ALbW8qaTNgH9uf63DRWmb7WABJq9h+Yqzzu43tl0taF3gt8GbgG5L+ms7TpawqaTvbVwNI2hZYtXxvUeeKNRgyjr33nAx8EngGwPZNLPkztydI2kHSbcDt5evNJX2jw8VqWTlmfUeKxL4lcCvww44Wqvu8GzhF0t2S7gFOAY6UtArwvztasgGQNvYeI2m27VdLur5pRl9PDbWTdDXFxhSzmv4fbrG9aWdL1hpJQ8Bs4PO2f9bp8nQzSWtQ5Jm/1hB7eYrvw5G2Z1cdv5cNZFNMuSDR/cABti/sdHnaNF/Sy1iyLduB9OCa17b/IKn5UC/tGbolsBPwdkmfAH4HXGL7250tVneR9CbglcC0xvfa9nEV3mJfYAXgSIoEH6WBTOzAQRR/Pr8b6LXE/i/ATGBjSX8E7gYO7WyR2vaHcj9MS1oBOJqyWaYX2L5R0u+B31M0xxwGvA5IYi9J+m9gZWBXimaYA4FrKr7NuyiGmX5P0sq2n6w4fs8ayKYYSZcC/0yxMcJrbD/S4SK1rWyrnNKLy6BKmg78F7AbxazEXwEf7JWxzZLmACtSjF+/HLjU9r2dLVV3kXST7c2avq4KnGV7j4rirwOcY3vrcmjl7ba/U0XsfjBwNXZJG1MkxNslnUFR2/pah4vVMklrAp+laAqwpMuB43olKQLYnk/v/ZXRbC/bD3W6EF3uqfLrk5JeQjEzd4MK4x/Bko2xv0MxqCCJvTRwiZ3iz7fGB+A04Bx6KLEDZ1KMnT6gfH0oxYiM3TpWojZJmkHRLro+S2+00SuzN5+RdDTPLX+vLGI2GX4h6XnACcB1FH1ClSwvoKLB/jBge4CykjZV0ka251Zxj143UE0xZS/6XOBVjfHTkn4NfNL2nI4WrkWSrrW99bBjc2xv06kytUvSlRSTe66lqdPU9k87Vqg2lOW/CrgZGGoc76FFzPYHvgi8gKIpTIBtr17T/VYEplU1iUvS6hSzV89tOrYl8JTtO6q4R68btMS+MvAK2zc0HVsPWGx7XudK1jpJ/0mx3saPykMHAq+0/dnOlao9vTY8czhJ19neqtPlGC9JdwJ7266tw1pSY1buZRQzc2vrCypXklzV9mN13aPXDFRiH07S84F1ykk+PUHS48AqLKkpTmHJeiu11bqqJOlzwJXNNa5eIunDwN+AX7D0evJ/6Vih2tCY2l/zPV5K0Q/0WoomkwXAZbY/XFH804H3UPzFdy2wBvBl2ydUEb/XDVxil3QxsA9F2+gNwEMUY5BH3ag4qtP0y2kBxQzaWpsCqibpX4Djgb+yZAVD235p50rVOkn/BbyIon+p+RfTWRXf58XAzhTJfVfgPtt7VhT7BttbSDoU2Br4N+Ba25tVEb/XDWLn6Rq2H5P0buA7tj8rqWdq7PBsG+lOFEnlMtvndLhIbbHdiwt/NfsI8PJydE8vWp1iF6vmoYemGP5biXKc/3yK1R2/DXygXKO9KsuXfWZvAb5u+xlJg1VLHcUgJvblyprEQcCnO12YdpVrqrycJZs8vEfS7rb/pYPFapuktYD1WHpUyaWdK1FbbqVIjD3J9hGTcJuvUlQ+DqGYqXuJpEtt/76i+N8C7gFuBC4t+8rSxl4axKaYtwL/i6JD571lW+AJtg8Y49JW49e6+qKkW4FNG1uMlR1HN9t+ZRXxJ4OkLwIHA7exZFSMbe/TuVK1TtLZFFPlL6IH92ydzBVCy4lJRwAfBda2PbXqezTdaznbWTmSAUzsdZN0CfAx4Ft1LHAl6Szgw42ZjmVN5Qu2D6ki/mSQNBfYzPaCMU8eX/wVh8ce6dgE4vf6nq21fkbLeCdS1NhXBX5LMTrmMtt3VRR/DYqJeq8rD11CMVEv6+IzgE0x5ZKrX6NYdtUUU8I/WOFwx5VtXzNsgasqaxFrArdLaqy78Wrgt5JmAfRIrfcuYHmaarsV+y3FDk1jHRuXXkngo6j7MwrFOP8v2X6g4rgNpwK3UDSpAvwjxcTD/Wu6X08ZuMRO8c0/HXhr+fqw8tjuFcWve/XFz1QYa0TlAl3rs3T79/eWeUH7ngRukHQhFTZlSHoRsBawUjlhpZG5VqdYkKoSku7muft50iujYpicFULfY/vHzQckXWj7DRXFf9mw5tNjJd2wzLMHzCAm9hnDFgs6TdKHKow/0uqLlW0FZvuSqmKNRNL3gZdRDAV9tv2bJetyVGFW+ajaPwCHA2sDzfunPg58qsL7NM/ynUZRSfi7iQaVtKPtK6psNlqG2j6jkqZR/BKdXs4Taf7l+pIq7lF6StJOti8v77sjS9anGXgD18Yu6QKKNWIao0oOAY6osCbRuE8tqy+WY8Ab37QVKJo0nqhqDLik24FN3MMfDEkHTPbyBJIut73TBGNcW65WOCkzW+v4jEr6IPAhiiT+R5Yk9seAk21/vaL7bAF8l2JiEsAjwOG2b6wifq8bxMS+LvB1YIfy0BUUbeyVLLtarotxAM9tyqhyg4Hm+70F2NZ2JTVSST8GjrZd2+Ydkjak2B5tE4oaLzDxpgxJo04ys/3l0d5v4z7NSXcKRQ3+vbY3n2DcqyjWpX8jI2y1V9Wom8n4jEr6gO3aF9cr140hywksbeCaYmzfRzHztC4/Ax6lmOZc55/TANg+R8UuPlWZDtxWds42t39P+N+sqVb7HYoRDV+hmJF4BEtqdhPRmPi0EUWncqO5Z2+KdUuqcmLT80UU46kPGvnUtryZYpXO11N8fuoyGZ/RP0tazfbjkv6douP6c7avqyK4pBcCnwdeYnsvSZsAOzi7WAGDWWP/EvA5iva484DNgQ/Z/r8Vxa91785y1mlDo7a4s+0dlnFJu/F3Hul4FW37klYvZ/02mhxutv2q8r3LbL92ovcoY/2KYtvDx8vXqwE/rmo6e90kbV5nk0Ldn9HyHo0NNnai+OvsP4FP2d6uovi/pKggfNr25pKWA65vfJ4G3cDV2IE9bH9c0n7APIqOr4uAShI7cKWkV9m+uaJ4w+3d9LxRW9y3quC2LylrQ68uD11j+8GKwp9OUStdoGKs3e8kvZ+iLfYFFd0DYF1gYdPrhRTNDpVR036ejWMVNmV8aKTp8a5uvfq6P6OwpOP9TcA3bf9M0jEVxp9u+0eSPglge5GkXto3t1aDmNiXL7++ETjD9l+GjeedqJ2Aw8shcQtYssDVhBcnkjQVuMn2VyYaa5R7HESxOcLFFGX/mqSP2f7JRGPbfnP59MMUE1eOBv6DojnmnyYav8n3gWvKGaIG9qPCUT2qfz/PXzQ9n0ZR/vsrjF/bZ7TJHyV9i6Jp6Ytlu/6UCuM/oWI3scaQze0pmpeCwWyK+QLFwkFPAdsCzwN+UeGfiOuNdLzCztmLbO9aRaxlxL8R2L1RS1ex29EFE+0YHHaPbSjW6VmPJb9oK00skramSGBQ7El6fYWxa93Pc4T7TaH4Hry+oni1fkbLe6wM7Emx3MXvVKzP9Crbv6oo/lYUEw03pZioNAM40D20BHedBi6xw7PrsD9me3E55Gs123+eYMxG+/GI45ld0Vrdko6nGOL1Q5asw06FnVLPtnuXr6cAN1bZdlkuKfAxnrsDUaUbQkt6AUs3ldxXUdyrbW9XjmLZn2I/z1tsb1hF/BHutxHwP7ZfPsE4k/IZHXbPWr4HZezlKDrKBcy1/UxVsXvdQDXFlLWIDYd1TK1J0/ZsE9BoP76WJePMG208Bqqalfia8uuxTfcwxUiKKvxS0vksGed/MFD1hhgP2a58gpKkrWxfJ2kfipErLwEepGhzv4OiTbwKjf08v8SS0SunVBR7+FwFAw8AH68g9GR9RqnzezDs5/jW8ti6khbb/uNE4/eDgaqxq1i/+Q6KBagae57+iqK3vpI9T8sa7qHABraPK8fNv9j21RXF/1eKH8LmH8jHgDlu2vJvAvE/QrH5yBblPS6zffZE4w67xxsoJoYNX1JgQuuBS/o/tj+kYmr5GyiaL7aUtCtwiO2jJhK/6T4rAe+l2EDCFAtcfdP201XEL+/xd8CGLKnt2hUta1z3Z7S8x40UlY3KvweT8XPc66rszOh65Z9qZ1PUQhuTlWZU/GE4iWIrsMZqi49TTIiqytYUW4K9mKI2dBTFLjUnS6qiVrca8AmK/offA1dWEHO4Iyh+cexJMcpnb4qa5EQ1/gpYZPthYIqkKbYvKu9Xle9S1Dy/StHO+/dU2zn7borVCs8Djmn6WpW6P6MAz9T1PZikn+OeNlBNMaVTgJMpVof7J4qxsFXazvZWkq4HsP2IpBUqjL8msJXtvwFI+izwE4rlS6+laB4YN9vHUiyotBnFD84lkubZ3m1ixV7K5jWNN34d8Bvg0bLv5FLgB5IepNrVCzca1pl8UVlDrcoHKYabXmV7V0kbs6TprQp1f0YB/lp2Ktf1Paj757inDVSNHcD2HUBjs4FDKIbGVemZclhiYxjWDJo6CCswfIz2M8B6tp+i2lmEDwJ/pugYrHKMOcBV5UzBStk+pnz6Fop/iw9T1HZ/z9Lj/yfq+nJ4HQCStqNYmqIqTzeadVQsCHYHRSdhVer+jEIxt+IpavoeTMLPcU8bxBo7FHswnkIxJvyRimN/leLPxBeUI1gOBP69wvinUyTGn5Wv9wbOKGuot000uKT3UtTUZ1D8JXCk7QnHHWYn4B11jaP20otaVbZ2uqSbKZLh8sA/SbqvfL0eFfzbN5lXds6eA/xa0iNUO4697s8ojbbvUl3r19f5c9zTBqrztKHsVf8TxbTzC2qIvzFF552AC23fXnH8xhhtAZdX2bZYjvM/s4qO2FHuUcs46mGjSZZ6qwg/sRUwl1XuhqqHa5b33JlieOt5theOdX4bcWv5jNb9PRh2r1p/jnvZQCb2iIh+NnBt7BER/W6gE7ukSsY1dyr+ZNwj8fs7/mTco9fj96KBTuwUY8B7Of5k3CPx+zv+ZNyj1+P3nEFP7BERfaevOk81whrWveSVm7U/2u+Rhx/m+Wuu2fL5c2+7o634Q0NDTJnS+u//RYsqG7gRUZf5tmdMJMCee+7p+fPnt3Tutddee74neZOXQR3HPk6Vrtv+HD8977xa4wPsstWE9lse05//fFet8SMqMOFhqfPnz2fOnNZGGUuaPtH7tSuJPSJiHLq5tSOJPSKiTQYWD1W9CkN1ktgjItpmPOIE2+6QxB4R0S7DUPfm9ST2iIjxSBt7REQfMTCUxB4R0V9SY4+I6CO2MyomIqLfdHONvevXipF0jqRrJd2aVdwiolu4xf86oRdq7O+0/RdJKwGzJf203P0ceHbJziT8iJg0Redpp0uxbL2Q2I+WtF/5fB1gQ4oNlgGwPROYCb2/CFhE9I5uborp6sQuaRdgN2AH209KuhiY1tFCRUSk83RC1gAeKZP6xsD2nS5QRIRJjX0izgPeI+kmYC5wVYfLExEBZILSuNleAOzV6XJERAyXGntERF/J6o4REX3FWd0xIqL/DGVUTERE/8jqjhERfSidpxER/cROjb1/1PuN3GDGjFrjAzz88B9rv0fEIEiNPSKijxhYnMQeEdFfUmOPiOgzSewREX3E6TyNiOg/qbFHRPSZJPaIiD5SjIrJkgIREX0li4BFRPQTO00xkpazvWgy7hURUbeB2BpP0v8CDgX+AMwHrgXeDFwJ7AjMkvQT4FRgBvAQcITt+ySdBvzC9k/KWH+zvWq5kfVxwMPARsClwPvsLm7YioiB0dfDHSVtAxwAbFnGu44isQM8z/bO5Xk/B75n+7uS3gl8FXjLGOG3BTYB7qXY/3R/4CcTLXNExER1c419SgUxdgJ+Zvsp248DP29674dNz3cATi+ff7+8bizX2L7L9mLgjJGukXSUpDmS5oyv+BER7bHN4qGhlh6dUEVTjEZ574lR3mv8ultE+QtGkoAVRjhnWa+xPROYWV7fvb9CI6KvdPOep1XU2C8H9pY0TdKqwJuWcd6VwNvK54eW1wHcA2xdPt8XWL7pmm0lbSBpCnBw0zURER015NYeY5G0p6S5ku6U9IkR3l9X0kWSrpd0k6Q3jhVzwond9mxgFnAjcBYwB3h0hFOPBo6QdBPwj8AHy+MnAztLugbYjqVr+b8FvgDcAtwNnD3R8kZETFRjVEwrj9FImgqcBOxF0Z94iKRNhp3278CPbG9JUTn+xljlq2q443/aPkbSyhSjV060fXLzCbbvAV4//ELbDwDbNx36ZNPzJ20fXFEZIyIqU1Hn6bbAnbbvApB0JkXLxW3NtwJWL5+vAdw/VtCqEvvM8rfMNOC7tq+rKG5ERPcpO08rsBbFMPGGeRQtF82OAX4l6QPAKsBuYwWtJLHbfnsVcYbFvBi4uOq4ERET1eYEpenDRu3NLAd9wMiDT4YHPgQ4zfaJknYAvi9p09Hm9GRJgYiIcWhjgtJ829ss4715wDpNr9fmuU0t7wL2BLD9W0nTgOnAg8u6YRWjYiIiBo5b/G8Ms4ENy9F/K1B0js4ads59wBsAJP09RZP3Q6MFTY09ImIcqug7tb1I0vuB84GpwKm2b5V0HDDH9izgX4GTJX2YopnmcI/RDpTEHhHRJlPdWjG2zwXOHXbsM03Pb6NYc6tlSexdZJWVVqn9Hq/YcFlNfdU44fSTao2/99b1ln9oaHGt8aNPVDcqphZJ7BERbRqIZXsjIgZNEntERJ/p6/XYIyIGT0tDGTsmiT0iok12NcMd65LEHhExDhkVExHRR6ocx16HJPaIiHHo5lExPbFWjKSjJd0u6QedLktEBC1ustGp5N8rNfb3AXvZvrvTBYmIALq697TrErukjwDvLF+eAmwMvBSYJelU21/pWOEiIkpDi5PYWyJpa+AIih1EBFwNHEaxFvGutud3sHgREUBjuGMSe6t2As62/QSApLOA1452gaSjgKMmoWwREc9KYm/dSNtEjarcYmomgKTu/ZeOiD7SuY7RVnTbqJhLgbdIWlnSKsB+wGUdLlNExHN4yC09OqGrauy2r5N0GnBNeegU29dLbVfkIyJqkzb2Ntn+MvDlYcfW70xpIiJG5iwpEBHRX7q4wp7EHhHRNneu/bwVSewREeOQNvaIiD6SPU8jIvpQEntERD+x8eKMiomI6CupsUdLFi1aWPs9brv9t7XGf+MWW9Qaf2hoca3xI1rVxXk9iT0iol3pPI2I6DdZUiAiot+YoXSeRkT0l9TYIyL6SFZ3jIjoR0nsERH9xd3bxJ7EHhExHmmKiYjoJzZDXbzRRq17nkpaX9Idkk6RdIukH0jaTdIVkn4naVtJq0g6VdJsSddL2re89nBJZ0k6rzz3S3WWNSKiVY0JSq08xiJpT0lzJd0p6RPLOOcgSbdJulXS6WPFnIwa+8uBtwJHAbOBtwM7AfsAnwJuA35j+52SngdcI+mC8totgC2BBcBcSV+z/YdJKHNExLKZSjbakDQVOAnYHZgHzJY0y/ZtTedsCHwS2NH2I5JeMFbcyUjsd9u+GUDSrcCFti3pZmB9YG1gH0kfLc+fBqxbPr/Q9qPltbcB6wFLJXZJR1H80oiImDzVtLFvC9xp+y4ASWcC+1JUeBuOBE6y/UhxWz84VtDJSOwLmp4PNb0eKu+/GDjA9tzmiyRtN+zaxYxQXtszgZnlNd3bmxERfaS1ZpbSdElzml7PLPMWwFosXVmdB2w37PpXAEi6ApgKHGP7vNFu2A2dp+cDH5D0gbImv6Xt6ztdqIiI0Qy13hQz3/Y2y3hPIxwbHng5YENgF4oWjsskbWr7r8u6Ya2dpy36D2B54CZJt5SvIyK6lss29lYeY5gHrNP0em3g/hHO+ZntZ2zfDcylSPTLVGuN3fY9wKZNrw9fxnv/PMK1pwGnNb1+cy2FjIgYh4rGsc8GNpS0AfBH4G0UA0yanQMcApwmaTpF08xdowXthqaYiIieU0Vit71I0vspmqSnAqfavlXSccAc27PK9/YoB5AsBj5m++HR4iaxR0S0ra3O09Ej2ecC5w479pmm5wY+Uj5aksQeEdGurO4YEdFfDHhxEntERF9JjT0iop+0uA5MpySxR0SMQxVrxdQliX3g1PthlOqd81Z3LUkaaSJgxHOlxh4R0Ucay/Z2qyT2iIh22biLN9pIYo+IGIfseRoR0WfSFBMR0U8y8zQior+k8zQiou+YocXd28g+5qBjSX+r4kaSjmnsayrpOEm7VRE3ImLSlU0xrTw6oSM19uYlKSMielIXN8W0PE1QhRMk3SLpZkkHl8dXlXShpOvK4/s2XfNpSXMlXQBs1HT8NEkHls/vkXRs0/Ubl8dnSPp1efxbku4tdw+JiOg4u7VHJ7Qz/3t/YAtgc2A34ARJLwaeBvazvRWwK3Bi+Utga4ptnrYsr331KLHnl9d/E/hoeeyzwG/K42cD67ZR1oiI2jQ6T7u1KaadxL4TcIbtxbYfAC6hSNYCPi/pJuACYC3ghcBrgbNtP2n7MWDWKLHPKr9eC6zfdL8zAWyfBzwy0oWSjpI0R9KcNv5fIiLGr7rNrGvRThv7slZHOhSYAWxt+xlJ9wDTyvda/b9aUH5d3FSmllZjsj0TmAkgqXsbvSKij5ihLl5SoJ0a+6XAwZKmSpoBvA64BlgDeLBM6rsC6zWdv5+klSStBuzdZtkuBw4CkLQH8Pw2r4+IqE03N8W0U2M/G9gBuJGiJv5x23+W9APg52VTyA3AHQC2r5P0w/LYvcBlbZbtWOCMspP2EuBPwONtxoiIqEcXj4oZM7HbXrX8auBj5aP5/fkUCX+ka48Hjh/h+OFNz9dvej4H2KV8+SjwD7YXSdoB2NX2AiIiOszORhvjtS7wIxU7NywEjuxweSIintXFFfbuTey2f0cxVDIiostkz9OIiP5iunpUTBJ7RESbTNrYIyL6TppiIiL6SgcXgmlBEntERLuyg1IMkilT2pnM3L7Hnnqq1vgrrrBSrfEXLKy3/DF5hhYnsUdE9I1sjRcR0W/SFBMR0W8yQSkiou90c2Kvt6crIqJPVbXRhqQ9yy1E75T0iVHOO1CSJW0zVszU2CMi2lTV6o6SpgInAbsD84DZkmbZvm3YeasBRwNXtxI3NfaIiHGoaKONbYE7bd9leyHFdqD7jnDefwBfothjekxjJnZJf2slUAtxjpH00fL5cZJ2qyJuRMTkay2pt5DY1wL+0PR6XnnsWZK2BNax/YtWS9eRphjbn+nEfSMiKtFeU8z0coe5hpnlXs0w8t7OzwYu96P4CnB4O8VruSlGhRMk3SLp5nLLOiStKulCSdeVx/dtuubTZafABcBGTcdPk3Rg+fweScc2Xb9xeXyGpF+Xx78l6V5J09v5n4uIqEsbNfb5trdpesxsCjMPWKfp9drA/U2vVwM2BS6WdA+wPTBrrA7UdtrY9we2ADYHdgNOkPRiijaf/WxvBewKnFj+EtgaeBvFZhn7A68eJfb88vpvAh8tj30W+E15/GyKHZUiIjquMfO0gqaY2cCGkjaQtAJFzpz17H3sR21Pt71+uY3oVcA+5Taiy9ROYt8JOMP2YtsPUGww/WqKPyU+L+km4AKK9qEXAq8Fzrb9pO3Hmgs7grPKr9cC6zfd78zyf+484JGRLpR0lKQ5w/7UiYiokfHQUEuPUaPYi4D3A+cDtwM/sn1r2Q+5z3hL104b+0htQQCHAjOArW0/U/65MK18r9VGqMYm1YubyrSs+y2l/LNmJoCk7p0xEBH9w+CKNlCyfS5w7rBjI/ZD2t6llZjt1NgvBQ6WNFXSDOB1wDXAGsCDZVLfFViv6fz9JK1UjsHcu417AVwOHAQgaQ/g+W1eHxFRm4qaYmrRTo39bGAH4EaKmvjHbf9Z0g/ge6qdAAAHx0lEQVSAn5dNITcAdwDYvk7SD8tj9wKXtVm2Y4Ezyk7aS4A/AY+3GSMiohbdvKTAmInd9qrlVwMfKx/N78+nSPgjXXs8cPwIxw9ver5+0/M5wC7ly0eBf7C9SNIOwK62FxAR0WFZtnf81gV+VI7jXAgc2eHyREQUbIYWV9TIXoOuTey2f0cxVDIiovukxh4R0V/c8qC/yZfEHhHRJmcHpYiIfmNc1UD2GiSxR0SMQ2rsERF9ZmiM5QI6KYk9KjU0tLjW+HvselCt8Y8/+Xu1xv/8h4+uNf7jj/+l1vhQf0110aKFtcavQjGrNIk9IqK/pCkmIqK/ZLhjRESfSedpRERfce39SRORxB4R0aZMUIqI6ENJ7BERfSaJPSKirzjDHSMi+o3JBKWIiL5hd/eSAu1sZt0Rks6RdK2kWyUd1enyREQUqzv2x2bWnfJO23+RtBIwW9JPbT/ceLNM9kn4ETGpslbMxBwtab/y+TrAhsCzid32TGAmgKTu7c2IiL6SUTHjJGkXYDdgB9tPSroYmNbRQkVEkMQ+EWsAj5RJfWNg+04XKCKinHra6VIsU7cn9vOA90i6CZgLXNXh8kREYGDIWStmXGwvAPbqdDkiIpbWuREvrejqxB4R0a2S2CMi+kwSe0REHyn6TjOOPSKijxh38ZICSewREeOQPU8jIvpM2tgjIvqK08YeUZXbb693jtpXPjW31vgvetFLa42/+urTa40PsNcBh9Ua/5sn/lut8atQ5Z6nkvYE/guYCpxi+wvD3v8I8G5gEfAQxcKI944Ws+uX7Y2I6EZVLNsraSpwEsVEzE2AQyRtMuy064FtbG8G/AT40lhlS2KPiBiHoaGhlh5j2Ba40/ZdthcCZwL7Np9g+yLbT5YvrwLWHitoEntERNsMHmrtMbq1gD80vZ5XHluWdwG/HCto2tgjIsahjeGO0yXNaXo9s9xHAkAjhh6BpMOAbYCdx7phEntERJva7Dydb3ubZbw3j2IDoYa1gfuHnyRpN+DTwM7l4oijSmKPiBiHikbFzAY2lLQB8EfgbcDbm0+QtCXwLWBP2w+2EjSJPSKibdWMY7e9SNL7gfMphjueavtWSccBc2zPAk4AVgV+LAngPtv7jBa30sQu6XnA221/o4JYuwAftf3mCRcsIqJiLYx4aYntc4Fzhx37TNPz3dqNWfWomOcB7xt+sByrGRHRFxpt7BMdx16XqhP7F4CXSbpB0mxJF0k6HbhZ0vqSbmmcKOmjko4pn79c0gWSbpR0naSXNQeV9GpJ10uqd9peRERLvGTf07EeHVB1G/sngE1tb1E2pfxP+fpuSeuPct0PgC/YPlvSNIpfOOsASHoN8DVgX9v3VVzeiIhxMYO7Vsw1tu8e7QRJqwFr2T4bwPbT5XGAvwdmAnvYfs4QoPK8o4Cjqix0RMRYBnl1xyeani9i6aafaeXXkQboN/ypPG9LRhjbCVAO9J8JIKl7/6Ujoo+4ss7TOlTdxv44sNoy3nsAeIGkNSWtCLwZwPZjwDxJbwGQtKKklctr/gq8Cfh82bQTEdFxja3xWnl0QqWJ3fbDwBVlJ+kJw957BjgOuBr4BXBH09v/CBwt6SbgSuBFTdc9AOwNnCRpuyrLGxExXt08Kqbyphjbbx/lva8CXx3h+O+A1w87fBdwcfn+fcArqytlRMTEDHIbe0REH+rcUMZWJLFHRIxDNrOOiOgjNgwNLe50MZYpiT0iom2d6xhtRRJ7RMQ4JLFHRPSZJPaIiD7TqclHrUhij56y4Oknxj5pAlZcceWxT5qABx4YdemkCVu48Ola4wNsv/f2tcb/5om1hq9GB1dubEUSe0REmwwMpcYeEdFf0hQTEdFXMtwxIqLvJLFHRPSRxp6n3SqJPSKibcZZUiAior9kEbCIiD6TppiIiD6TxB4R0UeKbe8yjj0ioq+kxl4jSUcBR3W6HBExWIaGUmOvje2ZwEwASd37KzQi+ktq7BER/cSY7q2xT+l0AVol6VxJL+l0OSIiGjNPW3l0Qs/U2G2/sdNliIhoSOdpRESfSWKPiOgrZihrxURE9I9uX92xZzpPIyK6SmPf07EeY5C0p6S5ku6U9IkR3l9R0g/L96+WtP5YMZPYIyLa5pb/G42kqcBJwF7AJsAhkjYZdtq7gEdsvxz4CvDFsUqXxB4RMQ72UEuPMWwL3Gn7LtsLgTOBfYedsy/w3fL5T4A3SNJoQdPGHhExDhUtKbAW8Iem1/OA7ZZ1ju1Fkh4F1gTmLytovyX2+cC9bZw/nVH+cSpQd/zJuEdXxV+w8Kla4z/00H21xh+HrvsMvWOXnWuNPw7txl+vgnueX963FdMkzWl6PbNcCgVgpJr38PabVs5ZSl8ldtsz2jlf0hzb29RVnrrjT8Y9Er+/40/GPXo9/khs71lRqHnAOk2v1wbuX8Y58yQtB6wB/GW0oGljj4jonNnAhpI2kLQC8DZg1rBzZgHvKJ8fCPzGY4y17Ksae0RELynbzN9P0bQzFTjV9q2SjgPm2J4FfBv4vqQ7KWrqbxsr7qAn9pljn9LV8SfjHonf3/En4x69Hr9Wts8Fzh127DNNz58G3tpOTHXz7KmIiGhf2tgjIvpMEntERJ9JYo+I6DNJ7BERfSaJPSKizySxR0T0mST2iIg+8/8Bezgc94XgxrEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translation, attention = translate_sentence(candidate)\n",
    "\n",
    "print(translation)\n",
    "\n",
    "display_attention(candidate, translation[:translation.index('<eos>')], attention)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ein mann mit einem orangefarbenen hut , der etwas anstarrt .\n",
      "a man in an orange hat starring at something .\n"
     ]
    }
   ],
   "source": [
    "candidate = ' '.join(vars(test_data.examples[0])['src'])\n",
    "candidate_translation = ' '.join(vars(test_data.examples[0])['trg'])\n",
    "\n",
    "print(candidate)\n",
    "print(candidate_translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'man', 'wearing', 'an', 'orange', 'hat', 'is', 'something', 'something', 'something', 'something', 'something', '.', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>', '<eos>']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVYAAAFFCAYAAAC37cW2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xu8XFV9/vHPQ8It3ESDioAEMYiI3AURrVoQwVrEigqtbRHFl1JrrfVCraJivbe1tl5BERQvIFUaFQGLCIhEknAHAamCRO1PohEDmITkPL8/9j4yOZ6cmcneZ/aeOc87r/06M3v2XrNmcuZ71qy91nfJNhERUZ+Nmq5ARMSoSWCNiKhZAmtERM0SWCMiapbAGhFRswTWiIiaJbBGRNQsgTUiomazm65AzDySngbMo+P3z/bnGqtQRM0SWGOgJH0e2BW4Dlhb7jaQwBojQ5nSGoMk6YfAHs4vXoyw9LHGoN0EPLrpSkRMp3QFxKDNBW6RdDWwanyn7aOaq1JEvRJYY9De2XQFIqZb+lhj4CTtDMy3/T+S5gCzbK9oul4RdUkfawyUpBOB84BPlbt2AM5vrkYR9UtgjUH7G+AQ4LcAtn8EPLLRGkXULIE1Bm2V7dXjdyTNphjHGjEyElhj0C6T9FZgc0nPAb4CfL3hOkXUKhevYqAkbQS8AjgcEHAR8OlMGIhRksAaEVGzjGONgZJ0CMVY1p0pfv8E2PbjmqxXRJ3SYo2BknQr8PfAEh5KwoLtXzVWqYiapcUag3av7W81XYmI6ZQWawyUpPcDs4Cvsm6ugGsaq1REzRJYY6AkXTrJbtv+44FXJmKaJLBGRNQsEwRioCQ9StJnJH2rvL+HpFc0Xa+IOiWwxqCdSTEp4DHl/duB1zdWm4hpkMAagzbX9rnAGIDtNXQMu4oYBQmsMWj3S3oEZeIVSU8F7m22ShH1yjjWGLQ3AAuAXSVdCWwHHNNslSLqlVEBMXBlqsAnUExnvc32gw1XKaJWCawxUJI2A04Cnk7RHXAF8EnbKxutWESNElhjoCSdC6wAzi53HQdsa/vFzdUqol4JrDFQkq63vXe3fRHDLKMCYtCuLUcCACDpIODKBusTUbu0WGMgJN1I0ae6McWFq5+W93cGbrG9Z4PVi6hVAmsMhKSdp3rc9l2DqkvEdEtgjYEp17u6Ia3TGHXpY42BsT0GXC/psU3XJWI6ZeZVDNr2wM2SrgbuH99p+6jmqhRRrwTWGLR3NV2BiOmWPtaIiJqljzUGStJTJS2SdJ+k1ZLWSvpt0/WKqFMCawzaRymmsf4I2Bx4ZbkvYmSkjzUGzvYdkmbZXgt8VtL3m65TRJ0SWGPQHpC0CXCdpA8CvwC2aLhOEbVKV0AM2l9S/N69lmK41U7Aixqt0ZCS9IFe9sXgZVRADISkS2wfKukDtt/SdH1GgaRrbO83Yd8Ntvdqqk5RSFdADMr2kp4JHCXpyxSrB/ye7WuaqdbwkfQaimThu0q6oeOhrUimsFZIizUGQtIxwCsoVg5YPOFh2/7jwddqOEnaBtgWeB9wcsdDK2z/uplaRacE1hgoSW+3/e6m6zHsktCm3XLxKgbtPZJeJukUAEmPlXRg05UaNklo025pscZASfoEMAb8se0nStoWuNj2Uxqu2tCR9B3gKUAS2rRMLl7FoB1kez9J1wLYXl6Oa43+tSahjaSNgUXAibYXNV2fpiWw9kDSHODnwItsX9J0fYbcg5JmUSzLgqTtKFqw0YfyPXy77cOarkvpBcAmwIkUAXZGS2DtzUuAmynmtc/owCppL2AeHb87tr/aRxH/AXwNeKSk9wDHAG+rs44zge21kh6QtI3te5uuD8WIjxOAz0maY/uBpivUpATW3pxAEVS/Kmlb28ubrlATJJ0B7EXxR2a8lWmg58Bq+wuSlgCHUoxlPdr2D+uu6wyxErhR0rdZt4/1dYOshKSdgEfaXijpfOClwGcHWYe2ycWrLiTtDnza9tPLK9nLbf9n0/VqgqRbbO9RsYyHT7J7he0Hq5Q7bMo8Cf8M/A64ENgbeL3ts/so468n22/7rFoq2Xs9TgHutf0RSU8ETrf99EHWoW0SWLuQ9CHgVtufKYe2nD9xGuGA6rEdRf/VPNb9Gn7CAOvwGeBfbd9SoYw7KfIDLKdosT6MIhHLLykufCypoaqtJ+k62/tIeiFwNPD3wKW29264an2RJOA24KnjkxMkXQUcb/u2RivXoHQFTKG80vki4MkAtn8q6VeSDrA9cfbQdPtv4Argf4C1A37ucWcBV0n6P2AVRWB0n3PTLwS+ZvsiAEmHA0cA5wIfBw6qt8qttXH583nAl2z/uohRvZM0n2L21R7AZuP7bT+urkr2YCuKlnbnjK+TmDBleaZJi3UK5WiA3Wxf17FvZ2Ct7aUDrst1tvcZ5HNOUoc7gDcAN9JxJd/2XX2Usdj2AZPta8NrHBRJ76doqf4OOJCi5f4N2z3/YZH0PeAdwIeBPwVeTvGZfkf9Ne65ThsBW9qe0atCZObVFGw/MCGobgtsM+igWvqGpOc18Lydfmp7ge2f2L5rfOuzjF9LeoukncvtzcDycvjQjBl2Zftk4GDggLJ/+X6KIUv92Lwc/qfy/+KdwMBzLkj6oqStJW0B3ALcJulNg65Hm6TF2oWk7wJHUXSbXAfcA1xm+w0DrscKioTQq4AHeehr+NYDrMPHKVpWXy/rAfQ33ErSXIpW1tMpXsP3KAa63ws81vYddda5zSTtyR9+jf9cH+dfCTwDOA/4DvAz4P22n1BzVbvVY7y/+C+A/YG3AEtmcvrC9LF2t43t30p6JfBZ2++YkKptIGxvNejnnMTmFAH18I59/Q63Wgb87XoenklB9R3AsygC6wXAkRR/ZHoOrMDrgTnA64B3A88G/qrWivZm4/J6xNHAR20/KGlGt9gSWLubLWl7ikkC/9RkRSTtAOzMuqMCLh/U89t+edUyJO0GvJE/HN0w09IGHkMxxOpa2y+X9Cjg032WMa+cPnofRf8qkl4M/KDWmnb3KeBO4Hrg8vI6xIzuY01XQBflL+rbgSttv0bS44AP2R7ociLlkhsvpejDGh8V4EEm3CiD4ieAR9nes5yFdZTtf+6jjOuBTwJL6BjdMFOGWY2TdLXtA8vJEs8GVgA32X5SH2VMtoLAH+xrgqTZttc0XY+mJLAOCUm3AXvZXtX14Omrw2XAm4BP2d633HdTPzlBJS2xvf901XFYlP3VbwWOBf6BotV5XS/fCiQdSTFM6yXAOR0PbQ3sYXugaRjLxNvvAP6o3HUZcGpLpto2IqMCupC0o6SvSfqlpP8n6b8k7dhAVX7MQ2MfmzLH9tUT9vXbKvm6pJMkbS/p4eNbXRUcFrZPsv0b258EngP8dR9dLT+nWIVhJUXLf3xbADx3OurbxRkULe6XlNtvmeFTWtPH2t1ngS8CLy7vv6zc95wB1+MBiiWjL2HdK/KDnBe+TNKuPJSZ6hiKWVP9GJ+G2Tkcx8AgB7U3TtLnKCZ8XGH71n7OtX09RZLrL45PBS6HAu7UUB6LXSd0jb1L0nXrPXoGSIu1u+1sf9b2mnI7E9iugXosoLjy+33WbaX0RNIuvezr4m8oLlTsLulnFFelX9NPAbZ3mWTrK6iW/d5d900XSbPKqc5VnAlsD/ynpP8tvwn9XZ9lfLscP/pwigtHn5X0bxXrtSF+J+n3uQEkHUIx8WHGSh9rF5L+h+JD8KVy13HAy20f2lilNsB6LnRsUH9nORB8I9srNrAuVcdvNn7RRkX2/kNd4QNUTop4CsXFq1cDv7O9ex/nX2t733Io4E7jQwEHPX5U0j4U0523KXctp8gVcP0g69Em6Qro7gTgoxTTBqFYXrivxCeSNqXIOTCPdYcYndpHGRs0L1xFdq4nAdtI+rOOh7buLKfHOqzzOsbntvf5OjZ4/GbHRZsdJP1Hx0Nb039fb1XXAv8t6Susm7KvpzG9ZZfOFsBVFF0CT7H9yz7r0IqhgOXsxL0lbV3en9FDrSCBtSvbP6WYeVXFf1PMLFpCR/9onz7LQ/PCn005L7yH854APJ9ixtSfduxfQZEtqx91vI4q4zfHL9ocxbrdICsoskMN0sOBX7HuFNJ+JkvcQDFLaU+K9/Q3kq6y3c9X6FOBi4Dv2V5UDgX8UR/n16L8P3wv8BjbR0raAzjY9mcGXZfWsJ1tig34IEWLaGOK1QOWAS/rs4ybaqjHkvLnjR37rujj/INrqEMdr+Pq8ddTvq8Cbu6zjNlN/17UtQFbUsxEuwtY1XR9NvA1fIui1Xz9+P9P5+/pTNzSYu3ucNtvVpE3cynF6IBLgZ4TEgPfl/Rk2zdWqMfKMnPQjyS9lmJe+CO7nSTpzbY/CPy5pOMmPu7+RhXU8ToWS3oYcDpFcL2PYpXRfvxosimTHmC6vKqTJST9LUW+hP0pguoZFF0C/dSh8Ry9pbm2z5X0j+Xzr5HUVGrLVkhg7a5y3kyKD9Dxkn7Chucx3dB54ePLniymHCZVQaXXoeKNe5/t3wCflHQhsLXtfnMvdKYd3Izij92gx8KeTjlZAsD2DZK+SLEqQC82A/6N4pvIhvYPtyFHL8D9kh7BQ8PwnkrRvTFjJbB293VJt1IMHzmpbCWs7LOMI2uoh4HPU+QKGA/2p1OsQbX+k+yvlzdvoZjpM4+H/t9Nf0k/Kr0O21axJtL+5f07N7CcX03Y9e8qcpOeUqV+fZpj++oJf2T7CZB72V5nyJakz9v+yz7r8JY+jp8ub6AYDririoxb21H0pc9YCaxd2D65nKf/W5crY9Jj3kxJW7u4QrpBw5Im+AJFC2mdJNN9OHtDz6/5dSyU9BRXWHteUuewqo0oWrCDzv5VdbLEOjkBJM2m/IPTh29Iep7tC/o8r1a2r5H0TIoLpQJu8wxbw2yijGOdgooVBOa7YzyeinWv1tr+WQ/nf8P288uvzuNv9HgTx/30CUr6niss0Fbl/Jpfxy0UH8A7KYYp9d0tIunSjnqsKcv6F9u391pGVeUV+NOAp1GM2/wJ8Bfukvi77Id8K0UKxvElogWsBk6z/Y991KENOXorfUZGVQLrFFTkmLyV4mvb/eW+i4G3uo81r8qLTn8B7GL71PIXb3vbPad3k3QoxeSEiVNaex03Wen8sow6XsfOwLYUCZoBLgd+0y0gTShjM/5wXLDdx3jaDSVpYoLzzSlazfeXlehp5pOKVVpvBB5n+13le/lo/2Euhm7lPByYz7pjmy/rp4wq6vqMjJpMaZ1C+XXmaxTp+sb/Em+3Ab8wHwOeShHYoPhK/dE+y3g5sA/Fwnt/Wm7PH+D5UM/rOJqir3guRV/c5+l/nPD5FPV/kGJUwX10DNKfZluV2wEU03m3pRgj/GqKSQ+92privTy2vL+C4v3tWTnj6jKKBRrfWf4cZD9znZ+R0dL0eK+2b8DulONFgbcBr9uAMq4pf17bse/6PsuoNC6w6vk1vo4bgC067m8B3NBnGZXH09bwXlwMbNVxfyvgwkH/TlC0VK8r7+8OnNPAe1H5MzJqW1qsXbjMPFSOWzyOooXVrwfLeeHjFzq2o/8LUAvLGS0bqur5UM/rEOsODVpLbzPIOn1f0pP7PGf9FSpSGG7a52mPpegXHbeaomuiV3W8lyttryzP37T8XR3oeldQ22dkpGRUQG8+QzHt8gZvWFq2/6D4uvRISe+hGIrytj7LeDrw1xXGwlY9H+p5HZ8FfiDpa+X9oyne364k3UgRiGYDL5f0Yzb8tXT6PMVQof+y/cY+zrm6fB0GXkiRiKRXdbyXS8vJFudTZLpaTjHttzJJj7b9f32cUvUzMlJy8aoH5ZXPXwAvsv0/G1jG7sChFEHgEts/7HLKxPN3nmy/e7zoU/X8jnIqvY6yjP14aJXWy21f2+N5k76Gcf2+lglliyL7/s19nLMfHRfhen0dHedXfi87ynomRXapC22v7nZ8D+V90/af9HF85c/IKElgjYioWfpYIyJqlsAaEVGzBNY+SHpVymhPHdpSRhvqUEcZbajDqEhg7U8dvzSjUkYb6tCWMtpQhzrKaEMdRkICa0REzTIqoMNkyZMHbf/9uyc4uueee9huu/UvFLtkSc+Lt0Y0YZntDV7p+IgjjvCyZct6OnbJkiUX2T5iQ59rQ2WCQMv84Op+k+n/oY1nb9z9oC7sDclMGNGTDR5vDLBs2TIWL+4tFYGkuVWea0MlsEbE0Gn7N+0E1ogYKgbWjrX7G1UCa0QMGePKy7dNrwTWiBguhrF2x9UE1ogYPm3vYx35caySzpe0RNLNmRUSMfwMjNk9bU2ZCS3WE2z/WtLmwKIy5+bE5ZMjYoi0vcU6EwLr6yS9sLy9E8XCa78PrGUrNi3ZiCFhO6MCmiTpWcBhwMG2H5D0XTpWswSwfRrFMsatmHkVEd2lxdqsbYDlZVDdnWJVzIgYchlu1awLgVdLugG4DVjYcH0ioqLi4lXTtZjaSAdW26uAI5uuR0TUK10BERF1ysWriIh6mbRYIyJq1+Tg/14ksLbMipUrK5ex6aZzKpexatUDlctITteYLmmxRkTUKtmtIiJq5WS3ioio31hGBURE1Gc8u1WbJbBGxNDJxauIiDo1nGu1F61OdC1pnqRbJX1a0k2SviDpMElXSvqRpAPL7fuSri1/PqE893hJX5V0YXnsB5t+PRFRD9s9bU0Zhhbr44EXU+RMXQT8OfB04CjgrcBfAX9ke42kw4D3Ai8qz90H2BdYBdwm6T9t391ZePKxRgwXA2tb3mIdhsD6E9s3Aki6GbjEtiXdCMyjSA14lqT5FO/5xh3nXmL73vLcW4CdgXUCa/KxRgyftvextroroLSq4/ZYx/0xij8M7wYutb0n8Kesm8i689y1DMcfkojoIl0B028b4Gfl7eMbrEdEDIBz8WogPgi8T9KVwKymKxMR0y8t1gps3wns2XH/+PU8tlvHaW8vHz8TOLPj+OdPVz0jYrDa3sfa6sAaETFRMSogU1ojImqVJCwREXVquP+0FwmsLbPtFltULqOOXzpJlcuImA5ZmiUiYhq0fbhVAmtEDJ20WCMiauQsf10/Sa8GHrD9uabrEhHNaPuaV0M180rSbNufTFCNmNnG3NvWC0lHSLpN0h2STp7k8cdKurRMTXqDpOd1K7P2wCrpzZJeV97+sKTvlLcPlXS2pMMlXSXpGklfkbRl+fgpkhaVeVdPU3lZWtJ3Jb1X0mXA30l6p6Q3djz2AUlXS7pd0jPK/XMknVu+CedI+oGkA+p+rRExeOOjAuqY0ippFvAx4EhgD+A4SXtMOOxtwLm29wWOBT7erdzpaLFeDjyjvH0AsKWkjSlyqN5YVvIw2/sBi4E3lMd+1PZTyixVmwOdU1AfZvuZtv91kuebbftA4PXAO8p9JwHLbe9Fkf1q//peXkQ0rcZcAQcCd9j+se3VwJeBF0x8OmDr8vY2wM+7FTodfaxLgP0lbUWRtu8aigD7DGABxV+FK8sG6SbAVeV5z5b0ZmAO8HDgZuDr5WPnTPF8X+143nnl7acDHwGwfZOkG9Z3chJdRwyZei9e7cC6OZqXAgdNOOadwMWS/hbYAjisW6G1B1bbD0q6E3g58H3gBuDZwK7AT4Bv2z6u8xxJm1E0rw+wfbekd7JuXtX7p3jK8ZyrnflWex7dnkTXEcOlzwkCcyUt7rh/WvmZHzdZrJhY+HHAmbb/VdLBwOcl7WmvP2HBdF28uhx4Y/nzCuDVwHXAQuAQSY+H3/eF7sZDQXRZ2ed6TMXn/x7wkvI59gCeXLG8iGiRsTIna7cNWGb7gI7ttAlFLQV26ri/I3/4Vf8VwLkAtq+iiFdzp6rfdAXWK4Dtgats/z9gJXCF7XsoklF/qfx6vhDY3fZvgNMp+mDPp1jbqoqPA9uVz/EWilbzvRXLjIiWcI//erAImC9pF0mbUFycWjDhmJ8ChwJIeiJFYL1nqkLV9hkMG6K80rex7ZWSdgUuAXYrO6enOm8k3ozkCoiWW2J7g0fp7PakJ/k/vvSlno49cu+9uz5XOXzq3ykS5Z9h+z2STgUW215Qfus9HdiSopvgzbYvnqrMoZsg0KM5wKXlaAQBr+kWVCNiOJh6cwXYvgC4YMK+Uzpu3wIc0k+ZIxlYba+gGIkQEaMmU1ojIuqVtIHRt9mzN6lcxkEHVV/e61/P/q/KZbz/7/++chn33HN394Om1O4PYGyYBNaIiJolH2tERK16HkrVmATWiBgqdrG1WQJrRAydjAqIiKhR3eNYp0MCa0QMnbaPChiqFQTWR9L5kpZIurlMA4ik+yS9R9L1khZKelTT9YyIGvSYi7XJ4DsSgRU4wfb+FLOtXifpERR5Exfa3psiy9aJTVYwImo0fgWr29aQUekKeJ2kF5a3dwLmA6uBb5T7lgDPmezEJLqOGD5ja9vdFTD0gVXSsygyeh9s+wFJ36VI6/WgH/ou0JkEex1JdB0xXIrGaLs/qkMfWCnWoFleBtXdgac2XaGImF4JrNPvQuDVZVLr2yiSZ0fEyGr2wlQvhj6w2l5FsXTtRFt2HHMecN7AKhUR08pjCawREbVJH2tExDRwprRGRNSr5Q3WBNa2WbOm+tJcV1/9zcpl3PDKyyqX8cvlUy5k2ZOtN9+8chkxYuz0sUZE1C19rBERNcqaVxER0yCBNSKiTjZem1EBERG1Sou1B5IEyHa7/wxFRCu0PK4OLh+rpDdIuqncXi9pnqQfSvo4cA2wk6RPSFpcJqx+V8e5d0p6l6RrJN1YJltB0naSvl3u/5SkuyTNLR97maSrJV1XPjZrUK81IqbP+MWrGZ/oWtL+wMuBgyiyT50IbAs8Afic7X1t3wX8k+0DgL2AZ0raq6OYZbb3Az4BvLHc9w7gO+X+rwGPLZ/vicBLgUNs70ORNvAvpvllRsQguP2BdVBdAU8Hvmb7fgBJXwWeAdxluzMb1UvKxNOzge2BPYAbyse+Wv5cAvxZR7kvBLB9oaTl5f5Dgf2BRUUvA5sDv5ysYkl0HTFszFguXgGg9ey///cHSLtQtESfYnu5pDMpElaPW1X+7Exavb5yBZxl+x+7VSyJriOGT9svXg2qj/Vy4GhJcyRtQdHKvGLCMVtTBNp7y4X/JksFONH3gJcASDqconsB4BLgGEmPLB97uKSdq7+MiGia0xVQsH1N2QK9utz1aWD5hGOul3QtcDPwY+DKHop+F/AlSS8FLgN+AaywvUzS24CLJW0EPAj8DXBXHa8nIhrW8hbrwIZb2f434N8m7N5zwjHHr+fceR23FwPPKu/eCzzX9hpJBwPPLhNfY/sc4Jw66h4R7dL2gZmtGMdawWOBc8tW6WqyxHXEjND2PtahDqy2fwTs23Q9ImKAbMZanuh6YBMEIiLqUPcEAUlHSLpN0h2STl7PMS+RdEs5eemL3coc6hZrTJ+VK++rXMbDt9qmchm3/+IXlc4/eM/9K9fhV7/6eeUyokaubzHBckbmx4DnAEspxr4vsH1LxzHzgX+kmHC0fHy00VTSYo2I4VOMueq+dXcgcIftH9teDXwZeMGEY04EPmZ7efHUnnSyUacE1ogYMr11A/TYFbADcHfH/aXlvk67AbtJulLSQklHdCs0XQERMXTGeu8KmCtpccf908rZluMmm705sfDZwHyKYZ47AldI2tP2b9b3pEMZWCXNA75he88uh44ffzRwe2e/SUQMJ/fXx7qsTOy0PkuBnTru7whM7FRfCiy0/SDwE0m3UQTaResrdKZ0BRxNkdAlIkZAjV0Bi4D5knaRtAlwLLBgwjHnA88GKNOS7kYxO3S9hjmwzpJ0ejn84WJJm0s6UdIiSddL+q8yN8HTgKOAD5W5WXdtuuIRUU1dgdX2GuC1wEXAD4Fzbd8s6VRJR5WHXQT8StItwKXAm2z/aqpyh7IroDQfOM72iZLOBV4EfNX26QCS/hl4he3/lLSAouvgvAbrGxG1qDfBiu0LgAsm7Dul47aBN5RbT4Y5sP7E9nXl7SXAPGDPMqA+DNiS4i/NlJKPNWLIOFNap9OqjttrKZJZnwkcXWbKOp6HkrWsV/KxRgwXA17b7o/qMPexTmYr4BeSNmbdpVhWlI9FxAhoez7WUQusbwd+AHwbuLVj/5eBN0m6NhevIoZcj0F15BNd1832nXTkcrX9Lx0Pf2KS468kw60iRkZduQKmy1AG1oiY2XLxKiKiRuNpA9ssgTUihouNW57oOoE1IoZO1ryKGWvNmtWVy9hvfrVrjsef9E+V63D7ddVz99x++3rzdfTszjtvrFzGqEhXQEREnTLzKiKiXrl4FRFROzO2tt2drKM28woASd9vug4RMU3c/imtI9litf20pusQEdOo5V0Bo9piva/8ub2ky8sE1zdJekbTdYuI6upbpHV6jGSLtcOfAxfZfk+5fvicpisUEdXk4lXzFgFnlGkEz+9IjP17SXQdMWT6W0ywESPZFTDO9uXAHwE/Az4v6a8mOeY02wd0WckxIlrDjI2N9bQ1ZaRbrJJ2Bn5m+3RJWwD7AZ9ruFoRUVG6Apr1LIoE1w8C9wF/0GKNiCGUwDp4trcsf54FnNVwdSKiRh6CPtaRDKwRMdpa3mBNYI2IYdPsrKpeJLBGxHAxjV7x70UCa7Tafff9ptL5Z33ivZXrcOCBf1K5jL/953dWLuMfXvaiymWMApM+1oiI2qUrICKiVg0nAuhBAmtEDJesIBARUb+xte0OrK3JFSDpYZJO6rj/LEnfWM+xn5ZUbZW5iBhK49mt2pzoujWBFXgYcFLXowDbr7RdfenMiBg+Q7CCQNfAKmkLSd+UdH2ZLPqlkg6VdK2kGyWdIWnT8tg7Jb1X0lWSFkvaT9JFkv5X0qs7ynyTpEWSbpD0rnL3+4Fdy6TUHyr3bSnpPEm3SvqCJJXnf1fSAeXt+yS9p6zfQkmPKvfvWt5fJOnU8eTXETHseguqrQ6swBHAz23vbXtP4ELgTOCltp9M0U/7mo7j77Z9MHBFedwxwFOBUwEkHQ7MBw4E9gH2l/RHwMnA/9rex/abyrL2BV4P7AE8DjhkkvptASy0vTdwOXBiuf8jwEdsPwX4eQ+vMyKGxCgE1huBwyR9oFzaZB7wE9u3l4+fRZE8+zXAAAAJ90lEQVTzdNyCjvN+YHuF7XuAlZIeBhxebtcC1wC7UwTayVxte6ntMeC68rknWg2M98Uu6TjmYOAr5e0vru/FSXpV2bpevL5jIqJdPOaetl5IOkLSbZLukHTyFMcdI8nj35an0nVUgO3bJe0PPA94H3Bxl1NWlT/HOm6P358NCHif7U9NqPS8KcoCWLue+j7oh/40re+Y9bJ9GnBaWYd2X2qMiFqzW5VLNn0MeA6wFFgkacHEaziStgJeB/ygl3J76WN9DPCA7bOBfwGeBsyT9PjykL8ELuv1hQAXASdI2rIsfwdJjwRWAFv1UU43C4HxOYDH1lhuRDSsxq6AA4E7bP/Y9mrgy8ALJjnu3cAHgZW9FNpL6+7JwIckjQEPUvSnbgN8RdJsinWlPtnLkwHYvljSE4GrymtR9wEvs/2/kq6UdBPwLeCbvZa5Hq8Hzpb0D2VZ91YsLyJaodb+0x2AuzvuLwUO6jxA0r7ATra/IemNvRTaS1fARRStzIn2neTYeR23z6S4eDXZYx+huLg08fw/n7Drux2Pvbbj9rM6bm/Zcfs84Lzy7s+Ap9q2pGOB9KFGjIL+ugLmTrh+clrZ/TdOkz9D+aC0EfBh4Ph+qjjKM6/2Bz5aDtH6DXBCw/WJiJr00WJd1mWh0KXATh33d2TdUURbAXsC3y2/YT8aWCDpKNvrbayNbGC1fQWwd9P1iIh6jc+8qskiYL6kXSi+5R4L/P6bs+17gbnj9yV9F3jjVEEVRjiwRsSoMq4p0bXtNZJeS9HdOQs4w/bNkk4FFtteMHUJk0tgjZFWNVE2wPXXX1q5jE8e9qHuB3XxD5VLGBEG17iAgO0LgAsm7DtlPcc+q5cyE1gjYugkbWBERM0SWCMialTzxatpkcAaEcPFZmxtu1dpbU0+1iS6joie2b1tDWlNYCWJriOiR+7xX1OS6DoihopHYQUBkug6IlrF2GM9bU1Jouskuo4YOm1vsSbRdRJdRwydsZqmtE6XJLqOiKFStEbb3RWQRNcRMXxaPkFAbZ/BsKEkzQF+15Ho+jjbky250HnOaL4ZQ22yPMR9nK1q5wM84hGPqVzGVTdV78Kf/+hHVy6jJZZ0yZE6pW222c6HHPLCno791rdOr/RcG2qUZ14l0XXEiGp7g3BkA2sSXUeMKjM2trbpSkxpZANrRIym8QkCbZbAGi1X9QNUvY/1d79bUbmMLTbdtHIZG200q9L5bW/l9SOBNSKiZgmsERG1ajZzVS8SWCNi6Jghn3k1KMnHGhG9sIsprb1sTWlNYCX5WCOiJ70lYGl12sDkY42Itml7roDkY42IoTP0LVaSjzUiWqbtgTX5WJOPNWK4NLxQYC+SjzUihoqBMa/taWtK8rFGxJBp9mt+L5KPdd1zRvPNmMGk6iMK58yp/kXqR0vvrFzGjo+YW+n8FuUKqJQjdcstH+YnP/mZPR27cOGC5GOtWfKxRoyotjcIRzawJh9rxGgqrl21e0rryAbWiBhVxi1fpTWBNSKGjivn6Z1eCazRalUvPm20UfWLV3Pn7li5jB/+/GeVy6hjYcRRkT7WiIhaufV9rG3KbhUR0dX4mld1TWmVdISk2yTdIenkSR5/g6RbyqRRl0jauVuZCawRMXTqCqySZgEfA46kSPZ03CS5nq8FDrC9F3Ae8MFu5bYmsCbRdUT0qsZE1wcCd9j+se3VwJeBdSYS2b7U9gPl3YVA10731gRWkug6Inpi8FhvW3c7AHd33F9a7lufV1BMuZ9SEl1HxNBxj/+AueNpQcvtVROKmmyoxaR9CJJeBhwAfGiyxzsl0XVEDJU+L14ts31Ax3bahOKWAjt13N+RSeKFpMOAfwKOsr1q4uMTJdF1El1HDJ0aRwUsAuZL2kXSJhQpRhd0HiBpX+BTFEH1l70UmkTXSXQdMWTqG8dqe42k11LkiZ4FnGH7ZkmnAottL6D46r8lRapUgJ/aPmqqcrsGoTLR9a9tn132U76aMtG17TvYsETX75b0Bdv3SdqBIs/rdCW6Pockuo4YKXUubW37AuCCCftO6bh9WL9lJtF1RAyV8T7WNkui63XPGc03Y4i1IVfAjjs+oXIZZ1xwTuUyDt9rn0rnr127pnIdalIp+fRmm23heTvv2dOxt91+dRJd1yyJriNGlGl3roCRDaxJdB0xutr+TXtkA2tEjCrXevFqOiSwxjSqnj901qxqv6J19LHOmbN15TLu/r97KpdRtb959uxNKtdhzZrVlcuoKkuzRERMg3QFRETULIE1IqJWLvoDWiyBNSKGTtsXE2xNPtYkuo6IXtgwNra2p60prQmsJNF1RPSkt8xWTfbDJtF1RAydoQ+sJNF1RLTMKATWJLqOiFaxx3rampJE10l0HTFc3P7hVr30sT4GeMD22cC/AE+jTHRdHrIhia5PkLRlWf4Okh7J9CW6hiS6jhgZBsY81tPWlCS6joih0/ZcAUl0ve45o/lmNKZ6EpbZszeudH4dSVh23XXfymW85aPvrVzGK5/73MplVFVTEpZKyac33ngTb7vto3s69p577k6i65ol0XXEiGp7g3BkA2sSXUeMpmFY82pkA2tEjCrjBqer9iKBNaZNeXGykqp9pBttVP1XvI7W0b3Lflu5jFkbzap0/sabbFa5Dvfd13yia2h/EpYE1ogYOukKiIioWQJrRESNijwA7R7HmsAaEUMnLdaIiJpl+euIiLqlxRoRUSdj0mJtNUmvAl7VdD0iojeZeTUEko81YvgksEZE1CyBNSKiVm50aetetGn562kl6YJyNYSIGGLjfazDvpjgSLD9PNtZrTViFIyve9Vt64GkIyTdJukOSSdP8vimks4pH//BetbnW8eMCawRMSrc879uJM0CPgYcCewBHCdpjwmHvQJYbvvxwIeBD3QrN4E1IoZOjctfHwjcYfvHtlcDXwYmLuH0AuCs8vZ5wKHqkhMzF68iYujUOKV1B+DujvtLgYPWd4ztNZLuBR4BLFtfoQms61oG3DXF43OZ4s3s0aiU0fX8HloMXctYvXpl5TJ6MGUZt966sHId/u6lU65j2VMZPZiyjFWrf9d4HUo7V3yOi8rn6cVmkhZ33D+tHLs+brKW58Q+hF6OWUcCawfb2031uKTFVVd8HJUy2lCHtpTRhjrUUUYb6tAL20fUWNxSYKeO+zsCEy9yjx+zVNJsYBvg11MVmj7WiJjJFgHzJe0iaRPgWGDBhGMWAH9d3j4G+I67jOVKizUiZqyyz/S1FN0Ls4AzbN8s6VRgse0FwGeAz0u6g6Klemy3chNY+3Na90NmTBltqENbymhDHeooow11GDjbFwAXTNh3SsftlcCL+ylTbZ9zGxExbNLHGhFRswTWiIiaJbBGRNQsgTUiomYJrBERNUtgjYioWQJrRETN/j8LvSqnI5pSDwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translation, attention = translate_sentence(candidate)\n",
    "\n",
    "print(translation)\n",
    "\n",
    "display_attention(candidate, translation[:translation.index('<eos>')], attention)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}