published/AutomaticTextGenerationCharbasedTPU.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"AutomaticTextGenerationCharbasedTPU.ipynb","version":"0.3.2","provenance":[{"file_id":"1FoT1yOyqU2NzI3NOwxLo9TlM62dXSltD","timestamp":1557927801067},{"file_id":"1eUk3XVeO8R9tBRlFrpR4rDNbOzRw6eHg","timestamp":1557927709519}],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"TPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"-E5bxJVBexLR","colab_type":"text"},"source":["# Automatic text generation with TPU\n","\n","This learns from a text the probabilities of the next characters according to a number of previous characters via a simple backprop neural net, and generates a sample text from that.\n","\n","This is basically my fumbling first attempt at both Python and Tensorflow - please forgive me. :-)"]},{"cell_type":"code","metadata":{"id":"NDDRJuM_TpiN","colab_type":"code","outputId":"871c0c65-f28c-4d3a-d092-3a794003b332","executionInfo":{"status":"ok","timestamp":1558080635242,"user_tz":-120,"elapsed":2216,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":67}},"source":["import tensorflow as tf\n","from tensorflow import keras\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","import sklearn as sk\n","import numpy as np\n","from sklearn.preprocessing import OneHotEncoder\n","import functools\n","import itertools\n","import time\n","import os\n","\n","print(tf.__version__)\n","print(pd.__version__)\n","print(sns.__version__)\n"],"execution_count":0,"outputs":[{"output_type":"stream","text":["1.13.1\n","0.24.2\n","0.9.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"u439xxFUwTJ5","colab_type":"code","outputId":"9e9a444c-7fb9-401a-aa56-757c1553653b","executionInfo":{"status":"ok","timestamp":1558013038724,"user_tz":-120,"elapsed":219495,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":138}},"source":["if 'COLAB_TPU_ADDR' not in os.environ:\n","  print('ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!')\n","else:\n","  TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']\n","  print ('TPU address is', TPU_WORKER)\n","\n","  with tf.Session(TPU_WORKER) as session:\n","    devices = session.list_devices()\n","    \n","  print('TPU devices:')\n","  print(devices)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["TPU address is grpc://10.39.183.106:8470\n","TPU devices:\n","[_DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:CPU:0, CPU, -1, 2233248729957894691), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 6067613282875460626), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 14167830084806039255), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 1413889682251107914), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 10259692159057102788), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 12855566120423276863), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 1890265412494136419), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 3538168929513831367), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 10967547779382030020), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 18150383357603607760), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, 10132940220367531106)]\n","TPU address is grpc://10.39.183.106:8470\n","TPU devices:\n","[_DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:CPU:0, CPU, -1, 2233248729957894691), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 6067613282875460626), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 14167830084806039255), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 1413889682251107914), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 10259692159057102788), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 12855566120423276863), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 1890265412494136419), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 3538168929513831367), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 10967547779382030020), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 18150383357603607760), _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, 10132940220367531106)]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"VRUA_xmpkdvq","colab_type":"code","colab":{}},"source":["# used number of chars used for prediction of the next char\n","PRECHARS=14\n"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7h6Hx2ZpFVWC","colab_type":"code","outputId":"a60daae5-c62d-48a8-c305-9ef618ba30ca","executionInfo":{"status":"ok","timestamp":1558082134433,"user_tz":-120,"elapsed":1376,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["import urllib\n","import re\n","\n","# link = \"http://www.textfiles.com/etext/MODERN/agrippa.txt\"\n","# link = \"http://www.textfiles.com/etext/MODERN/jargn10.txt\"\n","# link = \"http://www.textfiles.com/etext/MODERN/boh-10f8.txt\"\n","link = \"http://www.gutenberg.org/files/38126/38126-0.txt\"\n","\n","f = urllib.request.urlopen(link)\n","myfile = f.read().decode('utf-8','replace').casefold()\n","myfile = re.sub('[^a-zäöü.]+', ' ', myfile)\n","print(myfile[len(myfile)//2:len(myfile)//2+200])"],"execution_count":0,"outputs":[{"output_type":"stream","text":["ie was erwartet worauf diederich unzufrieden den rücken wandte. er belehrte guste. ich glaubte ich hätte eine erwischt. sie stehlen nämlich lumpen. jawohl. sie machen kinderkleider draus. und da guste\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"8lMQeRgNrfkn","colab_type":"code","colab":{}},"source":["# text = \"abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc\"\n","text = myfile\n","chars = list(text)\n","rolled = []\n","for i in range(1, len(chars)-PRECHARS-1):\n","    rolled.append(chars[i:i+PRECHARS])\n","labels=chars[PRECHARS+1:-1]\n","labels=np.transpose([labels])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"p7XBLu3Vf5LJ","colab_type":"code","colab":{}},"source":["enc = OneHotEncoder(sparse=False)\n","sdata = enc.fit_transform(rolled)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"caHjdsW1qym_","colab_type":"code","colab":{}},"source":["lenc = OneHotEncoder(sparse=False)\n","slabels = lenc.fit_transform(labels)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"mBWoldiScEPc","colab_type":"code","outputId":"ae1ef419-42ec-4a19-dca2-0a854b8a52d9","executionInfo":{"status":"ok","timestamp":1558013043154,"user_tz":-120,"elapsed":223896,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":353}},"source":["print(enc.inverse_transform(sdata)[:5])\n","print(lenc.inverse_transform(slabels)[:5])"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[['i' 'e' ' ' 'b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e']\n"," ['e' ' ' 'b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r']\n"," [' ' 'b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r' ' ']\n"," ['b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r' ' ' 'b']\n"," ['i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r' ' ' 'b' 'e']]\n","[['r']\n"," [' ']\n"," ['b']\n"," ['e']\n"," ['r']]\n","[['i' 'e' ' ' 'b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e']\n"," ['e' ' ' 'b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r']\n"," [' ' 'b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r' ' ']\n"," ['b' 'i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r' ' ' 'b']\n"," ['i' 'b' 'e' 'l' ' ' 'l' 'u' 't' 'h' 'e' 'r' ' ' 'b' 'e']]\n","[['r']\n"," [' ']\n"," ['b']\n"," ['e']\n"," ['r']]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"NFvdCvgmrws6","colab_type":"code","colab":{}},"source":["def createModel():\n","  model = tf.keras.Sequential()\n","  model.add(tf.keras.layers.Dense(2048, activation=tf.nn.relu, input_shape=[sdata.shape[1]]))\n","  model.add(keras.layers.Dropout(0.1))\n","  model.add(tf.keras.layers.Dense(512, activation=tf.nn.relu))\n","  model.add(keras.layers.Dropout(0.1))\n","  model.add(tf.keras.layers.Dense(256, activation=tf.nn.relu))\n","  model.add(keras.layers.Dropout(0.1))\n","  model.add(tf.keras.layers.Dense(slabels.shape[1], activation=tf.nn.softmax))\n","  return model"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"gP5qRNhnxoha","colab_type":"code","outputId":"e77a3cdc-445e-4b51-caa7-34efa38b8b43","executionInfo":{"status":"ok","timestamp":1558013051419,"user_tz":-120,"elapsed":232149,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":1599}},"source":["tf.keras.backend.clear_session()\n","\n","tpu_model = tf.contrib.tpu.keras_to_tpu_model(\n","    createModel(),\n","    strategy=tf.contrib.tpu.TPUDistributionStrategy(\n","        tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))\n","\n","\n","tpu_model.compile(\n","              optimizer=\"adam\",\n","              loss=tf.keras.losses.categorical_crossentropy,\n","              metrics=[tf.keras.metrics.categorical_crossentropy])\n","\n","tpu_model.summary()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","INFO:tensorflow:Querying Tensorflow master (grpc://10.39.183.106:8470) for TPU system metadata.\n","INFO:tensorflow:Found TPU system:\n","INFO:tensorflow:*** Num TPU Cores: 8\n","INFO:tensorflow:*** Num TPU Workers: 1\n","INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 2233248729957894691)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 6067613282875460626)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 14167830084806039255)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 1413889682251107914)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 10259692159057102788)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 12855566120423276863)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 1890265412494136419)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 3538168929513831367)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 10967547779382030020)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 18150383357603607760)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, 10132940220367531106)\n","WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","dense_input (InputLayer)     (None, 378)               0         \n","_________________________________________________________________\n","dense (Dense)                (None, 2048)              776192    \n","_________________________________________________________________\n","dropout (Dropout)            (None, 2048)              0         \n","_________________________________________________________________\n","dense_1 (Dense)              (None, 512)               1049088   \n","_________________________________________________________________\n","dropout_1 (Dropout)          (None, 512)               0         \n","_________________________________________________________________\n","dense_2 (Dense)              (None, 256)               131328    \n","_________________________________________________________________\n","dropout_2 (Dropout)          (None, 256)               0         \n","_________________________________________________________________\n","dense_3 (Dense)              (None, 27)                6939      \n","=================================================================\n","Total params: 1,963,547\n","Trainable params: 1,963,547\n","Non-trainable params: 0\n","_________________________________________________________________\n","INFO:tensorflow:Querying Tensorflow master (grpc://10.39.183.106:8470) for TPU system metadata.\n","INFO:tensorflow:Found TPU system:\n","INFO:tensorflow:*** Num TPU Cores: 8\n","INFO:tensorflow:*** Num TPU Workers: 1\n","INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 2233248729957894691)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 6067613282875460626)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 14167830084806039255)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 1413889682251107914)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 10259692159057102788)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 12855566120423276863)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 1890265412494136419)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 3538168929513831367)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 10967547779382030020)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 18150383357603607760)\n","INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, 10132940220367531106)\n","WARNING:tensorflow:tpu_model (from tensorflow.contrib.tpu.python.tpu.keras_support) is experimental and may change or be removed at any time, and without warning.\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","dense_input (InputLayer)     (None, 378)               0         \n","_________________________________________________________________\n","dense (Dense)                (None, 2048)              776192    \n","_________________________________________________________________\n","dropout (Dropout)            (None, 2048)              0         \n","_________________________________________________________________\n","dense_1 (Dense)              (None, 512)               1049088   \n","_________________________________________________________________\n","dropout_1 (Dropout)          (None, 512)               0         \n","_________________________________________________________________\n","dense_2 (Dense)              (None, 256)               131328    \n","_________________________________________________________________\n","dropout_2 (Dropout)          (None, 256)               0         \n","_________________________________________________________________\n","dense_3 (Dense)              (None, 27)                6939      \n","=================================================================\n","Total params: 1,963,547\n","Trainable params: 1,963,547\n","Non-trainable params: 0\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"juR6k8ICCkyj","colab_type":"code","outputId":"abb06f61-018f-450f-9e1e-cf03a0280b49","executionInfo":{"status":"ok","timestamp":1558013189851,"user_tz":-120,"elapsed":370572,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":1414}},"source":["# Display training progress by printing a single dot for each completed epoch\n","class PrintDot(tf.keras.callbacks.Callback):\n","  def on_epoch_end(self, epoch, logs):\n","    if epoch % 100 == 0: print('')\n","    print('.', end='')\n","\n","# The patience parameter is the amount of epochs to check for improvement\n","early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, min_delta=0.0001, restore_best_weights=False)\n","terminate_nan = keras.callbacks.TerminateOnNaN()\n","\n","import datetime\n","print(datetime.datetime.now())\n","history = tpu_model.fit(sdata, slabels, epochs=250, batch_size=1024, validation_split=0.1, verbose=0, callbacks=[terminate_nan,early_stop,PrintDot()])\n","print()\n","print(datetime.datetime.now())\n","history.history['loss'][-1]"],"execution_count":0,"outputs":[{"output_type":"stream","text":["2019-05-16 13:20:32.480511\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(128, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Cloning Adam {'lr': 0.0010000000474974513, 'beta_1': 0.8999999761581421, 'beta_2': 0.9990000128746033, 'decay': 0.0, 'epsilon': 1e-07, 'amsgrad': False}\n","INFO:tensorflow:Remapping placeholder for dense_input\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/tpu/python/tpu/keras_support.py:302: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb5277e9a20> []\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 3.0539023876190186 secs\n","INFO:tensorflow:Setting weights on TPU model.\n","INFO:tensorflow:CPU -> TPU lr: 0.0010000000474974513 {0.001}\n","INFO:tensorflow:CPU -> TPU beta_1: 0.8999999761581421 {0.9}\n","INFO:tensorflow:CPU -> TPU beta_2: 0.9990000128746033 {0.999}\n","INFO:tensorflow:CPU -> TPU decay: 0.0 {0.0}\n","WARNING:tensorflow:Cannot update non-variable config: epsilon\n","WARNING:tensorflow:Cannot update non-variable config: amsgrad\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(106,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(106, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(106, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb5277e9a20> [<tf.Variable 'tpu_140416142996088/Adam/iterations:0' shape=() dtype=int64>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526e378d0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526df1e80>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526e5f710>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526dc3d68>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526d66e10>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526d56eb8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526d22a58>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526c6e6d8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526c3d1d0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526c07d68>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526b9d2e8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526ae9a58>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526ab5e80>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526adda90>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526a2ac50>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5269f6f98>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526a1ba90>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb526930588>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5268fe898>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5268caf60>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb52683af98>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5268055c0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5267d0860>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb52679bf60>]\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 2.5342519283294678 secs\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(128, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Cloning Adam {'lr': 0.0010000000474974513, 'beta_1': 0.8999999761581421, 'beta_2': 0.9990000128746033, 'decay': 0.0, 'epsilon': 1e-07, 'amsgrad': False}\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb524f1d940> []\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 1.5077877044677734 secs\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(49,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(49, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(49, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb524f1d940> []\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 0.9590680599212646 secs\n","\n","....................................................................................................\n","...........\n","2019-05-16 13:23:45.174104\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["0.0718681970057539"]},"metadata":{"tags":[]},"execution_count":11},{"output_type":"stream","text":["2019-05-16 13:24:10.873164\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(128, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(128, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Cloning Adam {'lr': 0.0010000000474974513, 'beta_1': 0.8999999761581421, 'beta_2': 0.9990000128746033, 'decay': 0.0, 'epsilon': 1e-07, 'amsgrad': False}\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb520a1b4a8> []\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 3.0823709964752197 secs\n","INFO:tensorflow:Setting weights on TPU model.\n","INFO:tensorflow:CPU -> TPU lr: 0.0010000000474974513 {0.001}\n","INFO:tensorflow:CPU -> TPU beta_1: 0.8999999761581421 {0.9}\n","INFO:tensorflow:CPU -> TPU beta_2: 0.9990000128746033 {0.999}\n","INFO:tensorflow:CPU -> TPU decay: 0.0 {0.0}\n","WARNING:tensorflow:Cannot update non-variable config: epsilon\n","WARNING:tensorflow:Cannot update non-variable config: amsgrad\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=train (# of cores 8), [TensorSpec(shape=(106,), dtype=tf.int32, name='core_id0'), TensorSpec(shape=(106, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(106, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb520a1b4a8> [<tf.Variable 'tpu_140415935065840/Adam/iterations:0' shape=() dtype=int64>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5200e2240>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5200e2d68>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5200ffeb8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb5200c2358>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb52008c278>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb520058ba8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51ffa7be0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51ffd1898>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51ff9aac8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51feacba8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fe7af98>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fe444a8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fe12f98>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fddaf98>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fcf6d30>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fd1c2b0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fc68ac8>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fc35a90>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fba7be0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fb72748>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fb3dfd0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fb08e10>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fa7dbe0>, <tensorflow.contrib.tpu.python.tpu.keras_tpu_variables.ReplicatedVariable object at 0x7fb51fa48748>]\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 2.8895890712738037 secs\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(128,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(128, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(128, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Cloning Adam {'lr': 0.0010000000474974513, 'beta_1': 0.8999999761581421, 'beta_2': 0.9990000128746033, 'decay': 0.0, 'epsilon': 1e-07, 'amsgrad': False}\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb51e1ba1d0> []\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 1.4753246307373047 secs\n","INFO:tensorflow:New input shapes; (re-)compiling: mode=eval (# of cores 8), [TensorSpec(shape=(49,), dtype=tf.int32, name='core_id_10'), TensorSpec(shape=(49, 378), dtype=tf.float32, name='dense_input_10'), TensorSpec(shape=(49, 27), dtype=tf.float32, name='dense_3_target_10')]\n","INFO:tensorflow:Overriding default placeholder.\n","INFO:tensorflow:Remapping placeholder for dense_input\n","INFO:tensorflow:KerasCrossShard: <tensorflow.python.keras.optimizers.Adam object at 0x7fb51e1ba1d0> []\n","INFO:tensorflow:Started compiling\n","INFO:tensorflow:Finished compiling. Time elapsed: 0.9244117736816406 secs\n","\n",".............................................................................\n","2019-05-16 13:26:29.335419\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["0.08136985062510643"]},"metadata":{"tags":[]},"execution_count":25}]},{"cell_type":"code","metadata":{"id":"CQEcQ3TZxwmy","colab_type":"code","outputId":"50fb2e79-a107-4f51-c55d-f1c0c53f6a59","executionInfo":{"status":"ok","timestamp":1558013196056,"user_tz":-120,"elapsed":376771,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":319}},"source":["tpu_model.save_weights('/tmp/autotextgen.h5', overwrite=True)\n","model = createModel()\n","model.load_weights('/tmp/autotextgen.h5')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:Copying TPU weights to the CPU\n","INFO:tensorflow:TPU -> CPU lr: 0.0010000000474974513\n","INFO:tensorflow:TPU -> CPU beta_1: 0.8999999761581421\n","INFO:tensorflow:TPU -> CPU beta_2: 0.9990000128746033\n","INFO:tensorflow:TPU -> CPU decay: 0.0\n","INFO:tensorflow:TPU -> CPU epsilon: 1e-07\n","WARNING:tensorflow:Cannot update non-variable config: epsilon\n","INFO:tensorflow:TPU -> CPU amsgrad: False\n","WARNING:tensorflow:Cannot update non-variable config: amsgrad\n","INFO:tensorflow:Copying TPU weights to the CPU\n","INFO:tensorflow:TPU -> CPU lr: 0.0010000000474974513\n","INFO:tensorflow:TPU -> CPU beta_1: 0.8999999761581421\n","INFO:tensorflow:TPU -> CPU beta_2: 0.9990000128746033\n","INFO:tensorflow:TPU -> CPU decay: 0.0\n","INFO:tensorflow:TPU -> CPU epsilon: 1e-07\n","WARNING:tensorflow:Cannot update non-variable config: epsilon\n","INFO:tensorflow:TPU -> CPU amsgrad: False\n","WARNING:tensorflow:Cannot update non-variable config: amsgrad\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"U62ovbhWFDPl","colab_type":"code","outputId":"45329829-bb09-4554-f736-d2a9790c6445","executionInfo":{"status":"ok","timestamp":1558013289558,"user_tz":-120,"elapsed":933,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":283}},"source":["def plot_history(history):\n","    hist = pd.DataFrame(history.history)\n","    hist['epoch'] = history.epoch\n","\n","    plt.figure()\n","    plt.xlabel('Epoch')\n","    plt.ylabel('Mean Abs Error [MPG]')\n","    plt.semilogy(hist['epoch'], hist['loss'], label='Train Error')\n","    plt.plot(hist['epoch'], hist['val_loss'], label='Val Error')\n","    # plt.ylim([0,5])\n","    plt.legend()\n","\n","plot_history(history)"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VfX9x/HXJ3snZDADJOw9QgSR\nPbQ4gLoLaHHU0WpdtRa1rbO/2tZa9xZ3pTIc4EBFEFQEwiasMAKEQCAJuSF7fX9/nBsIGMi9yb25\nN8nn+XjcB/eee+697ySaT77nu8QYg1JKKeUoH08HUEop1bRo4VBKKeUULRxKKaWcooVDKaWUU7Rw\nKKWUcooWDqWUUk7RwqGUUsopWjiUUko5RQuHUkopp/h5OoA7xMbGmoSEBE/HUEqpJmPt2rXZxpg4\nR85tloUjISGBlJQUT8dQSqkmQ0T2OXquXqpSSinlFC0cSimlnKKFQymllFOaZR+HUqrpKi8vJyMj\ng5KSEk9HaZaCgoKIj4/H39+/3u+hhUMp5VUyMjIIDw8nISEBEfF0nGbFGENOTg4ZGRkkJibW+330\nUpVSyquUlJQQExOjRcMNRISYmJgGt+a0cCilvI4WDfdxxfdWC4edMYZ3Vqbz2aZDno6ilFJeTQuH\nnYjwYcoB3l6Z7ukoSikPysnJYdCgQQwaNIi2bdvSoUOHE4/Lysoceo/rr7+eHTt2OPyZr7/+OnFx\ncSc+Z9CgQU69vrFp53gNI7vF8fqKPRSUVhAWqN8apVqimJgYNmzYAMDDDz9MWFgY99577ynnGGMw\nxuDjU/vf3m+++abTnztjxgyefvrpMz5fUVGBn9/J30t1ZaipsrISX19fpzOdibY4ahjVPZaKKsOq\nPTmejqKU8jK7du2iT58+zJgxg759+3Lo0CFuvvlmkpOT6du3L48++uiJc0eOHMmGDRuoqKggKiqK\nWbNmMXDgQIYPH86RI0cc/sxvvvmGsWPHcskll9C/f/9aM7z33nv079+ffv368cADDwCc+Ny77rqL\nAQMGsHr1apd+L/TP6hqGdG5FkL8PK9KymdC7jafjKNXiPbIwla2Z+S59zz7tI3hoct96vXb79u28\n8847JCcnA/DEE08QHR1NRUUF48aN44orrqBPnz6nvMZmszFmzBieeOIJ7rnnHmbPns2sWbN+9t7v\nv/8+y5YtO/G4+pd9SkoKW7dupVOnTuzateuUDBkZGfz5z38mJSWFyMhIJk6cyKJFi5g0aRI2m43R\no0eftRVTX82qxSEik0XkVZvNVq/XB/n7MjQxhhVpR12cTCnVHHTt2vVE0QD44IMPSEpKIikpiW3b\ntrF169afvSY4OJgLL7wQgCFDhpCenl7re8+YMYMNGzacuAUEBAAwfPhwOnXqVGuGVatWMX78eGJj\nY/H392f69OksX74cgICAAC699FKXfN2na1YtDmPMQmBhcnLyTfV9j9HdY3n8s20cshXTLjLYhemU\nUs6qb8vAXUJDQ0/cT0tL45lnnmH16tVERUVxzTXX1Do/oroAAPj6+lJRUVHvz6zt8ZkEBwe7bVhz\ns2pxuMLI7rEArEjL9nASpZQ3y8/PJzw8nIiICA4dOsTixYsbPcOwYcNYunQpOTk5VFRUMGfOHMaM\nGeP2z21WLQ5X6NkmnLjwQFakZXNVckdPx1FKeamkpCT69OlDr1696Ny5MyNGjGjQ+53ex/HKK6/U\n+Zr4+Hgee+wxxo4dizGGyZMnc/HFFzvdqnGWGGPc+gGekJycbBqykdM9/9vAsp1HSXlwIj4+OoNV\nqca0bds2evfu7ekYzVpt32MRWWuMST7DS06hl6pqqqqE8hJGdo8lt7CMrYdcO5pDKaWaAy0c1SrL\n4a2L4as/M7Kb9nMopdSZaOGo5usP7QfDmtdonbOGXm3D+X6XDstVSqnTaeGoafxfoFUifHo74xJD\nWbP3GMVllZ5OpZRSXkULR00BITD1BTiWzozCtymrrGJ1eq6nUymllFfRwnG6hBEw9Bbid77NeX47\nWL5TL1cppVRNWjhqM/EhaJXA00GvsTBlF7bick8nUko1knHjxv1sMt/TTz/Nb3/727O+LiwsrNbj\nvr6+pyyX/sQTT7gsq6do4ahNQChMeZ7WFZm8UPUYP81/BoqPeTqVUqoRTJs2jTlz5pxybM6cOUyb\nNq1e7xccHHzKGlS1LXBYWXlqX6qjE/jcPdHvTLRwnEniKLjoSRID8vnFrscw/+oO710BB9d5OplS\nyo2uuOIKPvvssxObNqWnp5OZmcmoUaMoKChgwoQJJCUl0b9/fz755JN6f05CQgJ/+tOfSEpKYu7c\nuYwdO5a77rqL5ORknnnmGdLT0xk/fjwDBgxgwoQJ7N+/H4DrrruOW2+9lWHDhnHfffe55Gt2li45\ncjZDbyKn49Xc+NxbPJiwg6GZX8Cnd8Bvv/d0MqVahi9mweHNrn3Ptv3hwjNfLoqOjmbo0KF88cUX\nTJ06lTlz5nDVVVchIgQFBfHRRx8RERFBdnY25557LlOmTDnrYoLFxcUMGjToxOP777+fq6++GrA2\njVq3zvpj9OWXX6asrIzqVS8mT57MzJkzmTlzJrNnz+aOO+7g448/BiAjI4Mff/zRpZszOUMLRx16\ntougY7+RXL+9B6vP70vokvshKxXaeNeqnUop16m+XFVdON544w3A2nXvgQceYPny5fj4+HDw4EGy\nsrJo27btGd+r+lJVbaoLSG2PV65cyYIFCwC49tprT2ldXHnllR4rGqCFwyF3TezOZ5sP8fqxwdwp\nvrDpf3D+o3W/UCnVMGdpGbjT1KlTufvuu1m3bh1FRUUMGTIEsBYiPHr0KGvXrsXf35+EhIRal1J3\nVH2XTHf0PHfRPg4HdGsdztSB7Xl5jY2yxPGwaS5UVXk6llLKTcLCwhg3bhw33HDDKZ3iNpuN1q1b\n4+/vz9KlS9m3b5/bMpx33nknOunff/99Ro0a5bbPcpYWDgfdMaE7pRWVLDSj4Hgm7NN+DqWas2nT\nprFx48ZTCseMGTNISUmhf//+vPPOO/Tq1avO96nu46i+1TaqqjbPPfccb775JgMGDODdd9/lmWee\nqffX4mq6rLoTZs3fxKK1u9kUehs+/S61ZpkrpVxKl1V3P11WvRHd+4ueiH8IPwSch9n6KZQXezqS\nUko1Oi0cTogNC+TOid15+VgyUpoPO7/0dCSllGp0WjicNPO8BI5En0O2tKJy4/88HUepZqk5XkL3\nFq743mrhcJK/rw9/njKABeXnQdrXUJjj6UhKNStBQUHk5ORo8XADYww5OTkEBQU16H10Hkc9jOkR\nx9JOU/HN/Izj6+YSPupWT0dSqtmIj48nIyODo0d1ZWp3CAoKIj4+vkHvoYWjnq6/7GJ2PNeRkB/e\n1MKhlAv5+/uTmJjo6RjqLPRSVT11jg1jX+cr6ViynfRNOqdDKdVyaOFogHN+eRtFJpADXz3v6ShK\nKdVotHA0QKvoWPa2u5Dk40tYvyPd03GUUqpRaOFooC4X/p5gKWPdopd0FIhSqkXQwtFAwZ2TyY7s\nx2jbQr7dluXpOEop5XZaOFwgavStdPc5yBefzaeySlsdSqnmzesLh4iEisjbIvKaiMzwdJ7a+PW/\nnDL/CEbnL+STDQc9HUcppdzKI4VDRGaLyBER2XLa8UkiskNEdolI9drDlwHzjDE3AVMaPawjAkLw\nT5rOhb5rmL14NaUVlXW/RimlmihPtTjeAibVPCAivsALwIVAH2CaiPQB4oED9tO89jeyJN+IPxWM\nKfiC93/a7+k4SinlNh4pHMaY5UDuaYeHAruMMXuMMWXAHGAqkIFVPMCbL63F9cB0v4A7Aj7hi2+/\npaC0wtOJlFLKLbzpF3EHTrYswCoYHYAFwOUi8hKw8EwvFpGbRSRFRFI8tcaNTHken6AInqj4F299\nu9kjGZRSyt28qXDUyhhTaIy53hjzW2PM+2c571VjTLIxJjkuLq4xI54U3gb/q2aT6JNFl58eIPt4\n/TexV0opb+VNheMg0LHG43j7saYlcTTHht3LRfIjP334L0+nUUopl/OmwrEG6C4iiSISAPwK+NTD\nmeol5hf3syN8GBfsf5rD21d6Oo5SSrmUp4bjfgCsBHqKSIaI3GiMqQBuBxYD24APjTGpnsjXYD4+\nRM14k1wiKPnkHk+nUUoplzrrfhwiEu3Ae1QZY/Kc+VBjzLQzHP8c+NyZ96pJRCYDk7t161bft3CZ\nNm078E3nXzNx/9Ps3fQDiQNGeDqSUkq5hJxtYT4RKQEyATnLe/gaYzq5OlhDJCcnm5SUFE/HwJab\nTcCzfVgTOo7Rf9T9yZVS3ktE1hpjkh05t64dALcZYwbX8WHrHU7WwkRGx7K13UUMzVzEmm27Oad3\nV09HUkqpBqurj2O4A+/hyDktVteL7yZIytmy6EVddl0p1SyctXAYY06ZiCAiHUSkk/3mV9s56lSB\n8QM52iqJcccXsnjLIU/HUUqpBjtr4RCR+0XkrzUOrQQWAV8Bf3RnsOYkeuzvSPDJYtnnH1BRWeXp\nOEop1SB1Xaq6Evh3jcc5xpgBQF/gYrelqicRmSwir9psNk9HOYVv36mUBsZyfsFC5q3N8HQcpZRq\nkDrncRhjCms8fMZ+rBIIdleo+jLGLDTG3BwZGenpKKfyCyBg2PWM893A3G++p6xCWx1KqaarrsIR\nJiL+1Q+MMW8BiEggEOHGXM2OJN+AiA/nF33O3LUH6n6BUkp5qboKxzzgFREJqT4gIqHAy/bnlKMi\n2kPX8UwOXMeLS3drq0Mp1WTVVTj+AhwB9ovIWhFZB6QDWfbnlBOk2wQ6VB5EbPuZv077OpRSTdNZ\nJwDa+zJmicgjQPU6HruMMcVuT9YcdR0PwK9idvPC0k5cnhRPgJ83rTOplFJ1q2s4bncR+QRr5doH\ngFwtGg0Q2wMiOnBVq51kHCtmgbY6lFJNUF1/7s7GmrdxObAOeM7tiRrAW4fjniACXccRd3QlgzuE\n8fzSXZTrvA6lVBNTV+EIN8a8ZozZYYz5F5DQCJnqzWuH49bUdTxSYuPBwSVkHCvmo3VNb68qpVTL\nVlfhCBKRwSKSJCJJQPBpj5WzEscCwpCK9QyIj+TFZbuorNI1rJRSTUddq+MeBp46w2MDjHdHqGYt\nNAbaD0J2L+V3Y2dy63vr+GLLIS4Z0N7TyZRSyiF1jaoa20g5WpauE+D7/3DBtGC6xIXy0rLdXNy/\nHSJn2/ZEKaW8Q107AF52tueNMQtcG6eF6DoeVjyJz77vuXXMQO6bt4nladmM6RHn6WRKKVWnui5V\nzQM22G9w6k6ABtDCUR/x50BAGOz+ll9OuoinvtrJS8t2aeFQSjUJdXWOXwbsBAYAe4G/GWOut99u\ncHu65sovABJGwe5vCfDz4TejEvlpTy7r9h/zdDKllKpTXRs5fWyM+RUwBtgN/FtEvheRMY2Szkle\nP4+jpq7j4dheyN3DtKGdiArx56Vluz2dSiml6uToehclgA3IB8KAILclaoAmMY+jmn35EXYvJTTQ\nj5nDE/h6axZpWcc9m0sppepQ15Ij40XkVWAtMA54xhgzyBizuFHSNWcxXSGqM3z7OHzzCNf38yPY\n35fnvt3l6WRKKXVWdbU4vgGGAt8DgcCvReTZ6pvb0zVnInD1u9BpOPzwNFGvDmFh3Avs37ScH3dn\nezqdUkqdUV2jqm7AGj2l3KHdQJj2X8jbDylv0nXd2ywIXMHcOesou+dFAoJDPZ1QKaV+RoxpfnUh\nOTnZpKSkeDqG80qPc/DDe+mwew65IQlET38D4pM9nUop1QKIyFpjjEO/cOrq43jYgQ+r8xzloMBw\nOlz7Cs+0+welhfmYN86HlNmeTqWUUqeo61LVb0Qk/yzPC/Ar4GGXJVJcefVMpj7VmvcCnqT7d/9C\nkq4DH93wSSnlHer6bfQaEH6WW5j9HOVC7aOCufn8Qbx4fAxyPBMOrvV0JKWUOqGuRQ4faawg6lTX\nnZfA1WtGUZ7/KmbLJwR0PMfTkZRSCnB8AmCT0KRmjtfBz9eHBy4/lx8q+1KwYQE0w0EMSqmmqVkV\njiY1c9wBQzpHk9vpAqJLD7IndbWn4yilFOBA4RARXxG5uzHCqJ+b8MvrqUJY8/lbVOlOgUopL1Bn\n4TDGVALTGiGLqkVkXAdyoocwsGAFH6Yc8HQcpZRy+FLVDyLyvIiMqt5vXPccbzyxQ6+gl88B/vvF\nt+QWlnk6jlKqhXO0cAwC+gKPAv+23550Vyh1Kuk9GYCR5T/xxBfbPJxGKdXS1TUBEABjzDh3B1Fn\nERkP7ZO4xraREWszuPbcBPrHN48BAEqppsehFoeIRIrIUyKSYr/9W0T0N1dj6j2Z9oVb6RNs45GF\nqTTHNcaUUk2Do5eqZgPHgavst3zgTXeFUrXoPQWAR3vtI2XfMRZtOuThQEqplsrRwtHVGPOQMWaP\n/fYI0MWdwdRpYrtB674k2ZbQu10ET3yxnZLySk+nUkq1QI4WjmIRGVn9QERGAMXuiaTOaPA1yME1\nPHGecDCvmFeX7/F0IqVUC+Ro4bgVeEFE0kUkHXgeuMVtqeqpOS05UquBvwK/IAZmzWdS37a8tGw3\nh20lnk6llGphHJk57gP0NMYMBAYAA4wxg40xm9yezknNbcmRnwmJhr6XwaYPeXBiRyqrDH/X4blK\nqUbmyMzxKuA++/18Y8zZ9udQ7pZ8A5QV0PHgIm4d04VPNmSybMcRT6dSSrUgjl6q+kZE7hWRjiIS\nXX1zazJVu/hkaNsf1szmtnFd6RoXyoMfbaGwtMLTyZRSLYSjheNq4DZgObDWfmuCm3o3AyJWqyNr\nM4FZG/jH5QPItBXzr8U7PJ1MKdVCONrHcY0xJvG0mw7H9ZT+V0JAGKx5g+SEaK49tzNvr0xn3f5j\nnk6mlGoBHO3jeL4RsihHBYbDgKsgdQEU5fLHX/SkbUQQs+ZvoqyiytPplFLNnKOXqpaIyOUiIm5N\noxyXfANUlMDGDwgP8udvl/ZjZ1YBLy7b5elkSqlmztHCcQswFygVkXwROS4iOrrKk9r2h47nwsoX\noKyI8b3aMGVge15YuoudWcc9nU4p1Yw5VDiMMeHGGB9jTIAxJsL+OMLd4VQdJvwV8g/CTy8A8NDk\nPoQF+vGn+Zuo1N0ClVJuctbCISLX1Lg/4rTnbndXKOWghBHQ6xJY8R84nkVMWCAPTe7L+v15vP1j\nuqfTKaWaqbpaHPfUuP/cac/d4OIsqj7OfxQqS2Hp3wCYOqg943rG8a/FOziQW+ThcEqp5qiuwiFn\nuF/bY+UJMV1h6M2w/l3ISkVEePzS/vgIPPDRZt23QynlcnUVDnOG+7U9Vp4y+o8QGAFf/RmADlHB\nzLqwFyvSspm3NsPD4ZRSzU1dhaOXiGwSkc017lc/7tkI+ZQjQqJhzH2w+1tI+waAGcM6MzQhmoc/\nTWXP0QIPB1RKNSdytksZItL5bC82xuxzeaIGEJHJwORu3brdlJaW5uk4jauiDF4cBqYKblkBQRFk\n5hVz8bMraBMRxMe3jSDI39fTKZVSXkpE1hpjkh0596wtDmPMvrPdXBPXdZr9supn4xcAU1+EvP3w\n+b0AtI8K5qmrB7H98HEe/jTVwwGVUs2FoxMAVVPQeTiMmQWb/gcb5wAwrmdrbhvXlTlrDrBgnfZ3\nKKUaTgtHczP6Xug8Aj77A+TsBuDuiT0YlhjNgx9tIU1nlSulGsjpwiEirURkgDvCKBfw8YXLXgUf\nP5h/I1SU4efrw7PTBhMa6MuvZ69ma6auFqOUqj+HCoeILBORCPvmTeuA10TkKfdGU/UWGQ9TnoPM\n9fDdPwBoExHEOzcMwxi48uUfWbpddw1UStWPoy2OSPuWsZcB7xhjhgET3RdLNVifKdDnl7DmNSgv\nsQ61j+Dj20aQEBvKjW+v4Z2V6R6NqJRqmhwtHH4i0g64CljkxjzKlZKvhxIbbD/5I2sbGcSHtwxn\nfK/W/PWTVP7+xTadXa6UcoqjheNRYDGw2xizRkS6AC1sokQTlDAaIjvB+vdOORwa6Mcr1yZz7bmd\neeW7PfznG/1RKqUc5+fIScaYuVj7cVQ/3gNc7q5QykV8fGDQNPjun5B3AKI6nnjK10d4dGpfyiqq\neHZJGiEBvtw6pqsHwyqlmgpHO8e7iMhCETkqIkdE5BN7q0N5u0HTAXNiXkdNIsL/XdafyQPb88QX\n23l3ZXpjp1NKNUGOXqr6L/Ah0A5oj9X6+MBdoZQLtUqAhFGw4T2o+vl+5L4+wlNXDWRi79b85ZNU\nPkw50PgZlVJNiqOFI8QY864xpsJ+ew8Icmcw5UKDr4Fj6bD/x1qf9vf14fnpSYzqHst98zbpJlBK\nqbOqawfAaPvcjS9EZJaIJIhIZxG5D/i8cSKqBus9BQLCYf37ZzwlyN+X136dzPl92vDQp6k8tyRN\nR1sppWpVV+f4Wqx9N6o3bbqlxnMGuN8doZSLBYRAv8tg81y46J8QGF7raUH+vrw0I4n75m3i31/v\nxFZczoMX90ZE9+xSSp101sJhjEk803Mi4u/6OMptBl8D696G1I8g6ddnPM3P14cnrxxIeJAfr3+/\nF1txOX+/rD9+vrqsmVLK4tRvA7FMEJE3AF1qtSmJPwdie8CaN6COS1A+PsLDU/pyx4TuzF2bwa3v\nraW4rLKRgiqlvJ2jw3HPFZFngX3AJ8ByoJc7gykXE4FzfweHNsCeZQ6cLtxzfg8em9qXJduPcM0b\nq8grKnN/TqWU16urc/z/RCQN+BuwCRgMHDXGvG2MOdYYAZULDZoOYW3he8fXp7x2eAIvTE9ic4aN\nK19eSWZesRsDKqWagrpaHL8BsoCXgHeNMTlYneKqKfILhOG3wd7lkJHi8Msu6t+Ot28YymFbCVOe\n/4EfdmW7MaRSytvVVTjaAY8Dk4HdIvIuECwiDi1VorxQ8vUQFAUrnFsVf3jXGOb/7jyiQvy55o1V\n/OfrnVRW6d8QSrVEde05XmmM+dIYMxPoCnwM/AAcFJH/NkZAZ4jIZBF51WazeTqK9woMh2G3wI7P\n4Mg2p17ao004n94+gksHd+CZJWlc8/oqDtn00pVSLY3UZ5KXiEQAvzTGvOP6SA2XnJxsUlIcvxTT\n4hTlwn/6WhMDL3ulXm8xN+UAf/lkC6UVVQzp1IqJfdowsXcbusaF6rwPpZogEVlrjEl26NzmODtY\nC4cDvnwAVr0Md6yz1rOqh/TsQj5af5BvtmWRat+OtkebMKYN7cRlg+OJDNGpPko1FVo4tHDUzXYQ\nnhkIQ2bCxf9u8Ntl5hXzzbYs5q/NYGOGjUA/Hy4e0I7fjOxCn/YRLgislHInLRxaOBzzyW2weT7c\nnQqhMS572y0HbXywej+fbMikrKKKxy/tx1XJHet+oVLKY5wpHA7PHBeR80Rkuoj8uvpW/4jKKwy/\nHSqKIeUNl75tvw6R/O3S/qy4bxznJLbivnmbeGRhKhWVP1/WXSnV9Dg6c/xd4ElgJHCO/eZQZVJe\nrHVv6HY+rH4Vyktc/vatQgN4+/qh3DAikTd/SOfXs1dzrFBnnyvV1Dna4kgGRhhjfmeM+b39doc7\ng6lGct7vofAobPqfW97ez9eHv07uw5NXDiRl3zEmPvUdb/6wl9IKXftKqabK0cKxBWjrziDKQxJH\nQ9v+sPKFWncIdJUrhsTz0e/Oo2fbcB5ZuJUJ//6OBesydBKhUk2QQ53jIrIUGASsBkqrjxtjprgv\nWv1p57iTNn0IC26C6XOhxwVu/ShjDN/vyuYfX25ny8F8WoX407NtOD3bhNOjbTijusXRKSbErRmU\nUj/n8lFVIjKmtuPGmO+czNYotHA4qbLcGpob3QWuW9QoH1lVZVicepjlaUfZcfg4O7MKKCitINjf\nlycu78/UQR0aJYdSyuJM4XBozSlvLRDKRXz9Ydit8PVfIHMDtB/k9o/08REu7N+OC/u3A6yWyN7s\nQmbN38ydczawfn8eD17cG3/dQEopr+PMfhxrRKRARMpEpFJE8t0dTjWiITOtfcl/eMYjHy8idIkL\n4/2bhnH9iATe+jGd6a/9xPbD+RzJL6GgtIIq7Q9Ryis4usrt88CvgLlYI6x+DfRwVyjlAUGR1uKH\nK560ll6P98xoa39fHx6a3JdBHaOYNX8zk55eccrzPduE85tRiUwd1IEAP22NKOUJjvZxpBhjkkVk\nkzFmgP3YemPMYLcnrAft46in0uPw3BCI6gw3fmXtGuhB+3IKWbvvGEVllRSVVVBQUsFXW7PYfvg4\n7SKDuHFkItOGdiI0UFf5V6qhXN7HARSJSACwQUT+CRzCyf3KVRMQGA7j/wKf3g5b5kP/Kzwap3NM\nKJ1jQk85dvf5PVi28ygvL9vN459t4/UVe3nyyoGM7B7roZRKtTyO/vK/1n7u7UAh0BG43F2hlAcN\nmm7N6/j6ISivY6+NA6sh9WOoKD37eS4kIozr2Zr/3TKcebcOJzTQl2veWMUjC1MpKddJhUo1BocX\nORSRYKCTMWaHeyM1nF6qaqC9K+DtS2D8n2H0H2s/pzgPnh0MxbkQHG0VnKSZENe4XV/FZZX848vt\nvPVjOt1bh3HT6C6UlleSX1JBfkk5XePCuCIpHh8f3SNEqbNxxzyOyVhrVQUYYxJFZBDwqE4AbMbm\nzIDdS+H3ayGi3c+f/+Zh+P4/cPFTsPc72P4ZVFXAkOtgcuOPzFq+8yh/nLeRrPyTrR8/H6GiypDU\nKYonLh9AjzbhjZ5LqabCHYVjLTAeWFbdIS4im40x/RuU1E20cLhAzm54YRj0ngxXzD61o9x2EJ5L\nsnYQvPw161jBEVj2hLXS7lXvQp/G/5uiuKySQ7ZiwoP8CQ/yI9DPh4/WH+SxRVspKK3g1jFduW1c\nN4L8fRs9m1Lezh2d4+XGGNtpW4LqoPrmLKYrjP0TfPs4tO0Ho/5w8rml/wemyrqUVS2sNVz4D8hY\nA5/9ARJGQkh0o0YODvClS1zYKccuS4pnbM/WPL5oK899u4sXl+0mOjSAmNAAokMDGNQxilvGdCUy\nWHcrVMpRjnaOp4rIdMBXRLqLyHPAj27MpbzBqHuh/5Ww5FFI/cg6lpUKG/8LQ2+GVp1PPd/XH6a+\nYPV7LH6w8fOeQXRoAE9dPYggvCCZAAAY2ElEQVT/3jSMW0Z3YUKv1nSMDqG4vJKXvtvNuCeX8d5P\n+3S/EKUc5OilqhDgQeACQIDFwGPGGNdv4uACeqnKhcpL4J0pcGgjXPc5fPcE7F8Fd244c4tiyWPW\nRMIZ86H7xMbN66TUTBuPLtzKqr259GwTzu3ju3FulxjiwgM9HU2pRqVbx2rhcK3CbHhtvDWSqtQG\nEx+GkXef+fyKUnh5FJQXwe9Wgn8I2A5Y/SZRnSC2e2Mld4gx1oKLf/t8GwdyrSHInWNCGNK5FWN7\ntubCfm11zSzV7LmscIjIp2d7sY6qakGO7oDXz4fAMGuklX/w2c8/sBreuABC46AkDyrtO/+Jr7V5\n1NhZdb9HIyuvrGJTho21+3JJST/G2n3HyCkso11kENedl8CvhnY6pS+kpLwSHxFd+kQ1C64sHEeB\nA8AHwCqsy1QneOuquVo43CR3DyAQnejY+StfhH0/WB3tMd2gVYK10+D696wl3Cc/C4mj3Jm4Qaqq\nDMt2HuG15XtZuSeH0ABfBnaMIruglKz8UmzF5YQH+XF5UjwzhnWiuw73VU2YKwuHL3A+MA0YAHwG\nfGCMSXVFUHfRwuHl9iyDhXfCsXQ47w44/1GPr4tVly0Hbcz+YS97swtpHR5Im4ggWocHknakgC82\nH6assoqhCdGM69WakABfAvx8CPTzISYskD7tIrTPRHk9t/RxiEggVgH5F/CIMeb5+kd0Ly0cTUBZ\nESx+ANa+aRWOEXd6OlG95RSUMm9tBv9dvZ99OUW1ntM6PJC+7SM4JzGaywbH0zYyqJFTKnV2Li0c\n9oJxMVbRSAA+BWYbYw42MKfbaOFoIqqqYP6NkLoArnwb+v7S04kaxBhDQWkFZRVVlFVWUVpexSFb\nCamZNrZm5rMl08bOrAJ8BMb0iOPqczoytmdrnZCovIIrL1W9A/QDPgfmGGO2uCaie2nhaEJqDved\nuRA6DvV0IrdKzy5k7toDzFubcWJ5FD8fIcjflyB/XzpFB3PJgPZcMqAdrSO0VaIajysLRxXWarhw\n6kxxAYwxJqLeKd1IC0cTU5gDr0+A0ny48WurM72Zq6isYkVaNpsP2igpr6SkvIri8go2ZdhIzczH\nR2B41ximDGzPpL7tiAw588z2kvJKVu3NZdmOI+QWltGnXQT9OkTSt30EUSEBjfhVqaZM53Fo4Wh6\nsnfBGxOhJN/a8zxhJCSMhoQRXjds1912HTnOpxsy+XRjJuk5RQT4+jC6RxxTBrWnf4dIDtmKOXis\nmMy8EjZm5PHj7mxKyqsI9PMhOjSAQ7aT83JbhwfSoVUw7aOC6RAVTMfoEHq0DqNHm3BahWpRUSc1\nq8IhIl2wZq1HGmMc2llIC0cTlZ1mDddN/x4yUqCqHOJ6w2++tjaZamGMMWzKsPHpxkwWbco8ZeXf\nap1jQhjbI46xvVozvEsMQf6+5BaWkZpptVx2HSkgM6/YfiuhrMayKnHhgfRqG06fdhH0aR9B3/YR\nJMaG4atL0LdIXlM4RGQ2cAlwxBjTr8bxScAzgC/wujHmCQfea54WjhakrBC2fw4f3Qy9LrZW3PXy\nIbvuVFllWJOeS8axYtpHBdEhKpi2kUEE+jnesV5VZTiUX8LOrOOkZR1nZ1YB2w/ns/NwwYmCIgJh\ngX5E2FcY7tM+ggcu6k1smA4nbu68qXCMBgqAd6oLh31uyE6s+SEZwBqsEVu+wN9Pe4sbjDFH7K/T\nwtES/fgcfPVnmPAQjLrH02mapfLKKnYdKSA1M5/9uUUcLyknv7gCW3EZy9OyCQv04/8u7cekfqfu\ny5JbWMaR4yUkxITqyLBmwB3LqteLMWa5iCScdngosMsYswdAROYAU40xf8dqnSh10vDbIXM9fPsY\ntBsI3SZ4OlGz4+/rQ+92EfRu9/OxLjuzjnPPhxu49b11XDq4AzPPS+DH3dks2XaEdfuPYQz4iLU/\nfPfWYcSEBWIrLiO3sIxjheVEBPtxzbmduah/O13vqxlxex+HvXAsqtHiuAKYZIz5jf3xtcAwY8zt\nZ3h9DPA3rBbK6/YCU9t5NwM3A3Tq1GnIvn37XPyVKI8pK7TWyTqeCTcvs5YuUY2mvLKK577dxQtL\nd1FZZf2+6N8hkvG9WtMlLpTdRwvZdcS69JVXVEarkADrFupPWlYBe7ILT673dU6nWkeI2YrL+WZr\nFlsybRSWVlBYVklhaQVRwf5cfU4nzu0SjbTgS5WNwWsuVdnDJNCAwlEfeqmqGcrdA6+OBb9gGH0v\nJP0a/PS6e2PampnPtkP5jOweSxsH55hUr/f1+oq9/Lg7Bx+B7q3DGdgxkoEdo/AR4csth/lxdzbl\nlYaQAF8igvwJCfQlLNCPfTlF2IrL6dEmjGuHJ3BRv7b4+/lQ/WsryN/HqX4edWbeXjiGAw8bY35h\nf3w/wJlaEvWhhaOZOrjW2iBq/0qI6GD1eQy+VgtIE5GaaeOr1Cw2ZuSx8UAex4rKAegYHcxF/dox\nqV9bBsZH4VNjVFdJeSWfbszknZXpbDmY/7P39PURerQJZ0CHSPrHR9KnfQSdo0OIDg3QFoqTvL1w\n+GF1jk8ADmJ1jk935cKJWjiaMWOsRRKX/R0OrLKG68740NrnQzUZxhj25xZRUl5FjzZhdf6SN8aw\n/kAe6/YdO3FMRMgtLGXzwXw2ZeSRZy9EAKEBvnSMDqFVSAAFpRUUlFZwvKQcEBJjQ+gaF0aXuFDa\nRgafeH9jIDTQj77tI2gXGdTiCo/XFA4R+QAYC8QCWcBDxpg3ROQi4GmskVSzjTF/c+XnauFoAYyB\nnV/CglusFsf0/0GHJE+nUh5ijCHjWDHbDx/nQG4R+3OLyDhWRF6RtfR9mH14cWWlYW92IbuPFpBT\nWHbG94sNC6Bfh0iGJcYwfVinFrEnvdcUjsYmIpOByd26dbspLS3N03FUYziyHd6/Eoqy4YrZ0PNC\nTydSTUReURlHj5ciYrVeBMgrLmfLQRubMmxsyshjZ1YB4UF+3DAikRtGJBIZ4k9pRSWr9uSyZFsW\naUcKiAjyJyrEn8gQf0L8/aioqjqx0GVIgC+ju8cxpHMr/Lx8VFmLLRzVtMXRwhzPgg9+BYc2wKg/\nWB3neulKuUBqpo1nl6SxODWL8EA/hiS0Ys3eXArLKgny96FX2wgKSyuwFZeTV1xOWUUVPgIBfj74\n+/pQXFZJRZUhMtifMT3iGNktlvjoYNpHWhM4vWn+ixYOLRwtT1kRfPI7SP3IetxxGPS7AvpdDqEx\nns2mmrytmfk8920aWw/lM6JbLBN7t+a8rrE/+8VfWWVOWbLleEk536dls2T7EZZuP/Kzy2ORwf74\n+/oQ4Cv4+/kQFezPkM7RnNslmqGJ0UQG+5OeU8T6/cdYt/8YeUXlXNS/HeN7uX45fi0cWjharmPp\nsGU+bJ4PR1IhMAImPgRDbgAf775UoJq3qirDvtwiDuUVk2kr4VBeMTmFZZRVVlFeUUV5ZRWH80tY\nvz+P0oqqE8u/HC+pAKz7Qf6+ZBeUEhHkx+SB7blkQHv6x0cSFtjwudxaOLRwKIDDW6xdBvd+B/Hn\nwCVPQ9t+db9OKQ8qrahk4wEbq/bkkGkrYUB8JEmdWtGtdRgAP+zKZv66DBanHqak3FpjrFN0CL3b\nhdOrbQQ3j+5CaD0KiRYOLRyqmjGw6UNYfD8U58HoP8KYP2nrQzV5x0vKWbUnl+2H89l26DjbDuWT\naStmy8O/qFdHfIstHDqqSp1RUS58eT9smmP1ffzyRZ04qJqd0orKes+kd6ZwNKs/u4wxC40xN0dG\nRno6ivI2IdFw6csw8WHYMg/evQyKj9X1KqWalMZafqVZFQ6lzkoERt4Nl71uzTqfPQny9ns6lVJN\njhYO1fIMuBKu/QjyD8EbF1jb1iqlHKaFQ7VMiaPghi+gshzeugiO7vB0IqWaDC0cquVq0xeu+8wa\nefXmRZC11dOJlGoS3LoDoFJer3UvuP5zeHsyvHUxnP8oHD9ktUCyd4KvP3Q7H3pcAO0G6zBepdDh\nuEpZcnbD21MgP8N6HNkJ4npAST5krAEMhMZB8g0w9n6ro12pZqTFzuOophMAVb2U2KwlS6K7QmDY\nyeOFObB7CWxZADu/gNH3wfgHPRZTKXdosfM4lGqQoEhoN/DUogHWIokDroJpH1gr7y7/J6x61TMZ\nlfIC2sehlKNE4OL/WLPQv7jPKij9Lv/5eQVHYPtnsH2RtWpvjwug50UQ20MvcalmQS9VKeWs8mJr\n5nnGGqsz3dffmoVenAeZ62D/T4CBVokQFAGHNlqvi+4CvSfDwOlWp7xSXkT7OLRwKHcrzoO3LoGs\nzSeP+YdCTBfodYlVIFr3sVoY+ZnWNrfbP4c9S6GqAjoMgUHTrXWzgqPqlyFrqzUCrNsE13xNqkXT\nwqGFQzWGijLI2wdBUVb/iF9A3a8pOAqbP4T179v3C4mE0ffCsFucW3Rxx5cw73ooL4JR98K4B3Wo\nsGoQLRxaOJS3MwYy18Oyv0PaVxDV2brs1Wdq3f0gKW/CZ/dA2wHWJMYN70PfS+GXL4F/cOPkV81O\nix1VJSKTReRVm83m6ShKnZ0IdEiCGXPhmgUQEApzZ8J7l0FpQe2vMQa+fRwW3QVdJ1iz3qe+YBWc\n1I+tS2cFRxr361AtkrY4lPIGVZWw5g34cpa1W+GMuVbHerWSfFh4J6QugMHXWrsZ+tYYFLltIcy/\nCUyl1bfSfpA1tDhhNMR2a/yvRzU5eqlKC4dqqlI/gvm/gfaD4Zr5Vt9J5nqYe73VnzL+L9bS8LVd\nzspKhY1z4NAGayRXiQ0Q6PtLGDPr5EiuygrYvhBWvWJtrxvcytqvJCQaIuOhQzJ0HAqxPbXfpAXR\nwqGFQzVl2xZahaJtP6vvYsljENYaLn8DOg937D2MgWN7Yf178NPLVid6v8uhdW+rjyQ/A1olWOtw\nleZbc1OKcyF3z8kNrgIjocNgaNsf2g60/o3pdmpLx50KcyB3tzWMOSRG58C4mRYOLRyqqdvxJXx4\nLVSWQY9JVsd3SHT93qswB358Fla/ahWQxNEw7LfQ4xfgc9qOccZY63ZlrIYDq63WzpGtVg6AwAjr\n9V3HW8OAQ1vDwbXW3JX9K63O+Qv/CZEdTn3fqipY9bLVGho0HRLHnLkQZKyFNa/BlvknPzcoCmK7\nQ1gbqCix5tKUF1mtpaSZ1hDoxipozZQWDi0cqjnY96O1dtbAaa75a7so17p8FZ3o3Osqy62Vgg9t\nsorD7m/BdsB6TnzAVAFitWaO7QP/ILjstZPzSwqOwse3wq5vrLku5YUQ18sagtxtItgOWq2jY+mQ\n9rU1iTIgzPq6u4y1LtFlp0HOLijMtopTQKj179Ht1i6OER2sBSiHXAehsQ3/XrVAWji0cCjlPsZY\nv8h3L4GiHIgfCh3Psf76P7rTGh12ZBuMuQ86DYePbrUuf036OwyaYfXjrHrp5Iz6E+zFJ/kGGHD1\nqYMDzqSqEnYuhtWvwJ5lEBAOv/ibtaaYXtpyihYOLRxKeU5ZEXz2B9j4X+txTDe48i2rj6SaMdal\nsKwt1hyW6ESrY96ZSZCnO7INPv8jpK+w+m6mPAsR7e1zZtbB1k+tVk2bftaIs3YDIbxNQ75S99q5\n2LoEOO4Ba1kbN9PCoYVDKc/b8IFVGMbe//MVh92lqgrWvA7fPAQ+/taIst1LwbYffPysQpK3/+T5\nUZ1h8DXWLaK9c59VnAcHVlkd9+Ftrf6es60ekJ0GmRus/p+ozhDervZRa+XF8NVfrH4egOQb4eJ/\nu70F1WILh27kpJQCrA7+T26zOu67jofeU6DnhdYAg5J8OLzZulS280vY+53VV9P9Amv5/LYDrAUq\nz9bZnr0L3r/C6pupKSIeuo2H7r+w+md8/a1RcmvfslpCNfkGWHu/dBkDXcZBwkirP2fejXB0Gwy/\n3Tpv5fPWgINht7jwG/RzLbZwVNMWh1IKsOas1DXaKncPrHvXWrqlIMs65htojeKKT4aht0CbPifP\n3/8TfGAfsDD5WaslU3AYjmdZLaw9y6whzr4BVid+8TGI6mR13He/wPqMY/usls/hzdYgiIpiq4Uk\nYo0gu/Qla+BAVRX87xprA7Hpc6H7RHd9p7RwaOFQSjmtstz6RX50uzUE+cg265d6eZH1C3/EnVB4\nFBbcYvXHXDPPmmNS2/vsX2mtQVZwxGrFdBl/5smU5SVw4CdrtFpZoTVZMyzu5POlBTB7ktUaufHr\nny/JX5Rrz7zNOuf8R+v15Wvh0MKhlHKFolyrz2TVK1CUbR3reK61G2R959XUhy0DXhtvFZHgKBBf\nq3VSXmQVs2oBYXDPVmvFASdp4dDCoZRypfJi2PBf6y/6sQ9Yc1UaW1YqrH7NatGYKmtdMt8Aa2fJ\n1r2tuTGR8fXuRHemcOhUS6WUqot/MJxzo2cztOkLk5/2bAY7XcFMKaWUU7RwKKWUcooWDqWUUk7R\nwqGUUsopWjiUUko5RQuHUkoppzSrwiEik0XkVZvN5ukoSinVbDWrwmGMWWiMuTky0vlZk0oppRzT\nLGeOi8hRYF89Xx4LZLswjqtpvobRfA2j+RrGm/N1NsbE1X1aMy0cDSEiKY5Ou/cEzdcwmq9hNF/D\neHs+RzWrS1VKKaXcTwuHUkopp2jh+LlXPR2gDpqvYTRfw2i+hvH2fA7RPg6llFJO0RaHUkopp2jh\nsBORSSKyQ0R2icgsT+cBEJHZInJERLbUOBYtIl+LSJr931YeytZRRJaKyFYRSRWRO70pnz1LkIis\nFpGN9oyP2I8nisgq+8/6fyIS4MGMviKyXkQWeVs2e550EdksIhtEJMV+zJt+xlEiMk9EtovINhEZ\n7i35RKSn/ftWfcsXkbu8JV9DaOHA+p8XeAG4EOgDTBORPmd/VaN4C5h02rFZwBJjTHdgif2xJ1QA\nfzDG9AHOBW6zf8+8JR9AKTDeGDMQGARMEpFzgX8A/zHGdAOOAZ7coedOYFuNx96Urdo4Y8ygGsNI\nveln/AzwpTGmFzAQ63vpFfmMMTvs37dBwBCgCPjIW/I1iDGmxd+A4cDiGo/vB+73dC57lgRgS43H\nO4B29vvtgB2ezmjP8glwvhfnCwHWAcOwJmD51fazb+RM8Vi/OMYDiwDxlmw1MqYDsacd84qfMRAJ\n7MXeV+tt+U7LdAHwg7fmc/amLQ5LB+BAjccZ9mPeqI0x5pD9/mGgjSfDAIhIAjAYWIWX5bNfCtoA\nHAG+BnYDecaYCvspnvxZPw3cB1TZH8fgPdmqGeArEVkrIjfbj3nLzzgROAq8ab/c97qIhHpRvpp+\nBXxgv++N+ZyihaMJM9afLB4dFiciYcB84C5jTH7N57whnzGm0liXCuKBoUAvT+apJiKXAEeMMWs9\nnaUOI40xSViXcW8TkdE1n/Twz9gPSAJeMsYMBgo57bKPN/w3aO+nmgLMPf05b8hXH1o4LAeBjjUe\nx9uPeaMsEWkHYP/3iKeCiIg/VtF43xizwNvy1WSMyQOWYl3+iRIRP/tTnvpZjwCmiEg6MAfrctUz\nXpLtBGPMQfu/R7Cuzw/Fe37GGUCGMWaV/fE8rELiLfmqXQisM8Zk2R97Wz6naeGwrAG620e0BGA1\nKz/1cKYz+RSYab8/E6tvodGJiABvANuMMU/VeMor8gGISJyIRNnvB2P1wWzDKiBX2E/zSEZjzP3G\nmHhjTALWf2/fGmNmeEO2aiISKiLh1fexrtNvwUt+xsaYw8ABEelpPzQB2IqX5KthGicvU4H35XOe\npztZvOUGXATsxLoG/qCn89gzfQAcAsqx/rq6Ees6+BIgDfgGiPZQtpFYTexNwAb77SJvyWfPOABY\nb8+4Bfir/XgXYDWwC+vyQaCHf85jgUXels2eZaP9llr9/4WX/YwHASn2n/HHQCsvyxcK5ACRNY55\nTb763nTmuFJKKafopSqllFJO0cKhlFLKKVo4lFJKOUULh1JKKado4VBKKeUULRxK1YOIVJ628qnL\nFqoTkYSaKyIr5W386j5FKVWLYmMtZaJUi6MtDqVcyL5/xT/te1isFpFu9uMJIvKtiGwSkSUi0sl+\nvI2IfGTfM2SjiJxnfytfEXnNvo/IV/aZ70p5BS0cStVP8GmXqq6u8ZzNGNMfeB5rBVyA54C3jTED\ngPeBZ+3HnwW+M9aeIUlYM7QBugMvGGP6AnnA5W7+epRymM4cV6oeRKTAGBNWy/F0rM2j9tgXgTxs\njIkRkWysPRjK7ccPGWNiReQoEG+MKa3xHgnA18ba6AcR+RPgb4x53P1fmVJ10xaHUq5nznDfGaU1\n7lei/ZHKi2jhUMr1rq7x70r7/R+xVsEFmAGssN9fAvwWTmw6FdlYIZWqL/0rRqn6CbbvLFjtS2NM\n9ZDcViKyCavVMM1+7PdYO9X9EWvXuuvtx+8EXhWRG7FaFr/FWhFZKa+lfRxKuZC9jyPZGJPt6SxK\nuYteqlJKKeUUbXEopZRyirY4lFJKOUULh1JKKado4VBKKeUULRxKKaWcooVDKaWUU7RwKKWUcsr/\nA3gMSCGaohBSAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"c1hhaJX5EjE4","colab_type":"code","outputId":"1a2a78bf-72e5-4d64-9dfe-b7912bacb7ae","executionInfo":{"status":"ok","timestamp":1558013202862,"user_tz":-120,"elapsed":383565,"user":{"displayName":"Hans-Peter Störr","photoUrl":"https://lh5.googleusercontent.com/-UsuI5t4TsSg/AAAAAAAAAAI/AAAAAAAAAUU/oy9GqeEoAZE/s64/photo.jpg","userId":"04327305437195241378"}},"colab":{"base_uri":"https://localhost:8080/","height":373}},"source":["def nextchar(charlist):\n","  step = model.predict(enc.transform([charlist[-PRECHARS:]]))[0,:]\n","  return np.random.choice(lenc.categories_[0], p=step)\n","\n","for k in range(0, 10):\n","  start = np.random.randint(len(text)-PRECHARS)\n","  charlist = list(text[start:start+PRECHARS])\n","\n","  for i in range(0, 200):\n","    charlist.append(nextchar(charlist))\n","\n","  print(\"\".join(charlist))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["mit wir uns nachkommen schaffen von unserm vater. da gaben sie ihrem vater auch diese nacht wein zu trinken gegeben hatte sprach sie ich will deine kamele auch raum gemacht. da f hrte er den mann ins haus und z umt\n","n. da ging alles fleisch hatte seinen weg verderbt auf erden. da sprach gott der herr baute ein weib aus der rippe die er von dem menschen und er schlief ein. und er nahm eine seiner rippen und schlo die stelle mit\n","onat. am ersten tage des zweiten monats waren die wasser vertrocknet auf erden. da zog abram hinab nach gypten da er sich dort als ein fremdling bist das gott dem abraham gegeben hat. so entlie isaak den jakob da e\n"," sehr um seines sohnes statt. und abraham stellte sieben l mmer sollst du von meiner hand nehmen damit sie fr verhaut wind er ausgerottet werden aus seinem volk weil er meinen bund gebrochen hat. und gott sprach da\n"," m chtig geworden. da zog isaak von dannen und schlug sie und jagte sie bis nach hoba das n rdlich der stadt bethel und schlug sein zelt auf so da er bethel im westen und ai im osten hatte und baute dort dem herrn \n","gott breite jafet aus und lasse ihn wohnen in den zelten sems und kanaan sei sein knecht gott breite jafet aus und lasse ihn wohnen in den zelten sems und kanaan sei sein knecht noah aber lebte nach der sintflut. v\n","h getan habe. solange die erde steht soll nicht aufh ren saat und ernte frost und hitze sommer und winter tag und nacht und geben zeichen zeiten tage und jahre und seien lichter an der feste des himmels da sie schi\n","te danach jahre und zeugte s hne und t chter da sein ganzes alter ward jahre und starb. mahalalel war jahre alt und zeugte serug und lebte danach jahre und zeugte s hne und t chter da sein ganzes alter ward jahre u\n","ine augen auf und sah einen widder hinter sich in der hecke mit seinen h rnern h ngen und ging hin und setzte sich gegen ber und erhob ihre stimme und weinte. da erh rte gott die stimme des knaben. und der engel go\n","ir nicht so bist du deines eides ledig sein wenn du zu meiner verwandtschaft und meines vaters hause. dann sollst du deines eides ledig sein wenn du zu meiner verwandtschaft kommst und sie geben sie dir nicht so bi\n"," was da wimmelte auf erden und ber allen fischen im meer in eure h nde seien sie gegeben. allein esset kamen nach ihren geschlechtern und v lkern. von denen her haben sich die geschlechter auf dem ganzen erdboden. \n","den die ein jeder nach seiner art und b ume die da fr chte tragen in denen ihr same ist jekobses wieder zu ihr kommehe und mit ihm will ich meinen bgewentin der sollen gesegnet werden sollen denn dazu habe ich ihn \n","und alle vorr te und zogen davon. aach vierzig tagen tat noah an der arche das fenster auf das er gemacht hatte und lie einen raben ausfliegen der flog immer hin und her bis die wasser vertrockneten auf erden. noah\n","n jedes nach seiner art alles vieh nach seiner art und das vieh nach seiner art und alle v gel nach ihrer art alles was fliegen konnte alles was fittiche hatte das ging alles zu noah in die arche paarweise von alle\n","ans sind diese elischa tarsis die kitt er und die rodaniter. von diesen haben sich ausgebreitet und ihre grenzen waren von sidon in der richtung auf sodom gomorra adma zebojim bis nach lescha. das sind die s hne we\n"," sprach die erde bringe hervor lebendiges getier ein jedes nach seiner art und b ume die da fr chte tragen in denen ihr same ist. und es geschah so. und gott nannte das trockene erde und eiles vater vielten. und zu\n","wo dir s wohlgef llt. und zu sara sprach er siehe da ich habe euch gegeben alle pflanzen die samen bringen zu eurer speise. aber allen tieren auf erden und fruchtbare b ume auf erden und ihnen nichts tun. und er sp\n","remdling in der philister. da sprach isaak zu jakob tritt herzu mein sohn da ich dich betaste ob du mein sohn esau sprach zu seinem vater ich bin esau dein erstgeborener sohn ich habe getan wie du mir gesagt hast. \n"," hetiter vierhundert lot silber nach dem gewicht das im kauf gang und g be war. so wurde efrons acker in machpela stlich von mamre das ist hebron im lande kanaan gewohnt hatten. und er ging zu hagar die ward schwan\n","ihm eine frau aus gyptenland. abrahams bund mit abimelech und ein wegde sollte in keinem sohn wie hast du so bald gefunden mein sohn er aber antwortete siehe ich much einen tiesen m nnern tut nichts denn darum sind\n"],"name":"stdout"}]}]}