1. CVAE Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{u'catkin_ws_dir': u'/home/schmrlng/catkin_ws',\n",
       " u'data_dir': u'/data/trafficweaving',\n",
       " u'feature_extraction_dict': u'slim',\n",
       " u'models_dir': u'/data/trafficweaving/models'}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "# Set below to 0 or 1 for GPU\n",
    "%env CUDA_VISIBLE_DEVICES=\n",
    "from __future__ import absolute_import, division, print_function\n",
    "import json\n",
    "import os\n",
    "if os.path.isfile(\"config.json\"):\n",
    "    with open(\"config.json\", \"r\") as f:\n",
    "        config = json.load(f)\n",
    "else:\n",
    "    print(\"Please run setup.py in this directory before running any .ipynb's.\")\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import rosbag\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import tensorflow as tf\n",
    "import time\n",
    "import shutil\n",
    "import pickle\n",
    "from collections import OrderedDict, defaultdict\n",
    "\n",
    "from utils.bags import *\n",
    "from utils.learning import *\n",
    "from traffic_weaving_models import *\n",
    "\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/data/trafficweaving/models/CPU_checkpoints'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_type = TrafficWeavingCVAE\n",
    "cvd = %env CUDA_VISIBLE_DEVICES\n",
    "model_dir = os.path.join(config[\"models_dir\"], (\"GPU\" + cvd if cvd else \"CPU\") + \"_checkpoints\")\n",
    "# shutil.rmtree(model_dir, ignore_errors=True)\n",
    "\n",
    "sc = tf.ConfigProto(device_count={'GPU': 1}, allow_soft_placement=True, log_device_placement=False)\n",
    "rc = tf.estimator.RunConfig().replace(session_config=sc, model_dir=model_dir,\n",
    "                                      save_summary_steps=100, save_checkpoints_steps=5000,\n",
    "                                      keep_checkpoint_max=5000, tf_random_seed=None)\n",
    "rc.environment = None    # required due to a bug in tf.contrib.learn.Experiment.train_and_evaluate\n",
    "model_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'GMM_components': 16,\n",
       " 'K': 5,\n",
       " 'MLP_dropout_keep_prob': 0.9,\n",
       " 'MVG_latent_dim': 32,\n",
       " 'N': 2,\n",
       " 'alpha': 1,\n",
       " 'batch_size': 32,\n",
       " 'dec_GMM_proj_MLP_dims': None,\n",
       " 'dec_rnn_dim': [128],\n",
       " 'dec_sample_model_prob_crossover': 20000,\n",
       " 'dec_sample_model_prob_divisor': 6,\n",
       " 'dec_sample_model_prob_final': 0.0,\n",
       " 'dec_sample_model_prob_start': 0.0,\n",
       " 'enc_rnn_dim_future': [32],\n",
       " 'enc_rnn_dim_history': [32],\n",
       " 'fuzz_factor': 0.05,\n",
       " 'grad_clip': 1.0,\n",
       " 'k': 3,\n",
       " 'k_eval': 50,\n",
       " 'kl_crossover': 8000,\n",
       " 'kl_decay_rate': 0.99995,\n",
       " 'kl_exact': True,\n",
       " 'kl_min': 0.07,\n",
       " 'kl_sigmoid_divisor': 6,\n",
       " 'kl_weight': 1.0,\n",
       " 'kl_weight_start': 0.0001,\n",
       " 'latent_type': 'discrete',\n",
       " 'learning_decay_rate': 0.9999,\n",
       " 'learning_rate': 0.001,\n",
       " 'log_p_yt_xz_max': 50,\n",
       " 'log_sigma_max': 10,\n",
       " 'log_sigma_min': -10,\n",
       " 'min_learning_rate': 1e-05,\n",
       " 'minimum_history_length': 5,\n",
       " 'optimizer': tensorflow.python.training.adam.AdamOptimizer,\n",
       " 'optimizer_kwargs': {},\n",
       " 'p_z_x_MLP_dims': [16],\n",
       " 'prediction_horizon': 15,\n",
       " 'q_z_xy_MLP_dims': None,\n",
       " 'rnn_cell': tensorflow.contrib.rnn.python.ops.rnn_cell.LayerNormBasicLSTMCell,\n",
       " 'rnn_cell_kwargs': {'dropout_keep_prob': 0.75, 'layer_norm': False},\n",
       " 'rnn_io_dropout_keep_prob': 1.0,\n",
       " 'sample_model_during_dec': True,\n",
       " 'tau_decay_rate': 0.9999,\n",
       " 'tau_final': 0.001,\n",
       " 'tau_init': 2.0,\n",
       " 'use_iwae': False,\n",
       " 'use_z_logit_clipping': False,\n",
       " 'z_logit_clip_crossover': 8000,\n",
       " 'z_logit_clip_divisor': 6,\n",
       " 'z_logit_clip_final': 3.0,\n",
       " 'z_logit_clip_start': 0.05}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Edit model hyperparameters if desired, e.g.:\n",
    "# hps.N = 2\n",
    "# hps.K = 5\n",
    "hps.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "extract_dict = feature_extraction_dicts[config[\"feature_extraction_dict\"]]\n",
    "array_name = \"trajectories_\" + config[\"feature_extraction_dict\"]\n",
    "base_filename = os.path.join(config[\"data_dir\"], array_name)\n",
    "\n",
    "all_bags = pd.read_pickle(base_filename + \".pkl\")\n",
    "train_bags, eval_bags, test_bags = split_bag_df(all_bags, train_eval_test_split = [.8, .2, 0.], seed=123)\n",
    "\n",
    "train_data_dict = prepare_data(train_bags, extract_dict=extract_dict)\n",
    "save_as_npz_and_hdf5(base_filename + \"_train\", train_data_dict[\"input_dict\"])\n",
    "hps.add_hparam(\"cars_standardization\", {\"mean\": train_data_dict[\"cars_mean\"],\n",
    "                                        \"std\": train_data_dict[\"cars_std\"]})\n",
    "hps.add_hparam(\"extras_standardization\", {\"mean\": train_data_dict[\"extras_mean\"],\n",
    "                                          \"std\": train_data_dict[\"extras_std\"]})\n",
    "hps.add_hparam(\"label_standardization\", {\"mean\": train_data_dict[\"labels_mean\"],\n",
    "                                         \"std\": train_data_dict[\"labels_std\"]})\n",
    "\n",
    "hps.add_hparam(\"pred_indices\", train_data_dict[\"pred_indices\"])\n",
    "eval_data_dict = prepare_data(eval_bags, extract_dict=extract_dict)\n",
    "save_as_npz_and_hdf5(base_filename + \"_eval\", eval_data_dict[\"input_dict\"])\n",
    "\n",
    "train_input_function = tf.estimator.inputs.numpy_input_fn(train_data_dict[\"input_dict\"],\n",
    "                                                          y = train_data_dict[\"labels\"],\n",
    "                                                          batch_size = hps.batch_size,\n",
    "                                                          num_epochs = None,\n",
    "                                                          shuffle = True)\n",
    "\n",
    "eval_input_function = tf.estimator.inputs.numpy_input_fn(eval_data_dict[\"input_dict\"],\n",
    "                                                         y = eval_data_dict[\"labels\"],\n",
    "                                                         batch_size = 4,\n",
    "                                                         num_epochs = 1,\n",
    "                                                         shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': None, '_session_config': device_count {\n",
      "  key: \"GPU\"\n",
      "  value: 1\n",
      "}\n",
      "allow_soft_placement: true\n",
      ", '_keep_checkpoint_max': 5000, '_tf_random_seed': None, 'environment': None, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_save_checkpoints_steps': 5000, '_model_dir': u'/data/trafficweaving/models/CPU_checkpoints', '_save_summary_steps': 100}\n"
     ]
    }
   ],
   "source": [
    "laneswap_model = model_type()\n",
    "nn = tf.estimator.Estimator(laneswap_model.model_fn, params=hps, config=rc, model_dir=model_dir)\n",
    "experiment = tf.contrib.learn.Experiment(nn, train_input_function, eval_input_function,\n",
    "                                         train_steps=50000, min_eval_frequency=1, eval_steps=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/schmrlng/.local/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/monitors.py:269: __init__ (from tensorflow.contrib.learn.python.learn.monitors) is deprecated and will be removed after 2016-12-05.\n",
      "Instructions for updating:\n",
      "Monitors are deprecated. Please use tf.train.SessionRunHook.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-08:27:38\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-1\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-08:27:46\n",
      "INFO:tensorflow:Saving dict for global step 1: NLL_exact = 43.0704, NLL_p = 43.0737, NLL_q (IS) = 43.0869, global_step = 1, loss = 43.0869\n",
      "INFO:tensorflow:Validation (step 1): loss = 43.0869, NLL_p = 43.0737, NLL_exact = 43.0704, NLL_q (IS) = 43.0869, global_step = 1\n",
      "INFO:tensorflow:loss = 39.3238, step = 1\n",
      "INFO:tensorflow:kl_weight = 0.00257238, lr = 0.001, temp = 2.0\n",
      "INFO:tensorflow:global_step/sec: 5.06749\n",
      "INFO:tensorflow:loss = 11.1289, step = 101 (10.231 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00276442, lr = 0.000990147, temp = 1.98011 (10.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2667\n",
      "INFO:tensorflow:loss = -3.66838, step = 201 (9.740 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00297134, lr = 0.000980392, temp = 1.96041 (9.740 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3256\n",
      "INFO:tensorflow:loss = -10.7186, step = 301 (9.685 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00319428, lr = 0.000970735, temp = 1.94091 (9.685 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1762\n",
      "INFO:tensorflow:loss = -11.1563, step = 401 (9.829 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00343447, lr = 0.000961173, temp = 1.9216 (9.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2981\n",
      "INFO:tensorflow:loss = -12.2557, step = 501 (9.709 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00369324, lr = 0.000951707, temp = 1.90249 (9.709 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2642\n",
      "INFO:tensorflow:loss = -13.9393, step = 601 (9.743 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00397202, lr = 0.000942335, temp = 1.88356 (9.743 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2865\n",
      "INFO:tensorflow:loss = -14.1639, step = 701 (9.722 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00427233, lr = 0.000933056, temp = 1.86483 (9.722 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2253\n",
      "INFO:tensorflow:loss = -12.2023, step = 801 (9.779 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00459582, lr = 0.000923869, temp = 1.84628 (9.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1879\n",
      "INFO:tensorflow:loss = -16.6505, step = 901 (9.815 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00494428, lr = 0.000914774, temp = 1.82791 (9.815 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3333\n",
      "INFO:tensorflow:loss = -21.0251, step = 1001 (9.678 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0053196, lr = 0.00090577, temp = 1.80973 (9.678 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.176\n",
      "INFO:tensorflow:loss = -18.9529, step = 1101 (9.827 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00572384, lr = 0.000896855, temp = 1.79173 (9.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2402\n",
      "INFO:tensorflow:loss = -17.0599, step = 1201 (9.765 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0061592, lr = 0.000888028, temp = 1.77391 (9.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.238\n",
      "INFO:tensorflow:loss = -15.5178, step = 1301 (9.768 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00662803, lr = 0.00087929, temp = 1.75626 (9.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3699\n",
      "INFO:tensorflow:loss = -16.6101, step = 1401 (9.643 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00713288, lr = 0.000870639, temp = 1.73879 (9.643 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3856\n",
      "INFO:tensorflow:loss = -17.3069, step = 1501 (9.629 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00767648, lr = 0.000862073, temp = 1.7215 (9.629 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.312\n",
      "INFO:tensorflow:loss = -15.8941, step = 1601 (9.698 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00826175, lr = 0.000853593, temp = 1.70438 (9.698 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2443\n",
      "INFO:tensorflow:loss = -20.0865, step = 1701 (9.762 sec)\n",
      "INFO:tensorflow:kl_weight = 0.00889184, lr = 0.000845197, temp = 1.68742 (9.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1829\n",
      "INFO:tensorflow:loss = -16.8492, step = 1801 (9.820 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0095701, lr = 0.000836885, temp = 1.67064 (9.820 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1995\n",
      "INFO:tensorflow:loss = -19.4657, step = 1901 (9.804 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0103001, lr = 0.000828656, temp = 1.65402 (9.805 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2987\n",
      "INFO:tensorflow:loss = -18.4513, step = 2001 (9.710 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0110858, lr = 0.000820508, temp = 1.63757 (9.710 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2942\n",
      "INFO:tensorflow:loss = -20.9999, step = 2101 (9.714 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0119313, lr = 0.000812442, temp = 1.62128 (9.714 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2591\n",
      "INFO:tensorflow:loss = -19.5069, step = 2201 (9.747 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0128411, lr = 0.000804456, temp = 1.60516 (9.748 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2299\n",
      "INFO:tensorflow:loss = -17.6774, step = 2301 (9.775 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0138198, lr = 0.000796549, temp = 1.58919 (9.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3217\n",
      "INFO:tensorflow:loss = -17.2935, step = 2401 (9.688 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0148726, lr = 0.000788721, temp = 1.57339 (9.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2701\n",
      "INFO:tensorflow:loss = -20.3584, step = 2501 (9.737 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0160048, lr = 0.000780971, temp = 1.55774 (9.737 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2939\n",
      "INFO:tensorflow:loss = -21.5373, step = 2601 (9.714 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0172223, lr = 0.000773298, temp = 1.54225 (9.715 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3424\n",
      "INFO:tensorflow:loss = -17.8457, step = 2701 (9.669 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0185313, lr = 0.000765702, temp = 1.52691 (9.669 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3092\n",
      "INFO:tensorflow:loss = -20.3634, step = 2801 (9.700 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0199383, lr = 0.000758181, temp = 1.51172 (9.700 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2556\n",
      "INFO:tensorflow:loss = -20.1609, step = 2901 (9.751 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0214504, lr = 0.000750734, temp = 1.49669 (9.751 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3345\n",
      "INFO:tensorflow:loss = -16.0167, step = 3001 (9.676 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0230751, lr = 0.000743362, temp = 1.4818 (9.676 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2901\n",
      "INFO:tensorflow:loss = -17.9118, step = 3101 (9.718 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0248202, lr = 0.000736064, temp = 1.46706 (9.717 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.192\n",
      "INFO:tensorflow:loss = -19.3112, step = 3201 (9.811 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0266943, lr = 0.000728838, temp = 1.45247 (9.811 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1364\n",
      "INFO:tensorflow:loss = -22.3732, step = 3301 (9.866 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0287063, lr = 0.000721684, temp = 1.43803 (9.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2654\n",
      "INFO:tensorflow:loss = -22.041, step = 3401 (9.742 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0308658, lr = 0.000714601, temp = 1.42372 (9.741 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3532\n",
      "INFO:tensorflow:loss = -18.4937, step = 3501 (9.659 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0331827, lr = 0.000707589, temp = 1.40957 (9.659 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2631\n",
      "INFO:tensorflow:loss = -17.5048, step = 3601 (9.744 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0356676, lr = 0.000700646, temp = 1.39555 (9.744 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3505\n",
      "INFO:tensorflow:loss = -16.816, step = 3701 (9.661 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0383319, lr = 0.000693772, temp = 1.38167 (9.661 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3589\n",
      "INFO:tensorflow:loss = -19.5319, step = 3801 (9.654 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0411872, lr = 0.000686967, temp = 1.36793 (9.654 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3492\n",
      "INFO:tensorflow:loss = -21.7858, step = 3901 (9.662 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0442459, lr = 0.00068023, temp = 1.35432 (9.661 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3062\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = -19.5202, step = 4001 (9.703 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0475211, lr = 0.00067356, temp = 1.34085 (9.703 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3309\n",
      "INFO:tensorflow:loss = -19.9021, step = 4101 (9.680 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0510264, lr = 0.000666956, temp = 1.32752 (9.680 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2922\n",
      "INFO:tensorflow:loss = -19.9697, step = 4201 (9.716 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0547758, lr = 0.000660417, temp = 1.31432 (9.716 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2536\n",
      "INFO:tensorflow:loss = -18.7547, step = 4301 (9.753 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0587843, lr = 0.000653944, temp = 1.30125 (9.753 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.289\n",
      "INFO:tensorflow:loss = -19.802, step = 4401 (9.719 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0630671, lr = 0.000647535, temp = 1.28831 (9.719 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2785\n",
      "INFO:tensorflow:loss = -20.1633, step = 4501 (9.729 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0676399, lr = 0.000641191, temp = 1.27549 (9.729 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2093\n",
      "INFO:tensorflow:loss = -21.1498, step = 4601 (9.795 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0725193, lr = 0.000634909, temp = 1.26281 (9.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2865\n",
      "INFO:tensorflow:loss = -18.1793, step = 4701 (9.721 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0777217, lr = 0.000628689, temp = 1.25025 (9.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2841\n",
      "INFO:tensorflow:loss = -18.1863, step = 4801 (9.725 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0832644, lr = 0.000622532, temp = 1.23782 (9.725 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2832\n",
      "INFO:tensorflow:loss = -21.4976, step = 4901 (9.725 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0891646, lr = 0.000616436, temp = 1.22551 (9.724 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.33433\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-08:35:56\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-5001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-08:36:03\n",
      "INFO:tensorflow:Saving dict for global step 5001: NLL_exact = -26.1156, NLL_p = -25.8119, NLL_q (IS) = -25.8962, global_step = 5001, loss = -25.8962\n",
      "INFO:tensorflow:Validation (step 5001): loss = -25.8962, NLL_p = -25.8119, NLL_exact = -26.1156, NLL_q (IS) = -25.8962, global_step = 5001\n",
      "INFO:tensorflow:loss = -20.7421, step = 5001 (19.896 sec)\n",
      "INFO:tensorflow:kl_weight = 0.0954399, lr = 0.000610401, temp = 1.21332 (19.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.27569\n",
      "INFO:tensorflow:loss = -18.0252, step = 5101 (9.772 sec)\n",
      "INFO:tensorflow:kl_weight = 0.102108, lr = 0.000604425, temp = 1.20126 (9.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1928\n",
      "INFO:tensorflow:loss = -22.324, step = 5201 (9.811 sec)\n",
      "INFO:tensorflow:kl_weight = 0.109186, lr = 0.000598509, temp = 1.18931 (9.811 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1999\n",
      "INFO:tensorflow:loss = -18.8289, step = 5301 (9.804 sec)\n",
      "INFO:tensorflow:kl_weight = 0.116691, lr = 0.000592652, temp = 1.17749 (9.804 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2643\n",
      "INFO:tensorflow:loss = -17.043, step = 5401 (9.742 sec)\n",
      "INFO:tensorflow:kl_weight = 0.124641, lr = 0.000586853, temp = 1.16578 (9.742 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1839\n",
      "INFO:tensorflow:loss = -18.3286, step = 5501 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 0.133051, lr = 0.000581112, temp = 1.15419 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2127\n",
      "INFO:tensorflow:loss = -18.5519, step = 5601 (9.792 sec)\n",
      "INFO:tensorflow:kl_weight = 0.141937, lr = 0.000575429, temp = 1.14271 (9.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1062\n",
      "INFO:tensorflow:loss = -22.7393, step = 5701 (9.895 sec)\n",
      "INFO:tensorflow:kl_weight = 0.151313, lr = 0.000569801, temp = 1.13135 (9.895 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2884\n",
      "INFO:tensorflow:loss = -20.5929, step = 5801 (9.720 sec)\n",
      "INFO:tensorflow:kl_weight = 0.161193, lr = 0.00056423, temp = 1.1201 (9.719 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1381\n",
      "INFO:tensorflow:loss = -20.5792, step = 5901 (9.864 sec)\n",
      "INFO:tensorflow:kl_weight = 0.171588, lr = 0.000558714, temp = 1.10896 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2341\n",
      "INFO:tensorflow:loss = -19.3484, step = 6001 (9.771 sec)\n",
      "INFO:tensorflow:kl_weight = 0.182507, lr = 0.000553253, temp = 1.09793 (9.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2082\n",
      "INFO:tensorflow:loss = -16.4665, step = 6101 (9.796 sec)\n",
      "INFO:tensorflow:kl_weight = 0.19396, lr = 0.000547846, temp = 1.08702 (9.796 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3041\n",
      "INFO:tensorflow:loss = -17.9035, step = 6201 (9.705 sec)\n",
      "INFO:tensorflow:kl_weight = 0.20595, lr = 0.000542494, temp = 1.07621 (9.705 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1931\n",
      "INFO:tensorflow:loss = -20.3015, step = 6301 (9.810 sec)\n",
      "INFO:tensorflow:kl_weight = 0.218481, lr = 0.000537194, temp = 1.06551 (9.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2185\n",
      "INFO:tensorflow:loss = -20.9426, step = 6401 (9.787 sec)\n",
      "INFO:tensorflow:kl_weight = 0.231552, lr = 0.000531947, temp = 1.05491 (9.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2168\n",
      "INFO:tensorflow:loss = -16.4574, step = 6501 (9.787 sec)\n",
      "INFO:tensorflow:kl_weight = 0.245161, lr = 0.000526753, temp = 1.04442 (9.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1603\n",
      "INFO:tensorflow:loss = -19.8217, step = 6601 (9.842 sec)\n",
      "INFO:tensorflow:kl_weight = 0.259299, lr = 0.00052161, temp = 1.03404 (9.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1505\n",
      "INFO:tensorflow:loss = -19.5566, step = 6701 (9.851 sec)\n",
      "INFO:tensorflow:kl_weight = 0.273958, lr = 0.000516518, temp = 1.02376 (9.851 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2033\n",
      "INFO:tensorflow:loss = -19.6329, step = 6801 (9.801 sec)\n",
      "INFO:tensorflow:kl_weight = 0.289122, lr = 0.000511477, temp = 1.01358 (9.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3076\n",
      "INFO:tensorflow:loss = -18.5303, step = 6901 (9.702 sec)\n",
      "INFO:tensorflow:kl_weight = 0.304773, lr = 0.000506486, temp = 1.0035 (9.702 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2148\n",
      "INFO:tensorflow:loss = -21.9658, step = 7001 (9.790 sec)\n",
      "INFO:tensorflow:kl_weight = 0.320889, lr = 0.000501545, temp = 0.993524 (9.790 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1652\n",
      "INFO:tensorflow:loss = -18.3649, step = 7101 (9.838 sec)\n",
      "INFO:tensorflow:kl_weight = 0.337444, lr = 0.000496653, temp = 0.983646 (9.838 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2466\n",
      "INFO:tensorflow:loss = -21.3117, step = 7201 (9.759 sec)\n",
      "INFO:tensorflow:kl_weight = 0.354408, lr = 0.00049181, temp = 0.973866 (9.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1993\n",
      "INFO:tensorflow:loss = -22.4601, step = 7301 (9.805 sec)\n",
      "INFO:tensorflow:kl_weight = 0.371747, lr = 0.000487015, temp = 0.964184 (9.805 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2131\n",
      "INFO:tensorflow:loss = -16.8254, step = 7401 (9.791 sec)\n",
      "INFO:tensorflow:kl_weight = 0.389422, lr = 0.000482267, temp = 0.954598 (9.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2553\n",
      "INFO:tensorflow:loss = -20.7491, step = 7501 (9.752 sec)\n",
      "INFO:tensorflow:kl_weight = 0.407393, lr = 0.000477567, temp = 0.945108 (9.752 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2674\n",
      "INFO:tensorflow:loss = -18.6739, step = 7601 (9.739 sec)\n",
      "INFO:tensorflow:kl_weight = 0.425615, lr = 0.000472914, temp = 0.935712 (9.739 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2002\n",
      "INFO:tensorflow:loss = -20.6704, step = 7701 (9.803 sec)\n",
      "INFO:tensorflow:kl_weight = 0.444042, lr = 0.000468307, temp = 0.926409 (9.803 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.221\n",
      "INFO:tensorflow:loss = -20.088, step = 7801 (9.784 sec)\n",
      "INFO:tensorflow:kl_weight = 0.462624, lr = 0.000463745, temp = 0.917199 (9.784 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2858\n",
      "INFO:tensorflow:loss = -19.2296, step = 7901 (9.722 sec)\n",
      "INFO:tensorflow:kl_weight = 0.481311, lr = 0.00045923, temp = 0.908081 (9.722 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1052\n",
      "INFO:tensorflow:loss = -18.1127, step = 8001 (9.896 sec)\n",
      "INFO:tensorflow:kl_weight = 0.50005, lr = 0.000454759, temp = 0.899053 (9.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1854\n",
      "INFO:tensorflow:loss = -20.1852, step = 8101 (9.818 sec)\n",
      "INFO:tensorflow:kl_weight = 0.518789, lr = 0.000450332, temp = 0.890116 (9.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2393\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = -21.2545, step = 8201 (9.766 sec)\n",
      "INFO:tensorflow:kl_weight = 0.537476, lr = 0.00044595, temp = 0.881267 (9.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2756\n",
      "INFO:tensorflow:loss = -18.5694, step = 8301 (9.731 sec)\n",
      "INFO:tensorflow:kl_weight = 0.556058, lr = 0.000441611, temp = 0.872506 (9.732 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3114\n",
      "INFO:tensorflow:loss = -21.0066, step = 8401 (9.698 sec)\n",
      "INFO:tensorflow:kl_weight = 0.574485, lr = 0.000437316, temp = 0.863833 (9.697 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1548\n",
      "INFO:tensorflow:loss = -18.775, step = 8501 (9.849 sec)\n",
      "INFO:tensorflow:kl_weight = 0.592707, lr = 0.000433063, temp = 0.855246 (9.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2926\n",
      "INFO:tensorflow:loss = -18.6899, step = 8601 (9.716 sec)\n",
      "INFO:tensorflow:kl_weight = 0.610678, lr = 0.000428853, temp = 0.846744 (9.716 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2085\n",
      "INFO:tensorflow:loss = -19.9572, step = 8701 (9.795 sec)\n",
      "INFO:tensorflow:kl_weight = 0.628353, lr = 0.000424684, temp = 0.838327 (9.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1973\n",
      "INFO:tensorflow:loss = -18.1632, step = 8801 (9.807 sec)\n",
      "INFO:tensorflow:kl_weight = 0.645692, lr = 0.000420557, temp = 0.829993 (9.807 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1909\n",
      "INFO:tensorflow:loss = -21.6972, step = 8901 (9.812 sec)\n",
      "INFO:tensorflow:kl_weight = 0.662656, lr = 0.000416471, temp = 0.821743 (9.812 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2585\n",
      "INFO:tensorflow:loss = -20.1946, step = 9001 (9.749 sec)\n",
      "INFO:tensorflow:kl_weight = 0.679211, lr = 0.000412426, temp = 0.813575 (9.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1494\n",
      "INFO:tensorflow:loss = -18.2028, step = 9101 (9.853 sec)\n",
      "INFO:tensorflow:kl_weight = 0.695327, lr = 0.000408421, temp = 0.805488 (9.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2272\n",
      "INFO:tensorflow:loss = -14.9827, step = 9201 (9.777 sec)\n",
      "INFO:tensorflow:kl_weight = 0.710978, lr = 0.000404455, temp = 0.797481 (9.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2228\n",
      "INFO:tensorflow:loss = -20.2881, step = 9301 (9.782 sec)\n",
      "INFO:tensorflow:kl_weight = 0.726142, lr = 0.00040053, temp = 0.789554 (9.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1842\n",
      "INFO:tensorflow:loss = -18.7671, step = 9401 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 0.740801, lr = 0.000396643, temp = 0.781706 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1676\n",
      "INFO:tensorflow:loss = -21.0446, step = 9501 (9.835 sec)\n",
      "INFO:tensorflow:kl_weight = 0.754939, lr = 0.000392795, temp = 0.773937 (9.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1562\n",
      "INFO:tensorflow:loss = -18.7433, step = 9601 (9.847 sec)\n",
      "INFO:tensorflow:kl_weight = 0.768548, lr = 0.000388985, temp = 0.766244 (9.847 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2542\n",
      "INFO:tensorflow:loss = -17.5117, step = 9701 (9.752 sec)\n",
      "INFO:tensorflow:kl_weight = 0.781619, lr = 0.000385214, temp = 0.758628 (9.752 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2005\n",
      "INFO:tensorflow:loss = -21.0526, step = 9801 (9.803 sec)\n",
      "INFO:tensorflow:kl_weight = 0.79415, lr = 0.000381479, temp = 0.751088 (9.803 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2816\n",
      "INFO:tensorflow:loss = -22.4661, step = 9901 (9.726 sec)\n",
      "INFO:tensorflow:kl_weight = 0.80614, lr = 0.000377782, temp = 0.743623 (9.727 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.40386\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-08:44:15\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-10001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-08:44:23\n",
      "INFO:tensorflow:Saving dict for global step 10001: NLL_exact = -27.458, NLL_p = -27.1341, NLL_q (IS) = -27.3189, global_step = 10001, loss = -27.3189\n",
      "INFO:tensorflow:Validation (step 10001): loss = -27.3189, NLL_p = -27.1341, NLL_exact = -27.458, NLL_q (IS) = -27.3189, global_step = 10001\n",
      "INFO:tensorflow:loss = -19.0638, step = 10001 (19.775 sec)\n",
      "INFO:tensorflow:kl_weight = 0.817593, lr = 0.000374122, temp = 0.736232 (19.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.2758\n",
      "INFO:tensorflow:loss = -18.7873, step = 10101 (9.813 sec)\n",
      "INFO:tensorflow:kl_weight = 0.828512, lr = 0.000370498, temp = 0.728915 (9.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2577\n",
      "INFO:tensorflow:loss = -19.6852, step = 10201 (9.748 sec)\n",
      "INFO:tensorflow:kl_weight = 0.838907, lr = 0.00036691, temp = 0.72167 (9.748 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.307\n",
      "INFO:tensorflow:loss = -20.5489, step = 10301 (9.703 sec)\n",
      "INFO:tensorflow:kl_weight = 0.848787, lr = 0.000363358, temp = 0.714498 (9.703 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2902\n",
      "INFO:tensorflow:loss = -23.7467, step = 10401 (9.718 sec)\n",
      "INFO:tensorflow:kl_weight = 0.858163, lr = 0.000359841, temp = 0.707397 (9.718 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2412\n",
      "INFO:tensorflow:loss = -20.4835, step = 10501 (9.764 sec)\n",
      "INFO:tensorflow:kl_weight = 0.867049, lr = 0.00035636, temp = 0.700367 (9.764 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2607\n",
      "INFO:tensorflow:loss = -20.3044, step = 10601 (9.746 sec)\n",
      "INFO:tensorflow:kl_weight = 0.875459, lr = 0.000352913, temp = 0.693407 (9.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2189\n",
      "INFO:tensorflow:loss = -19.902, step = 10701 (9.786 sec)\n",
      "INFO:tensorflow:kl_weight = 0.883409, lr = 0.0003495, temp = 0.686516 (9.786 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2066\n",
      "INFO:tensorflow:loss = -20.053, step = 10801 (9.797 sec)\n",
      "INFO:tensorflow:kl_weight = 0.890914, lr = 0.000346121, temp = 0.679693 (9.797 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2393\n",
      "INFO:tensorflow:loss = -17.6176, step = 10901 (9.766 sec)\n",
      "INFO:tensorflow:kl_weight = 0.897992, lr = 0.000342776, temp = 0.672939 (9.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3053\n",
      "INFO:tensorflow:loss = -19.1554, step = 11001 (9.704 sec)\n",
      "INFO:tensorflow:kl_weight = 0.90466, lr = 0.000339464, temp = 0.666251 (9.704 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2756\n",
      "INFO:tensorflow:loss = -23.171, step = 11101 (9.732 sec)\n",
      "INFO:tensorflow:kl_weight = 0.910935, lr = 0.000336185, temp = 0.65963 (9.732 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2265\n",
      "INFO:tensorflow:loss = -19.0605, step = 11201 (9.778 sec)\n",
      "INFO:tensorflow:kl_weight = 0.916836, lr = 0.000332939, temp = 0.653076 (9.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.27\n",
      "INFO:tensorflow:loss = -18.9232, step = 11301 (9.737 sec)\n",
      "INFO:tensorflow:kl_weight = 0.922378, lr = 0.000329725, temp = 0.646586 (9.737 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2572\n",
      "INFO:tensorflow:loss = -19.6567, step = 11401 (9.750 sec)\n",
      "INFO:tensorflow:kl_weight = 0.927581, lr = 0.000326543, temp = 0.640161 (9.750 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2595\n",
      "INFO:tensorflow:loss = -23.7946, step = 11501 (9.747 sec)\n",
      "INFO:tensorflow:kl_weight = 0.93246, lr = 0.000323393, temp = 0.6338 (9.747 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2129\n",
      "INFO:tensorflow:loss = -17.8202, step = 11601 (9.792 sec)\n",
      "INFO:tensorflow:kl_weight = 0.937033, lr = 0.000320274, temp = 0.627502 (9.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2987\n",
      "INFO:tensorflow:loss = -17.2156, step = 11701 (9.710 sec)\n",
      "INFO:tensorflow:kl_weight = 0.941316, lr = 0.000317186, temp = 0.621267 (9.710 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2365\n",
      "INFO:tensorflow:loss = -19.9167, step = 11801 (9.769 sec)\n",
      "INFO:tensorflow:kl_weight = 0.945324, lr = 0.000314128, temp = 0.615094 (9.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2664\n",
      "INFO:tensorflow:loss = -19.2075, step = 11901 (9.741 sec)\n",
      "INFO:tensorflow:kl_weight = 0.949074, lr = 0.000311102, temp = 0.608982 (9.741 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2681\n",
      "INFO:tensorflow:loss = -20.4579, step = 12001 (9.739 sec)\n",
      "INFO:tensorflow:kl_weight = 0.952579, lr = 0.000308105, temp = 0.602931 (9.739 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2761\n",
      "INFO:tensorflow:loss = -19.3378, step = 12101 (9.731 sec)\n",
      "INFO:tensorflow:kl_weight = 0.955854, lr = 0.000305138, temp = 0.596941 (9.731 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2875\n",
      "INFO:tensorflow:loss = -19.2147, step = 12201 (9.722 sec)\n",
      "INFO:tensorflow:kl_weight = 0.958913, lr = 0.000302201, temp = 0.59101 (9.722 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2836\n",
      "INFO:tensorflow:loss = -20.722, step = 12301 (9.723 sec)\n",
      "INFO:tensorflow:kl_weight = 0.961768, lr = 0.000299293, temp = 0.585138 (9.723 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 10.2416\n",
      "INFO:tensorflow:loss = -18.911, step = 12401 (9.764 sec)\n",
      "INFO:tensorflow:kl_weight = 0.964432, lr = 0.000296414, temp = 0.579324 (9.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.138\n",
      "INFO:tensorflow:loss = -17.3913, step = 12501 (9.864 sec)\n",
      "INFO:tensorflow:kl_weight = 0.966917, lr = 0.000293563, temp = 0.573568 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2628\n",
      "INFO:tensorflow:loss = -20.752, step = 12601 (9.744 sec)\n",
      "INFO:tensorflow:kl_weight = 0.969234, lr = 0.000290741, temp = 0.56787 (9.744 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2785\n",
      "INFO:tensorflow:loss = -20.9638, step = 12701 (9.729 sec)\n",
      "INFO:tensorflow:kl_weight = 0.971394, lr = 0.000287947, temp = 0.562228 (9.729 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2877\n",
      "INFO:tensorflow:loss = -20.7234, step = 12801 (9.720 sec)\n",
      "INFO:tensorflow:kl_weight = 0.973406, lr = 0.000285181, temp = 0.556643 (9.720 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3312\n",
      "INFO:tensorflow:loss = -21.785, step = 12901 (9.679 sec)\n",
      "INFO:tensorflow:kl_weight = 0.97528, lr = 0.000282442, temp = 0.551113 (9.680 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2505\n",
      "INFO:tensorflow:loss = -19.9092, step = 13001 (9.755 sec)\n",
      "INFO:tensorflow:kl_weight = 0.977025, lr = 0.000279731, temp = 0.545638 (9.755 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2452\n",
      "INFO:tensorflow:loss = -15.7382, step = 13101 (9.761 sec)\n",
      "INFO:tensorflow:kl_weight = 0.97865, lr = 0.000277046, temp = 0.540218 (9.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1886\n",
      "INFO:tensorflow:loss = -17.9706, step = 13201 (9.815 sec)\n",
      "INFO:tensorflow:kl_weight = 0.980162, lr = 0.000274389, temp = 0.534851 (9.816 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3307\n",
      "INFO:tensorflow:loss = -19.5994, step = 13301 (9.680 sec)\n",
      "INFO:tensorflow:kl_weight = 0.981569, lr = 0.000271757, temp = 0.529538 (9.680 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2086\n",
      "INFO:tensorflow:loss = -19.111, step = 13401 (9.796 sec)\n",
      "INFO:tensorflow:kl_weight = 0.982878, lr = 0.000269152, temp = 0.524278 (9.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2861\n",
      "INFO:tensorflow:loss = -16.0016, step = 13501 (9.721 sec)\n",
      "INFO:tensorflow:kl_weight = 0.984095, lr = 0.000266573, temp = 0.51907 (9.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.29\n",
      "INFO:tensorflow:loss = -20.4539, step = 13601 (9.718 sec)\n",
      "INFO:tensorflow:kl_weight = 0.985227, lr = 0.00026402, temp = 0.513914 (9.718 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3665\n",
      "INFO:tensorflow:loss = -19.6081, step = 13701 (9.647 sec)\n",
      "INFO:tensorflow:kl_weight = 0.98628, lr = 0.000261491, temp = 0.50881 (9.647 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2859\n",
      "INFO:tensorflow:loss = -19.2595, step = 13801 (9.722 sec)\n",
      "INFO:tensorflow:kl_weight = 0.987259, lr = 0.000258989, temp = 0.503756 (9.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3046\n",
      "INFO:tensorflow:loss = -16.8538, step = 13901 (9.705 sec)\n",
      "INFO:tensorflow:kl_weight = 0.988169, lr = 0.000256511, temp = 0.498752 (9.705 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3769\n",
      "INFO:tensorflow:loss = -19.2366, step = 14001 (9.637 sec)\n",
      "INFO:tensorflow:kl_weight = 0.989014, lr = 0.000254057, temp = 0.493798 (9.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3201\n",
      "INFO:tensorflow:loss = -21.6583, step = 14101 (9.689 sec)\n",
      "INFO:tensorflow:kl_weight = 0.9898, lr = 0.000251628, temp = 0.488894 (9.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2613\n",
      "INFO:tensorflow:loss = -21.4937, step = 14201 (9.746 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99053, lr = 0.000249223, temp = 0.484038 (9.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1904\n",
      "INFO:tensorflow:loss = -21.2997, step = 14301 (9.813 sec)\n",
      "INFO:tensorflow:kl_weight = 0.991208, lr = 0.000246843, temp = 0.479231 (9.813 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2583\n",
      "INFO:tensorflow:loss = -21.1976, step = 14401 (9.748 sec)\n",
      "INFO:tensorflow:kl_weight = 0.991838, lr = 0.000244486, temp = 0.474471 (9.748 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2962\n",
      "INFO:tensorflow:loss = -18.1261, step = 14501 (9.713 sec)\n",
      "INFO:tensorflow:kl_weight = 0.992424, lr = 0.000242152, temp = 0.469759 (9.713 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2224\n",
      "INFO:tensorflow:loss = -18.4025, step = 14601 (9.783 sec)\n",
      "INFO:tensorflow:kl_weight = 0.992967, lr = 0.000239841, temp = 0.465094 (9.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3271\n",
      "INFO:tensorflow:loss = -22.3351, step = 14701 (9.683 sec)\n",
      "INFO:tensorflow:kl_weight = 0.993472, lr = 0.000237554, temp = 0.460475 (9.683 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2806\n",
      "INFO:tensorflow:loss = -17.3028, step = 14801 (9.727 sec)\n",
      "INFO:tensorflow:kl_weight = 0.993941, lr = 0.000235289, temp = 0.455902 (9.727 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3334\n",
      "INFO:tensorflow:loss = -19.0836, step = 14901 (9.677 sec)\n",
      "INFO:tensorflow:kl_weight = 0.994376, lr = 0.000233047, temp = 0.451375 (9.677 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 15001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.53235\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-08:52:32\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-15001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-08:52:40\n",
      "INFO:tensorflow:Saving dict for global step 15001: NLL_exact = -27.7588, NLL_p = -27.5245, NLL_q (IS) = -27.6431, global_step = 15001, loss = -27.6431\n",
      "INFO:tensorflow:Validation (step 15001): loss = -27.6431, NLL_p = -27.5245, NLL_exact = -27.7588, NLL_q (IS) = -27.6431, global_step = 15001\n",
      "INFO:tensorflow:loss = -18.9945, step = 15001 (19.687 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99478, lr = 0.000230827, temp = 0.446893 (19.687 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.2666\n",
      "INFO:tensorflow:loss = -19.5673, step = 15101 (9.791 sec)\n",
      "INFO:tensorflow:kl_weight = 0.995156, lr = 0.00022863, temp = 0.442455 (9.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1613\n",
      "INFO:tensorflow:loss = -18.9096, step = 15201 (9.842 sec)\n",
      "INFO:tensorflow:kl_weight = 0.995504, lr = 0.000226454, temp = 0.438062 (9.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1196\n",
      "INFO:tensorflow:loss = -18.5929, step = 15301 (9.882 sec)\n",
      "INFO:tensorflow:kl_weight = 0.995828, lr = 0.000224299, temp = 0.433712 (9.882 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1587\n",
      "INFO:tensorflow:loss = -20.781, step = 15401 (9.844 sec)\n",
      "INFO:tensorflow:kl_weight = 0.996128, lr = 0.000222167, temp = 0.429405 (9.844 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1813\n",
      "INFO:tensorflow:loss = -19.9285, step = 15501 (9.822 sec)\n",
      "INFO:tensorflow:kl_weight = 0.996407, lr = 0.000220055, temp = 0.425142 (9.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1522\n",
      "INFO:tensorflow:loss = -21.6627, step = 15601 (9.850 sec)\n",
      "INFO:tensorflow:kl_weight = 0.996666, lr = 0.000217965, temp = 0.42092 (9.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2167\n",
      "INFO:tensorflow:loss = -19.4604, step = 15701 (9.787 sec)\n",
      "INFO:tensorflow:kl_weight = 0.996906, lr = 0.000215895, temp = 0.416741 (9.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2009\n",
      "INFO:tensorflow:loss = -19.5062, step = 15801 (9.804 sec)\n",
      "INFO:tensorflow:kl_weight = 0.997129, lr = 0.000213846, temp = 0.412604 (9.804 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1457\n",
      "INFO:tensorflow:loss = -16.9053, step = 15901 (9.856 sec)\n",
      "INFO:tensorflow:kl_weight = 0.997335, lr = 0.000211817, temp = 0.408507 (9.856 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3204\n",
      "INFO:tensorflow:loss = -19.2757, step = 16001 (9.689 sec)\n",
      "INFO:tensorflow:kl_weight = 0.997528, lr = 0.000209808, temp = 0.404452 (9.689 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1596\n",
      "INFO:tensorflow:loss = -21.481, step = 16101 (9.843 sec)\n",
      "INFO:tensorflow:kl_weight = 0.997706, lr = 0.00020782, temp = 0.400436 (9.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3461\n",
      "INFO:tensorflow:loss = -20.7638, step = 16201 (9.666 sec)\n",
      "INFO:tensorflow:kl_weight = 0.997871, lr = 0.000205851, temp = 0.396461 (9.666 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2976\n",
      "INFO:tensorflow:loss = -20.9135, step = 16301 (9.711 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998025, lr = 0.000203902, temp = 0.392525 (9.711 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.309\n",
      "INFO:tensorflow:loss = -20.1743, step = 16401 (9.700 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998167, lr = 0.000201972, temp = 0.388629 (9.701 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.269\n",
      "INFO:tensorflow:loss = -18.9154, step = 16501 (9.738 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:kl_weight = 0.998299, lr = 0.000200062, temp = 0.384771 (9.738 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.267\n",
      "INFO:tensorflow:loss = -19.7244, step = 16601 (9.740 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998422, lr = 0.00019817, temp = 0.380952 (9.740 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3698\n",
      "INFO:tensorflow:loss = -22.3064, step = 16701 (9.644 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998536, lr = 0.000196297, temp = 0.37717 (9.644 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2423\n",
      "INFO:tensorflow:loss = -21.5866, step = 16801 (9.763 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998642, lr = 0.000194443, temp = 0.373426 (9.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2929\n",
      "INFO:tensorflow:loss = -17.6243, step = 16901 (9.715 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99874, lr = 0.000192608, temp = 0.36972 (9.714 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3\n",
      "INFO:tensorflow:loss = -19.9128, step = 17001 (9.709 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998831, lr = 0.00019079, temp = 0.36605 (9.709 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.175\n",
      "INFO:tensorflow:loss = -18.8185, step = 17101 (9.829 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998915, lr = 0.000188991, temp = 0.362417 (9.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2541\n",
      "INFO:tensorflow:loss = -23.3406, step = 17201 (9.752 sec)\n",
      "INFO:tensorflow:kl_weight = 0.998993, lr = 0.00018721, temp = 0.35882 (9.752 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1757\n",
      "INFO:tensorflow:loss = -16.9557, step = 17301 (9.827 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999066, lr = 0.000185446, temp = 0.355259 (9.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2373\n",
      "INFO:tensorflow:loss = -17.0454, step = 17401 (9.769 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999133, lr = 0.0001837, temp = 0.351733 (9.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2577\n",
      "INFO:tensorflow:loss = -18.5709, step = 17501 (9.749 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999196, lr = 0.000181971, temp = 0.348243 (9.748 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2322\n",
      "INFO:tensorflow:loss = -20.775, step = 17601 (9.773 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999254, lr = 0.00018026, temp = 0.344787 (9.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2176\n",
      "INFO:tensorflow:loss = -17.9363, step = 17701 (9.787 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999308, lr = 0.000178565, temp = 0.341365 (9.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1743\n",
      "INFO:tensorflow:loss = -20.8638, step = 17801 (9.830 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999358, lr = 0.000176888, temp = 0.337978 (9.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2241\n",
      "INFO:tensorflow:loss = -22.4932, step = 17901 (9.779 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999404, lr = 0.000175227, temp = 0.334624 (9.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1824\n",
      "INFO:tensorflow:loss = -18.2527, step = 18001 (9.821 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999447, lr = 0.000173582, temp = 0.331304 (9.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3781\n",
      "INFO:tensorflow:loss = -20.5271, step = 18101 (9.636 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999487, lr = 0.000171954, temp = 0.328017 (9.636 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1652\n",
      "INFO:tensorflow:loss = -20.9328, step = 18201 (9.837 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999524, lr = 0.000170342, temp = 0.324762 (9.837 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1358\n",
      "INFO:tensorflow:loss = -21.3219, step = 18301 (9.866 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999559, lr = 0.000168747, temp = 0.32154 (9.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2625\n",
      "INFO:tensorflow:loss = -20.8122, step = 18401 (9.744 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999591, lr = 0.000167167, temp = 0.31835 (9.744 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2301\n",
      "INFO:tensorflow:loss = -20.6274, step = 18501 (9.775 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99962, lr = 0.000165603, temp = 0.315192 (9.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1352\n",
      "INFO:tensorflow:loss = -21.0583, step = 18601 (9.867 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999648, lr = 0.000164054, temp = 0.312065 (9.867 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2101\n",
      "INFO:tensorflow:loss = -22.3255, step = 18701 (9.794 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999673, lr = 0.000162521, temp = 0.308969 (9.793 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2147\n",
      "INFO:tensorflow:loss = -22.8652, step = 18801 (9.789 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999697, lr = 0.000161003, temp = 0.305904 (9.789 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2958\n",
      "INFO:tensorflow:loss = -20.6011, step = 18901 (9.713 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999718, lr = 0.0001595, temp = 0.302869 (9.714 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2479\n",
      "INFO:tensorflow:loss = -22.9286, step = 19001 (9.758 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999739, lr = 0.000158012, temp = 0.299865 (9.758 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0502\n",
      "INFO:tensorflow:loss = -20.4102, step = 19101 (9.950 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999758, lr = 0.000156539, temp = 0.296891 (9.950 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.186\n",
      "INFO:tensorflow:loss = -21.6899, step = 19201 (9.818 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999775, lr = 0.000155081, temp = 0.293946 (9.817 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.236\n",
      "INFO:tensorflow:loss = -20.4453, step = 19301 (9.769 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999791, lr = 0.000153637, temp = 0.29103 (9.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1662\n",
      "INFO:tensorflow:loss = -18.6808, step = 19401 (9.837 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999806, lr = 0.000152207, temp = 0.288144 (9.837 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2091\n",
      "INFO:tensorflow:loss = -19.9721, step = 19501 (9.795 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999821, lr = 0.000150792, temp = 0.285286 (9.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2601\n",
      "INFO:tensorflow:loss = -19.6925, step = 19601 (9.747 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999833, lr = 0.000149391, temp = 0.282457 (9.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2402\n",
      "INFO:tensorflow:loss = -20.8982, step = 19701 (9.765 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999846, lr = 0.000148004, temp = 0.279656 (9.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.273\n",
      "INFO:tensorflow:loss = -18.1943, step = 19801 (9.735 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999857, lr = 0.00014663, temp = 0.276882 (9.735 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2288\n",
      "INFO:tensorflow:loss = -21.1409, step = 19901 (9.776 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999867, lr = 0.00014527, temp = 0.274137 (9.776 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.41583\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:00:51\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-20001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:00:59\n",
      "INFO:tensorflow:Saving dict for global step 20001: NLL_exact = -28.0315, NLL_p = -27.8084, NLL_q (IS) = -27.918, global_step = 20001, loss = -27.918\n",
      "INFO:tensorflow:Validation (step 20001): loss = -27.918, NLL_p = -27.8084, NLL_exact = -28.0315, NLL_q (IS) = -27.918, global_step = 20001\n",
      "INFO:tensorflow:loss = -21.0162, step = 20001 (20.145 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999877, lr = 0.000143924, temp = 0.271418 (20.145 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.19224\n",
      "INFO:tensorflow:loss = -19.5649, step = 20101 (9.737 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999886, lr = 0.000142591, temp = 0.268727 (9.737 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2059\n",
      "INFO:tensorflow:loss = -19.8891, step = 20201 (9.797 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999894, lr = 0.000141272, temp = 0.266063 (9.797 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2844\n",
      "INFO:tensorflow:loss = -21.8248, step = 20301 (9.724 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999901, lr = 0.000139965, temp = 0.263425 (9.724 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1658\n",
      "INFO:tensorflow:loss = -20.6024, step = 20401 (9.837 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999909, lr = 0.000138672, temp = 0.260813 (9.837 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2475\n",
      "INFO:tensorflow:loss = -21.9246, step = 20501 (9.759 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999915, lr = 0.000137391, temp = 0.258227 (9.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2095\n",
      "INFO:tensorflow:loss = -21.5459, step = 20601 (9.795 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999921, lr = 0.000136123, temp = 0.255667 (9.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1832\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = -19.9078, step = 20701 (9.820 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999927, lr = 0.000134868, temp = 0.253133 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0536\n",
      "INFO:tensorflow:loss = -19.8755, step = 20801 (9.947 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999932, lr = 0.000133625, temp = 0.250623 (9.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1793\n",
      "INFO:tensorflow:loss = -21.1674, step = 20901 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999937, lr = 0.000132395, temp = 0.248139 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2898\n",
      "INFO:tensorflow:loss = -19.7078, step = 21001 (9.719 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999942, lr = 0.000131177, temp = 0.245679 (9.719 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0644\n",
      "INFO:tensorflow:loss = -21.7728, step = 21101 (9.936 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999946, lr = 0.000129971, temp = 0.243244 (9.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1939\n",
      "INFO:tensorflow:loss = -19.8544, step = 21201 (9.810 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99995, lr = 0.000128777, temp = 0.240833 (9.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1843\n",
      "INFO:tensorflow:loss = -19.4259, step = 21301 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999954, lr = 0.000127595, temp = 0.238446 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0791\n",
      "INFO:tensorflow:loss = -20.879, step = 21401 (9.921 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999957, lr = 0.000126424, temp = 0.236083 (9.920 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2515\n",
      "INFO:tensorflow:loss = -17.3177, step = 21501 (9.756 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99996, lr = 0.000125266, temp = 0.233744 (9.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1756\n",
      "INFO:tensorflow:loss = -19.1436, step = 21601 (9.827 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999963, lr = 0.000124119, temp = 0.231427 (9.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1334\n",
      "INFO:tensorflow:loss = -19.0611, step = 21701 (9.868 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999966, lr = 0.000122983, temp = 0.229134 (9.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2603\n",
      "INFO:tensorflow:loss = -19.3966, step = 21801 (9.745 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999968, lr = 0.000121858, temp = 0.226864 (9.745 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.159\n",
      "INFO:tensorflow:loss = -17.2225, step = 21901 (9.844 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99997, lr = 0.000120745, temp = 0.224616 (9.844 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2476\n",
      "INFO:tensorflow:loss = -19.6547, step = 22001 (9.759 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999972, lr = 0.000119643, temp = 0.22239 (9.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0849\n",
      "INFO:tensorflow:loss = -17.3977, step = 22101 (9.917 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999974, lr = 0.000118552, temp = 0.220187 (9.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.07\n",
      "INFO:tensorflow:loss = -19.5335, step = 22201 (9.930 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999976, lr = 0.000117471, temp = 0.218006 (9.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1013\n",
      "INFO:tensorflow:loss = -21.8705, step = 22301 (9.900 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999978, lr = 0.000116402, temp = 0.215846 (9.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2374\n",
      "INFO:tensorflow:loss = -20.4821, step = 22401 (9.768 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99998, lr = 0.000115343, temp = 0.213708 (9.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1904\n",
      "INFO:tensorflow:loss = -20.2785, step = 22501 (9.813 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999981, lr = 0.000114295, temp = 0.211591 (9.813 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2451\n",
      "INFO:tensorflow:loss = -22.1621, step = 22601 (9.761 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999982, lr = 0.000113257, temp = 0.209495 (9.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1829\n",
      "INFO:tensorflow:loss = -23.8092, step = 22701 (9.821 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999984, lr = 0.000112229, temp = 0.20742 (9.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0882\n",
      "INFO:tensorflow:loss = -21.727, step = 22801 (9.912 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999985, lr = 0.000111212, temp = 0.205365 (9.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.302\n",
      "INFO:tensorflow:loss = -19.9301, step = 22901 (9.707 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999986, lr = 0.000110204, temp = 0.203332 (9.707 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1851\n",
      "INFO:tensorflow:loss = -22.879, step = 23001 (9.818 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999987, lr = 0.000109207, temp = 0.201318 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1764\n",
      "INFO:tensorflow:loss = -23.7143, step = 23101 (9.826 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999988, lr = 0.00010822, temp = 0.199324 (9.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2065\n",
      "INFO:tensorflow:loss = -21.9139, step = 23201 (9.798 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999989, lr = 0.000107242, temp = 0.197351 (9.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1832\n",
      "INFO:tensorflow:loss = -19.6845, step = 23301 (9.820 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99999, lr = 0.000106274, temp = 0.195396 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1975\n",
      "INFO:tensorflow:loss = -19.7125, step = 23401 (9.807 sec)\n",
      "INFO:tensorflow:kl_weight = 0.99999, lr = 0.000105316, temp = 0.193462 (9.808 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0977\n",
      "INFO:tensorflow:loss = -19.0918, step = 23501 (9.902 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999991, lr = 0.000104368, temp = 0.191546 (9.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.198\n",
      "INFO:tensorflow:loss = -20.0133, step = 23601 (9.806 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999992, lr = 0.000103428, temp = 0.18965 (9.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1845\n",
      "INFO:tensorflow:loss = -22.8058, step = 23701 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999992, lr = 0.000102499, temp = 0.187772 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1713\n",
      "INFO:tensorflow:loss = -20.3283, step = 23801 (9.831 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999993, lr = 0.000101578, temp = 0.185914 (9.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1675\n",
      "INFO:tensorflow:loss = -20.3098, step = 23901 (9.835 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999993, lr = 0.000100667, temp = 0.184073 (9.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2039\n",
      "INFO:tensorflow:loss = -19.6984, step = 24001 (9.801 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999994, lr = 9.97642e-05, temp = 0.182251 (9.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1671\n",
      "INFO:tensorflow:loss = -21.7816, step = 24101 (9.835 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999994, lr = 9.88709e-05, temp = 0.180447 (9.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2429\n",
      "INFO:tensorflow:loss = -22.4353, step = 24201 (9.763 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999995, lr = 9.79864e-05, temp = 0.178661 (9.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2551\n",
      "INFO:tensorflow:loss = -21.0593, step = 24301 (9.751 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999995, lr = 9.71107e-05, temp = 0.176893 (9.751 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.225\n",
      "INFO:tensorflow:loss = -21.0215, step = 24401 (9.780 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999995, lr = 9.62438e-05, temp = 0.175143 (9.780 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2868\n",
      "INFO:tensorflow:loss = -20.8371, step = 24501 (9.722 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999996, lr = 9.53854e-05, temp = 0.17341 (9.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2459\n",
      "INFO:tensorflow:loss = -18.2655, step = 24601 (9.760 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999996, lr = 9.45357e-05, temp = 0.171694 (9.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.196\n",
      "INFO:tensorflow:loss = -20.6945, step = 24701 (9.808 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999996, lr = 9.36944e-05, temp = 0.169995 (9.809 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2555\n",
      "INFO:tensorflow:loss = -22.3519, step = 24801 (9.750 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999997, lr = 9.28614e-05, temp = 0.168313 (9.750 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1855\n",
      "INFO:tensorflow:loss = -21.3999, step = 24901 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999997, lr = 9.20367e-05, temp = 0.166648 (9.819 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 25001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.46062\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:09:12\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-25001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:09:20\n",
      "INFO:tensorflow:Saving dict for global step 25001: NLL_exact = -27.9743, NLL_p = -27.7684, NLL_q (IS) = -27.876, global_step = 25001, loss = -27.876\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Validation (step 25001): loss = -27.876, NLL_p = -27.7684, NLL_exact = -27.9743, NLL_q (IS) = -27.876, global_step = 25001\n",
      "INFO:tensorflow:loss = -17.3927, step = 25001 (19.743 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999997, lr = 9.12203e-05, temp = 0.164999 (19.742 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.2187\n",
      "INFO:tensorflow:loss = -20.4163, step = 25101 (9.989 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999997, lr = 9.0412e-05, temp = 0.163367 (9.989 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1455\n",
      "INFO:tensorflow:loss = -23.9093, step = 25201 (9.857 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999997, lr = 8.96117e-05, temp = 0.161751 (9.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2066\n",
      "INFO:tensorflow:loss = -20.1452, step = 25301 (9.797 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999998, lr = 8.88193e-05, temp = 0.160151 (9.797 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2175\n",
      "INFO:tensorflow:loss = -19.5299, step = 25401 (9.788 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999998, lr = 8.80349e-05, temp = 0.158567 (9.788 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1279\n",
      "INFO:tensorflow:loss = -22.1972, step = 25501 (9.873 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999998, lr = 8.72583e-05, temp = 0.156999 (9.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.231\n",
      "INFO:tensorflow:loss = -19.1978, step = 25601 (9.775 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999998, lr = 8.64894e-05, temp = 0.155447 (9.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1359\n",
      "INFO:tensorflow:loss = -18.6805, step = 25701 (9.865 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999998, lr = 8.57282e-05, temp = 0.15391 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1759\n",
      "INFO:tensorflow:loss = -23.2293, step = 25801 (9.828 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999998, lr = 8.49745e-05, temp = 0.152388 (9.828 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.268\n",
      "INFO:tensorflow:loss = -20.3251, step = 25901 (9.739 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 8.42283e-05, temp = 0.150881 (9.739 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1517\n",
      "INFO:tensorflow:loss = -18.7417, step = 26001 (9.850 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 8.34896e-05, temp = 0.14939 (9.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2463\n",
      "INFO:tensorflow:loss = -19.5753, step = 26101 (9.760 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 8.27582e-05, temp = 0.147913 (9.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2063\n",
      "INFO:tensorflow:loss = -19.2995, step = 26201 (9.798 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 8.20341e-05, temp = 0.146451 (9.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1866\n",
      "INFO:tensorflow:loss = -21.748, step = 26301 (9.817 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 8.13172e-05, temp = 0.145003 (9.817 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.236\n",
      "INFO:tensorflow:loss = -21.0699, step = 26401 (9.770 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 8.06074e-05, temp = 0.14357 (9.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1795\n",
      "INFO:tensorflow:loss = -20.1752, step = 26501 (9.823 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.99047e-05, temp = 0.142151 (9.823 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1206\n",
      "INFO:tensorflow:loss = -20.5157, step = 26601 (9.881 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.9209e-05, temp = 0.140746 (9.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1754\n",
      "INFO:tensorflow:loss = -21.747, step = 26701 (9.828 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.85202e-05, temp = 0.139355 (9.828 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1699\n",
      "INFO:tensorflow:loss = -21.0976, step = 26801 (9.832 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.78382e-05, temp = 0.137978 (9.832 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1957\n",
      "INFO:tensorflow:loss = -21.2407, step = 26901 (9.809 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.71631e-05, temp = 0.136615 (9.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1296\n",
      "INFO:tensorflow:loss = -24.9364, step = 27001 (9.872 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.64947e-05, temp = 0.135266 (9.872 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1989\n",
      "INFO:tensorflow:loss = -19.7007, step = 27101 (9.805 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.58329e-05, temp = 0.133929 (9.805 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2029\n",
      "INFO:tensorflow:loss = -21.3144, step = 27201 (9.801 sec)\n",
      "INFO:tensorflow:kl_weight = 0.999999, lr = 7.51777e-05, temp = 0.132606 (9.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2034\n",
      "INFO:tensorflow:loss = -22.7207, step = 27301 (9.801 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.4529e-05, temp = 0.131297 (9.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0792\n",
      "INFO:tensorflow:loss = -21.521, step = 27401 (9.921 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.38868e-05, temp = 0.13 (9.921 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2415\n",
      "INFO:tensorflow:loss = -24.7034, step = 27501 (9.764 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.3251e-05, temp = 0.128716 (9.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.18\n",
      "INFO:tensorflow:loss = -18.7565, step = 27601 (9.823 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.26215e-05, temp = 0.127445 (9.823 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2746\n",
      "INFO:tensorflow:loss = -19.792, step = 27701 (9.733 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.19983e-05, temp = 0.126186 (9.732 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1243\n",
      "INFO:tensorflow:loss = -22.4999, step = 27801 (9.877 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.13813e-05, temp = 0.124941 (9.877 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2457\n",
      "INFO:tensorflow:loss = -18.83, step = 27901 (9.761 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.07704e-05, temp = 0.123707 (9.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1892\n",
      "INFO:tensorflow:loss = -23.2634, step = 28001 (9.814 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 7.01656e-05, temp = 0.122486 (9.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0801\n",
      "INFO:tensorflow:loss = -19.5263, step = 28101 (9.921 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.95668e-05, temp = 0.121277 (9.920 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.134\n",
      "INFO:tensorflow:loss = -20.7035, step = 28201 (9.869 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.8974e-05, temp = 0.12008 (9.869 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2031\n",
      "INFO:tensorflow:loss = -21.0277, step = 28301 (9.799 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.8387e-05, temp = 0.118895 (9.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2084\n",
      "INFO:tensorflow:loss = -21.2913, step = 28401 (9.798 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.78059e-05, temp = 0.117721 (9.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1776\n",
      "INFO:tensorflow:loss = -18.2544, step = 28501 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.72306e-05, temp = 0.11656 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0894\n",
      "INFO:tensorflow:loss = -22.6882, step = 28601 (9.911 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.66611e-05, temp = 0.11541 (9.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1843\n",
      "INFO:tensorflow:loss = -18.6172, step = 28701 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.60972e-05, temp = 0.114271 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1942\n",
      "INFO:tensorflow:loss = -21.3083, step = 28801 (9.810 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.55389e-05, temp = 0.113144 (9.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2022\n",
      "INFO:tensorflow:loss = -19.7953, step = 28901 (9.802 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.49861e-05, temp = 0.112028 (9.802 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1001\n",
      "INFO:tensorflow:loss = -19.9416, step = 29001 (9.901 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.44389e-05, temp = 0.110923 (9.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2168\n",
      "INFO:tensorflow:loss = -19.4516, step = 29101 (9.788 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.38971e-05, temp = 0.109829 (9.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.223\n",
      "INFO:tensorflow:loss = -18.263, step = 29201 (9.781 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.33607e-05, temp = 0.108745 (9.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2756\n",
      "INFO:tensorflow:loss = -19.7026, step = 29301 (9.733 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.28296e-05, temp = 0.107673 (9.733 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1507\n",
      "INFO:tensorflow:loss = -22.475, step = 29401 (9.852 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.23039e-05, temp = 0.106612 (9.852 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 10.3083\n",
      "INFO:tensorflow:loss = -19.6289, step = 29501 (9.700 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.17833e-05, temp = 0.10556 (9.700 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2657\n",
      "INFO:tensorflow:loss = -21.0692, step = 29601 (9.742 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.12679e-05, temp = 0.10452 (9.742 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2058\n",
      "INFO:tensorflow:loss = -20.3758, step = 29701 (9.798 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.07577e-05, temp = 0.10349 (9.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1906\n",
      "INFO:tensorflow:loss = -19.7979, step = 29801 (9.813 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 6.02526e-05, temp = 0.10247 (9.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1748\n",
      "INFO:tensorflow:loss = -19.4741, step = 29901 (9.828 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.97524e-05, temp = 0.10146 (9.828 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 30001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.45278\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:17:33\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-30001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:17:40\n",
      "INFO:tensorflow:Saving dict for global step 30001: NLL_exact = -27.9427, NLL_p = -27.76, NLL_q (IS) = -27.8333, global_step = 30001, loss = -27.8333\n",
      "INFO:tensorflow:Validation (step 30001): loss = -27.8333, NLL_p = -27.76, NLL_exact = -27.9427, NLL_q (IS) = -27.8333, global_step = 30001\n",
      "INFO:tensorflow:loss = -22.6481, step = 30001 (19.709 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.92573e-05, temp = 0.10046 (19.708 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.27687\n",
      "INFO:tensorflow:loss = -22.6128, step = 30101 (9.821 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.8767e-05, temp = 0.09947 (9.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.082\n",
      "INFO:tensorflow:loss = -20.2535, step = 30201 (9.919 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.82817e-05, temp = 0.09849 (9.919 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1792\n",
      "INFO:tensorflow:loss = -22.6179, step = 30301 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.78012e-05, temp = 0.0975198 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2412\n",
      "INFO:tensorflow:loss = -20.0728, step = 30401 (9.765 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.73255e-05, temp = 0.0965592 (9.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2114\n",
      "INFO:tensorflow:loss = -24.6605, step = 30501 (9.793 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.68545e-05, temp = 0.0956081 (9.793 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1275\n",
      "INFO:tensorflow:loss = -22.7081, step = 30601 (9.874 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.63882e-05, temp = 0.0946666 (9.874 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1274\n",
      "INFO:tensorflow:loss = -19.3452, step = 30701 (9.875 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.59265e-05, temp = 0.0937344 (9.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1097\n",
      "INFO:tensorflow:loss = -16.5255, step = 30801 (9.890 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.54694e-05, temp = 0.0928115 (9.890 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2376\n",
      "INFO:tensorflow:loss = -19.7446, step = 30901 (9.769 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.50169e-05, temp = 0.0918977 (9.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2226\n",
      "INFO:tensorflow:loss = -25.2035, step = 31001 (9.782 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.45689e-05, temp = 0.0909931 (9.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2439\n",
      "INFO:tensorflow:loss = -23.006, step = 31101 (9.762 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.41253e-05, temp = 0.0900975 (9.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1253\n",
      "INFO:tensorflow:loss = -18.3925, step = 31201 (9.876 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.36861e-05, temp = 0.0892107 (9.876 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1755\n",
      "INFO:tensorflow:loss = -21.2874, step = 31301 (9.827 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.32514e-05, temp = 0.0883328 (9.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1659\n",
      "INFO:tensorflow:loss = -20.3589, step = 31401 (9.837 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.28209e-05, temp = 0.0874637 (9.837 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2422\n",
      "INFO:tensorflow:loss = -21.0343, step = 31501 (9.764 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.23948e-05, temp = 0.0866031 (9.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2232\n",
      "INFO:tensorflow:loss = -22.7827, step = 31601 (9.781 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.19728e-05, temp = 0.0857512 (9.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1802\n",
      "INFO:tensorflow:loss = -21.4493, step = 31701 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.15551e-05, temp = 0.0849077 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1693\n",
      "INFO:tensorflow:loss = -15.3416, step = 31801 (9.834 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.11415e-05, temp = 0.0840727 (9.834 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1476\n",
      "INFO:tensorflow:loss = -21.608, step = 31901 (9.854 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.07321e-05, temp = 0.0832459 (9.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2351\n",
      "INFO:tensorflow:loss = -19.9802, step = 32001 (9.770 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 5.03267e-05, temp = 0.0824274 (9.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1511\n",
      "INFO:tensorflow:loss = -21.839, step = 32101 (9.851 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.99254e-05, temp = 0.081617 (9.851 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1862\n",
      "INFO:tensorflow:loss = -17.8514, step = 32201 (9.817 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.9528e-05, temp = 0.0808146 (9.817 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2579\n",
      "INFO:tensorflow:loss = -18.2493, step = 32301 (9.749 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.91346e-05, temp = 0.0800203 (9.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2661\n",
      "INFO:tensorflow:loss = -20.3804, step = 32401 (9.741 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.87451e-05, temp = 0.0792339 (9.741 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2356\n",
      "INFO:tensorflow:loss = -24.0752, step = 32501 (9.770 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.83595e-05, temp = 0.0784553 (9.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1504\n",
      "INFO:tensorflow:loss = -21.9258, step = 32601 (9.852 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.79778e-05, temp = 0.0776844 (9.852 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2325\n",
      "INFO:tensorflow:loss = -19.5223, step = 32701 (9.773 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.75998e-05, temp = 0.0769212 (9.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1472\n",
      "INFO:tensorflow:loss = -20.5448, step = 32801 (9.854 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.72256e-05, temp = 0.0761656 (9.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1344\n",
      "INFO:tensorflow:loss = -18.1504, step = 32901 (9.868 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.68551e-05, temp = 0.0754175 (9.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1525\n",
      "INFO:tensorflow:loss = -19.5464, step = 33001 (9.850 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.64883e-05, temp = 0.0746769 (9.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2114\n",
      "INFO:tensorflow:loss = -24.1448, step = 33101 (9.793 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.61252e-05, temp = 0.0739437 (9.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1092\n",
      "INFO:tensorflow:loss = -22.1866, step = 33201 (9.892 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.57656e-05, temp = 0.0732177 (9.892 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2301\n",
      "INFO:tensorflow:loss = -20.6155, step = 33301 (9.775 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.54097e-05, temp = 0.072499 (9.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 9.97507\n",
      "INFO:tensorflow:loss = -20.2263, step = 33401 (10.025 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.50573e-05, temp = 0.0717874 (10.025 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2205\n",
      "INFO:tensorflow:loss = -23.5459, step = 33501 (9.784 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.47084e-05, temp = 0.0710829 (9.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2522\n",
      "INFO:tensorflow:loss = -19.206, step = 33601 (9.755 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.4363e-05, temp = 0.0703854 (9.755 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2167\n",
      "INFO:tensorflow:loss = -21.3354, step = 33701 (9.788 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.4021e-05, temp = 0.0696949 (9.788 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1592\n",
      "INFO:tensorflow:loss = -21.1574, step = 33801 (9.843 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.36824e-05, temp = 0.0690112 (9.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2464\n",
      "INFO:tensorflow:loss = -18.7407, step = 33901 (9.759 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.33472e-05, temp = 0.0683343 (9.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3083\n",
      "INFO:tensorflow:loss = -23.9377, step = 34001 (9.701 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.30153e-05, temp = 0.0676642 (9.702 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1891\n",
      "INFO:tensorflow:loss = -20.9409, step = 34101 (9.814 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.26867e-05, temp = 0.0670007 (9.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2666\n",
      "INFO:tensorflow:loss = -19.5643, step = 34201 (9.741 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.23614e-05, temp = 0.0663439 (9.741 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1638\n",
      "INFO:tensorflow:loss = -21.8335, step = 34301 (9.839 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.20393e-05, temp = 0.0656936 (9.839 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2228\n",
      "INFO:tensorflow:loss = -18.108, step = 34401 (9.782 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.17205e-05, temp = 0.0650497 (9.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2642\n",
      "INFO:tensorflow:loss = -22.6071, step = 34501 (9.742 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.14048e-05, temp = 0.0644123 (9.742 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2498\n",
      "INFO:tensorflow:loss = -19.8033, step = 34601 (9.756 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.10922e-05, temp = 0.0637812 (9.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2391\n",
      "INFO:tensorflow:loss = -18.5714, step = 34701 (9.767 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.07828e-05, temp = 0.0631563 (9.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.145\n",
      "INFO:tensorflow:loss = -19.8243, step = 34801 (9.857 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.04764e-05, temp = 0.0625378 (9.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1448\n",
      "INFO:tensorflow:loss = -21.4226, step = 34901 (9.857 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 4.01731e-05, temp = 0.0619253 (9.858 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 35001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.49161\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:25:54\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-35001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:26:01\n",
      "INFO:tensorflow:Saving dict for global step 35001: NLL_exact = -27.9609, NLL_p = -27.7511, NLL_q (IS) = -27.851, global_step = 35001, loss = -27.851\n",
      "INFO:tensorflow:Validation (step 35001): loss = -27.851, NLL_p = -27.7511, NLL_exact = -27.9609, NLL_q (IS) = -27.851, global_step = 35001\n",
      "INFO:tensorflow:loss = -20.4601, step = 35001 (19.714 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.98728e-05, temp = 0.061319 (19.714 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.2561\n",
      "INFO:tensorflow:loss = -20.5244, step = 35101 (9.848 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.95755e-05, temp = 0.0607186 (9.848 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2205\n",
      "INFO:tensorflow:loss = -22.6905, step = 35201 (9.785 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.92812e-05, temp = 0.0601243 (9.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1756\n",
      "INFO:tensorflow:loss = -20.8636, step = 35301 (9.827 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.89898e-05, temp = 0.0595359 (9.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1252\n",
      "INFO:tensorflow:loss = -19.2166, step = 35401 (9.876 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.87012e-05, temp = 0.0589533 (9.876 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1934\n",
      "INFO:tensorflow:loss = -21.1942, step = 35501 (9.810 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.84156e-05, temp = 0.0583765 (9.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1731\n",
      "INFO:tensorflow:loss = -21.6691, step = 35601 (9.831 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.81328e-05, temp = 0.0578055 (9.831 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.195\n",
      "INFO:tensorflow:loss = -21.313, step = 35701 (9.809 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.78528e-05, temp = 0.0572402 (9.809 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2449\n",
      "INFO:tensorflow:loss = -18.707, step = 35801 (9.761 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.75756e-05, temp = 0.0566805 (9.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2208\n",
      "INFO:tensorflow:loss = -18.4214, step = 35901 (9.784 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.73012e-05, temp = 0.0561263 (9.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2218\n",
      "INFO:tensorflow:loss = -20.0688, step = 36001 (9.782 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.70295e-05, temp = 0.0555777 (9.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.173\n",
      "INFO:tensorflow:loss = -21.4551, step = 36101 (9.830 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.67605e-05, temp = 0.0550345 (9.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2477\n",
      "INFO:tensorflow:loss = -19.3716, step = 36201 (9.758 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.64941e-05, temp = 0.0544967 (9.758 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1832\n",
      "INFO:tensorflow:loss = -20.9999, step = 36301 (9.821 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.62304e-05, temp = 0.0539643 (9.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1987\n",
      "INFO:tensorflow:loss = -23.3346, step = 36401 (9.805 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.59694e-05, temp = 0.0534372 (9.805 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1637\n",
      "INFO:tensorflow:loss = -19.8066, step = 36501 (9.839 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.57109e-05, temp = 0.0529153 (9.838 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1614\n",
      "INFO:tensorflow:loss = -19.2467, step = 36601 (9.841 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.54551e-05, temp = 0.0523986 (9.841 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2196\n",
      "INFO:tensorflow:loss = -20.5519, step = 36701 (9.785 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.52017e-05, temp = 0.0518871 (9.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2085\n",
      "INFO:tensorflow:loss = -20.2337, step = 36801 (9.796 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.49509e-05, temp = 0.0513807 (9.796 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1613\n",
      "INFO:tensorflow:loss = -20.0414, step = 36901 (9.842 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.47026e-05, temp = 0.0508793 (9.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1148\n",
      "INFO:tensorflow:loss = -21.8871, step = 37001 (9.885 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.44567e-05, temp = 0.0503828 (9.886 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2255\n",
      "INFO:tensorflow:loss = -19.8943, step = 37101 (9.779 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.42133e-05, temp = 0.0498914 (9.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0391\n",
      "INFO:tensorflow:loss = -21.5718, step = 37201 (9.962 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.39724e-05, temp = 0.0494048 (9.962 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0347\n",
      "INFO:tensorflow:loss = -19.9275, step = 37301 (9.964 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.37338e-05, temp = 0.0489231 (9.964 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1285\n",
      "INFO:tensorflow:loss = -20.7509, step = 37401 (9.874 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.34976e-05, temp = 0.0484461 (9.874 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1788\n",
      "INFO:tensorflow:loss = -19.0952, step = 37501 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.32637e-05, temp = 0.0479739 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1271\n",
      "INFO:tensorflow:loss = -19.9563, step = 37601 (9.875 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.30322e-05, temp = 0.0475064 (9.875 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1355\n",
      "INFO:tensorflow:loss = -22.4916, step = 37701 (9.866 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.2803e-05, temp = 0.0470436 (9.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.121\n",
      "INFO:tensorflow:loss = -21.1061, step = 37801 (9.882 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.2576e-05, temp = 0.0465853 (9.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1894\n",
      "INFO:tensorflow:loss = -19.2169, step = 37901 (9.814 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.23513e-05, temp = 0.0461316 (9.814 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 10.1736\n",
      "INFO:tensorflow:loss = -20.1749, step = 38001 (9.829 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.21289e-05, temp = 0.0456825 (9.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2776\n",
      "INFO:tensorflow:loss = -19.2404, step = 38101 (9.730 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.19087e-05, temp = 0.0452378 (9.730 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2075\n",
      "INFO:tensorflow:loss = -18.6933, step = 38201 (9.797 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.16906e-05, temp = 0.0447975 (9.797 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1551\n",
      "INFO:tensorflow:loss = -20.2038, step = 38301 (9.847 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.14747e-05, temp = 0.0443616 (9.847 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1395\n",
      "INFO:tensorflow:loss = -24.5397, step = 38401 (9.863 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.1261e-05, temp = 0.0439301 (9.863 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2603\n",
      "INFO:tensorflow:loss = -21.4077, step = 38501 (9.746 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.10494e-05, temp = 0.0435028 (9.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.138\n",
      "INFO:tensorflow:loss = -22.6904, step = 38601 (9.864 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.08399e-05, temp = 0.0430798 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2632\n",
      "INFO:tensorflow:loss = -19.9292, step = 38701 (9.744 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.06325e-05, temp = 0.042661 (9.744 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2338\n",
      "INFO:tensorflow:loss = -20.0078, step = 38801 (9.772 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.04272e-05, temp = 0.0422464 (9.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2296\n",
      "INFO:tensorflow:loss = -19.9907, step = 38901 (9.776 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.02239e-05, temp = 0.0418359 (9.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1945\n",
      "INFO:tensorflow:loss = -20.2161, step = 39001 (9.809 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 3.00226e-05, temp = 0.0414295 (9.809 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2125\n",
      "INFO:tensorflow:loss = -20.9575, step = 39101 (9.792 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.98233e-05, temp = 0.0410271 (9.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3063\n",
      "INFO:tensorflow:loss = -19.3859, step = 39201 (9.703 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.96261e-05, temp = 0.0406288 (9.703 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1613\n",
      "INFO:tensorflow:loss = -20.5239, step = 39301 (9.841 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.94307e-05, temp = 0.0402344 (9.841 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.117\n",
      "INFO:tensorflow:loss = -21.2904, step = 39401 (9.885 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.92374e-05, temp = 0.0398439 (9.885 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2055\n",
      "INFO:tensorflow:loss = -20.4958, step = 39501 (9.798 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.90459e-05, temp = 0.0394573 (9.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1534\n",
      "INFO:tensorflow:loss = -20.1771, step = 39601 (9.849 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.88563e-05, temp = 0.0390746 (9.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0941\n",
      "INFO:tensorflow:loss = -22.2893, step = 39701 (9.907 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.86687e-05, temp = 0.0386957 (9.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2574\n",
      "INFO:tensorflow:loss = -21.3143, step = 39801 (9.749 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.84829e-05, temp = 0.0383205 (9.749 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1918\n",
      "INFO:tensorflow:loss = -21.3424, step = 39901 (9.812 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.82989e-05, temp = 0.0379491 (9.812 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.18795\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:34:15\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-40001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:34:22\n",
      "INFO:tensorflow:Saving dict for global step 40001: NLL_exact = -27.9253, NLL_p = -27.733, NLL_q (IS) = -27.818, global_step = 40001, loss = -27.818\n",
      "INFO:tensorflow:Validation (step 40001): loss = -27.818, NLL_p = -27.733, NLL_exact = -27.9253, NLL_q (IS) = -27.818, global_step = 40001\n",
      "INFO:tensorflow:loss = -22.8837, step = 40001 (20.054 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.81168e-05, temp = 0.0375813 (20.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.25315\n",
      "INFO:tensorflow:loss = -19.2462, step = 40101 (9.866 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.79365e-05, temp = 0.0372173 (9.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.198\n",
      "INFO:tensorflow:loss = -18.9183, step = 40201 (9.806 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.7758e-05, temp = 0.0368568 (9.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2393\n",
      "INFO:tensorflow:loss = -21.2026, step = 40301 (9.767 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.75813e-05, temp = 0.0365 (9.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0888\n",
      "INFO:tensorflow:loss = -21.9594, step = 40401 (9.912 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.74063e-05, temp = 0.0361467 (9.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2276\n",
      "INFO:tensorflow:loss = -23.8359, step = 40501 (9.777 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.72331e-05, temp = 0.0357969 (9.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1738\n",
      "INFO:tensorflow:loss = -21.4741, step = 40601 (9.829 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.70616e-05, temp = 0.0354506 (9.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1382\n",
      "INFO:tensorflow:loss = -22.2221, step = 40701 (9.864 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.68918e-05, temp = 0.0351077 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.124\n",
      "INFO:tensorflow:loss = -20.5199, step = 40801 (9.878 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.67236e-05, temp = 0.0347683 (9.878 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2255\n",
      "INFO:tensorflow:loss = -22.521, step = 40901 (9.779 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.65572e-05, temp = 0.0344322 (9.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1789\n",
      "INFO:tensorflow:loss = -20.7072, step = 41001 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.63924e-05, temp = 0.0340995 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1796\n",
      "INFO:tensorflow:loss = -21.7487, step = 41101 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.62293e-05, temp = 0.03377 (9.825 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1541\n",
      "INFO:tensorflow:loss = -21.3087, step = 41201 (9.848 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.60678e-05, temp = 0.0334439 (9.848 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0844\n",
      "INFO:tensorflow:loss = -18.2865, step = 41301 (9.917 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.59079e-05, temp = 0.033121 (9.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1201\n",
      "INFO:tensorflow:loss = -20.1808, step = 41401 (9.881 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.57495e-05, temp = 0.0328013 (9.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.213\n",
      "INFO:tensorflow:loss = -19.7957, step = 41501 (9.791 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.55928e-05, temp = 0.0324848 (9.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2121\n",
      "INFO:tensorflow:loss = -20.8355, step = 41601 (9.792 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.54376e-05, temp = 0.0321715 (9.792 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.12\n",
      "INFO:tensorflow:loss = -22.0281, step = 41701 (9.882 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.5284e-05, temp = 0.0318613 (9.882 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2202\n",
      "INFO:tensorflow:loss = -18.5535, step = 41801 (9.784 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.51319e-05, temp = 0.0315541 (9.784 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1991\n",
      "INFO:tensorflow:loss = -21.6449, step = 41901 (9.806 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.49813e-05, temp = 0.03125 (9.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.232\n",
      "INFO:tensorflow:loss = -21.3324, step = 42001 (9.772 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.48322e-05, temp = 0.030949 (9.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2398\n",
      "INFO:tensorflow:loss = -21.685, step = 42101 (9.767 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.46845e-05, temp = 0.0306509 (9.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1934\n",
      "INFO:tensorflow:loss = -17.9626, step = 42201 (9.809 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.45384e-05, temp = 0.0303558 (9.809 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1393\n",
      "INFO:tensorflow:loss = -20.1294, step = 42301 (9.862 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.43937e-05, temp = 0.0300637 (9.863 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0954\n",
      "INFO:tensorflow:loss = -18.5226, step = 42401 (9.906 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.42505e-05, temp = 0.0297744 (9.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0278\n",
      "INFO:tensorflow:loss = -20.3525, step = 42501 (9.972 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.41086e-05, temp = 0.029488 (9.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1034\n",
      "INFO:tensorflow:loss = -20.7751, step = 42601 (9.898 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.39682e-05, temp = 0.0292045 (9.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2127\n",
      "INFO:tensorflow:loss = -21.107, step = 42701 (9.791 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.38292e-05, temp = 0.0289238 (9.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1294\n",
      "INFO:tensorflow:loss = -20.7009, step = 42801 (9.873 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.36916e-05, temp = 0.0286459 (9.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0585\n",
      "INFO:tensorflow:loss = -21.6539, step = 42901 (9.942 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.35553e-05, temp = 0.0283708 (9.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1568\n",
      "INFO:tensorflow:loss = -22.0283, step = 43001 (9.845 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.34204e-05, temp = 0.0280984 (9.845 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0589\n",
      "INFO:tensorflow:loss = -20.119, step = 43101 (9.941 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.32868e-05, temp = 0.0278287 (9.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.3051\n",
      "INFO:tensorflow:loss = -21.6475, step = 43201 (9.704 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.31546e-05, temp = 0.0275617 (9.704 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2262\n",
      "INFO:tensorflow:loss = -24.7424, step = 43301 (9.778 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.30237e-05, temp = 0.0272973 (9.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1853\n",
      "INFO:tensorflow:loss = -17.2642, step = 43401 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.28941e-05, temp = 0.0270356 (9.820 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1562\n",
      "INFO:tensorflow:loss = -20.4466, step = 43501 (9.845 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.27657e-05, temp = 0.0267765 (9.845 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1388\n",
      "INFO:tensorflow:loss = -21.0493, step = 43601 (9.863 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.26387e-05, temp = 0.02652 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1443\n",
      "INFO:tensorflow:loss = -19.0991, step = 43701 (9.858 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.25129e-05, temp = 0.026266 (9.858 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1789\n",
      "INFO:tensorflow:loss = -17.3979, step = 43801 (9.824 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.23884e-05, temp = 0.0260145 (9.824 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1804\n",
      "INFO:tensorflow:loss = -21.5574, step = 43901 (9.823 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.22651e-05, temp = 0.0257656 (9.823 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1456\n",
      "INFO:tensorflow:loss = -20.5717, step = 44001 (9.857 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.2143e-05, temp = 0.0255191 (9.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1778\n",
      "INFO:tensorflow:loss = -22.5226, step = 44101 (9.825 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.20222e-05, temp = 0.0252751 (9.825 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1379\n",
      "INFO:tensorflow:loss = -22.8998, step = 44201 (9.864 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.19025e-05, temp = 0.0250335 (9.863 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1522\n",
      "INFO:tensorflow:loss = -18.973, step = 44301 (9.850 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.17841e-05, temp = 0.0247943 (9.851 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1973\n",
      "INFO:tensorflow:loss = -23.1716, step = 44401 (9.806 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.16668e-05, temp = 0.0245575 (9.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1406\n",
      "INFO:tensorflow:loss = -25.355, step = 44501 (9.862 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.15507e-05, temp = 0.024323 (9.863 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1186\n",
      "INFO:tensorflow:loss = -19.4204, step = 44601 (9.884 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.14357e-05, temp = 0.0240909 (9.884 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2294\n",
      "INFO:tensorflow:loss = -19.9822, step = 44701 (9.775 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.13219e-05, temp = 0.0238611 (9.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1417\n",
      "INFO:tensorflow:loss = -21.7772, step = 44801 (9.860 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.12092e-05, temp = 0.0236336 (9.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0517\n",
      "INFO:tensorflow:loss = -22.7872, step = 44901 (9.949 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.10977e-05, temp = 0.0234083 (9.949 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 45001 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 9.49215\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:42:37\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-45001\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:42:44\n",
      "INFO:tensorflow:Saving dict for global step 45001: NLL_exact = -27.9999, NLL_p = -27.8168, NLL_q (IS) = -27.8968, global_step = 45001, loss = -27.8968\n",
      "INFO:tensorflow:Validation (step 45001): loss = -27.8968, NLL_p = -27.8168, NLL_exact = -27.9999, NLL_q (IS) = -27.8968, global_step = 45001\n",
      "INFO:tensorflow:loss = -17.5073, step = 45001 (19.613 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.09872e-05, temp = 0.0231853 (19.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 5.29197\n",
      "INFO:tensorflow:loss = -21.4749, step = 45101 (9.818 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.08779e-05, temp = 0.0229645 (9.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1601\n",
      "INFO:tensorflow:loss = -20.7941, step = 45201 (9.843 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.07696e-05, temp = 0.0227459 (9.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0562\n",
      "INFO:tensorflow:loss = -22.6431, step = 45301 (9.943 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.06624e-05, temp = 0.0225295 (9.943 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0997\n",
      "INFO:tensorflow:loss = -22.1639, step = 45401 (9.902 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.05563e-05, temp = 0.0223152 (9.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1864\n",
      "INFO:tensorflow:loss = -19.3022, step = 45501 (9.816 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.04513e-05, temp = 0.0221031 (9.816 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1969\n",
      "INFO:tensorflow:loss = -21.6699, step = 45601 (9.807 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.03472e-05, temp = 0.0218931 (9.807 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1724\n",
      "INFO:tensorflow:loss = -19.4565, step = 45701 (9.830 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.02443e-05, temp = 0.0216851 (9.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1842\n",
      "INFO:tensorflow:loss = -19.8003, step = 45801 (9.819 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.01423e-05, temp = 0.0214793 (9.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1033\n",
      "INFO:tensorflow:loss = -17.9279, step = 45901 (9.897 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 2.00414e-05, temp = 0.0212755 (9.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1399\n",
      "INFO:tensorflow:loss = -21.503, step = 46001 (9.864 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.99414e-05, temp = 0.0210737 (9.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2115\n",
      "INFO:tensorflow:loss = -20.8437, step = 46101 (9.793 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.98425e-05, temp = 0.0208739 (9.793 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0109\n",
      "INFO:tensorflow:loss = -23.4359, step = 46201 (9.989 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.97445e-05, temp = 0.0206761 (9.989 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1356\n",
      "INFO:tensorflow:loss = -19.8757, step = 46301 (9.867 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.96476e-05, temp = 0.0204803 (9.867 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1134\n",
      "INFO:tensorflow:loss = -22.0513, step = 46401 (9.888 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.95515e-05, temp = 0.0202864 (9.888 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 10.1599\n",
      "INFO:tensorflow:loss = -19.0789, step = 46501 (9.843 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.94565e-05, temp = 0.0200945 (9.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1027\n",
      "INFO:tensorflow:loss = -22.5587, step = 46601 (9.898 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.93624e-05, temp = 0.0199044 (9.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1808\n",
      "INFO:tensorflow:loss = -22.6834, step = 46701 (9.823 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.92692e-05, temp = 0.0197163 (9.823 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1575\n",
      "INFO:tensorflow:loss = -20.8336, step = 46801 (9.845 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.91769e-05, temp = 0.01953 (9.845 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1664\n",
      "INFO:tensorflow:loss = -18.0127, step = 46901 (9.836 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.90856e-05, temp = 0.0193456 (9.836 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2062\n",
      "INFO:tensorflow:loss = -20.0049, step = 47001 (9.798 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.89952e-05, temp = 0.019163 (9.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2092\n",
      "INFO:tensorflow:loss = -23.4774, step = 47101 (9.795 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.89057e-05, temp = 0.0189823 (9.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1323\n",
      "INFO:tensorflow:loss = -19.972, step = 47201 (9.870 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.8817e-05, temp = 0.0188033 (9.870 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0765\n",
      "INFO:tensorflow:loss = -22.4722, step = 47301 (9.924 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.87293e-05, temp = 0.0186261 (9.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2199\n",
      "INFO:tensorflow:loss = -19.9842, step = 47401 (9.785 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.86424e-05, temp = 0.0184507 (9.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1944\n",
      "INFO:tensorflow:loss = -20.5864, step = 47501 (9.808 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.85564e-05, temp = 0.018277 (9.808 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1855\n",
      "INFO:tensorflow:loss = -20.9976, step = 47601 (9.818 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.84712e-05, temp = 0.0181051 (9.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.15\n",
      "INFO:tensorflow:loss = -20.9418, step = 47701 (9.853 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.83869e-05, temp = 0.0179348 (9.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1999\n",
      "INFO:tensorflow:loss = -21.3723, step = 47801 (9.804 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.83035e-05, temp = 0.0177663 (9.804 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0598\n",
      "INFO:tensorflow:loss = -19.5339, step = 47901 (9.942 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.82208e-05, temp = 0.0175994 (9.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.156\n",
      "INFO:tensorflow:loss = -21.96, step = 48001 (9.844 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.8139e-05, temp = 0.0174342 (9.844 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1636\n",
      "INFO:tensorflow:loss = -19.7514, step = 48101 (9.840 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.8058e-05, temp = 0.0172707 (9.840 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0358\n",
      "INFO:tensorflow:loss = -22.0741, step = 48201 (9.964 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.79778e-05, temp = 0.0171087 (9.964 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1049\n",
      "INFO:tensorflow:loss = -23.8166, step = 48301 (9.897 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.78984e-05, temp = 0.0169484 (9.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1901\n",
      "INFO:tensorflow:loss = -19.6699, step = 48401 (9.812 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.78198e-05, temp = 0.0167897 (9.812 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.0509\n",
      "INFO:tensorflow:loss = -20.2964, step = 48501 (9.950 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.7742e-05, temp = 0.0166325 (9.950 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1491\n",
      "INFO:tensorflow:loss = -19.5573, step = 48601 (9.852 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.76649e-05, temp = 0.016477 (9.852 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2396\n",
      "INFO:tensorflow:loss = -21.1239, step = 48701 (9.767 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.75886e-05, temp = 0.0163229 (9.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.2358\n",
      "INFO:tensorflow:loss = -21.8348, step = 48801 (9.769 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.75131e-05, temp = 0.0161704 (9.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.145\n",
      "INFO:tensorflow:loss = -18.8715, step = 48901 (9.858 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.74383e-05, temp = 0.0160195 (9.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1588\n",
      "INFO:tensorflow:loss = -21.6589, step = 49001 (9.843 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.73643e-05, temp = 0.01587 (9.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1461\n",
      "INFO:tensorflow:loss = -22.8781, step = 49101 (9.857 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.7291e-05, temp = 0.015722 (9.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1557\n",
      "INFO:tensorflow:loss = -22.0128, step = 49201 (9.846 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.72185e-05, temp = 0.0155755 (9.846 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1515\n",
      "INFO:tensorflow:loss = -19.2948, step = 49301 (9.851 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.71466e-05, temp = 0.0154304 (9.851 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1553\n",
      "INFO:tensorflow:loss = -20.8754, step = 49401 (9.847 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.70755e-05, temp = 0.0152868 (9.847 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1681\n",
      "INFO:tensorflow:loss = -19.1984, step = 49501 (9.835 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.70051e-05, temp = 0.0151446 (9.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1529\n",
      "INFO:tensorflow:loss = -19.6002, step = 49601 (9.849 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.69354e-05, temp = 0.0150038 (9.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1761\n",
      "INFO:tensorflow:loss = -21.4679, step = 49701 (9.827 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.68663e-05, temp = 0.0148645 (9.827 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1895\n",
      "INFO:tensorflow:loss = -21.8411, step = 49801 (9.814 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.6798e-05, temp = 0.0147265 (9.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 10.1743\n",
      "INFO:tensorflow:loss = -21.9157, step = 49901 (9.828 sec)\n",
      "INFO:tensorflow:kl_weight = 1.0, lr = 1.67303e-05, temp = 0.0145899 (9.828 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 50000 into /data/trafficweaving/models/GPU1_checkpoints/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: -22.3999.\n",
      "INFO:tensorflow:Starting evaluation at 2017-10-13-09:50:59\n",
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-50000\n",
      "INFO:tensorflow:Finished evaluation at 2017-10-13-09:51:07\n",
      "INFO:tensorflow:Saving dict for global step 50000: NLL_exact = -27.9938, NLL_p = -27.7986, NLL_q (IS) = -27.8868, global_step = 50000, loss = -27.8868\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({'NLL_exact': -27.993811,\n",
       "  'NLL_p': -27.798567,\n",
       "  'NLL_q (IS)': -27.88678,\n",
       "  'global_step': 50000,\n",
       "  'loss': -27.88678},\n",
       " [])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment.train_and_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /data/trafficweaving/models/GPU1_checkpoints/model.ckpt-50000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /data/trafficweaving/models/1507924360/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "state_dim = train_data_dict[\"input_dict\"][\"car1\"].shape[2]\n",
    "extras_dim = train_data_dict[\"input_dict\"][\"extras\"].shape[2]\n",
    "ph = hps.prediction_horizon\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    serving_input_receiver_fn = tf.estimator.export.build_raw_serving_input_receiver_fn(\n",
    "        {\"car1\": tf.placeholder(tf.float32, shape=[1, None, state_dim], name=\"car1\"),\n",
    "         \"car2\": tf.placeholder(tf.float32, shape=[1, None, state_dim], name=\"car2\"),\n",
    "         \"extras\": tf.placeholder(tf.float32, shape=[1, None, extras_dim], name=\"extras\"),\n",
    "         \"traj_lengths\": tf.placeholder(tf.int32, shape=[1], name=\"traj_lengths\"),\n",
    "         \"car1_future_x\": tf.placeholder(tf.float32, shape=[None, ph, state_dim/2], name=\"car1_future_x\"),\n",
    "         \"car1_future_y\": tf.placeholder(tf.float32, shape=[None, ph, state_dim/2], name=\"car1_future_y\"),\n",
    "         \"sample_ct\": tf.placeholder(tf.int32, shape=[1], name=\"sample_ct\")})\n",
    "    nn.export_savedmodel(config[\"models_dir\"], serving_input_receiver_fn,\n",
    "#                        checkpoint_path=\"/home/schmrlng/tf_training/laneswap_cvae/CPU/model.ckpt-11001\"\n",
    "                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}