RayTest_smallInputs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d7b692a5-7f8c-4c30-8364-269cf3d8ab4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from energyflow.archs import PFN\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import ray\n",
    "from ray import tune\n",
    "from ray.train import Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a0e3f21e-83ff-4cfc-a2e6-2806cd6b7a3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b9a5547f-c12b-4245-baa2-319e2d391909",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install -U tensorboardx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fefdc683-bfee-45e4-8fd1-9c7e3b9e8346",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install py-rsync"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7c73037e-b142-42d2-ac96-dfa8b101e966",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compile_model(config):\n",
    "        compile_opts={ 'loss':'categorical_crossentropy',\n",
    "                       'optimizer':'adam',\n",
    "                       'metrics':'acc',\n",
    "                       'weighted_metrics':[\"categorical_crossentropy\"],\n",
    "                      }\n",
    "        print(\"compile_model...\")\n",
    "        #print(\"in_dim:{}\".format(in_dim))\n",
    "        Phi_sizes = config['Phi_sizes']\n",
    "        F_sizes = config['F_sizes']\n",
    "#         latent_dropout = config['latent_dropout']\n",
    "#         F_dropouts = config['F_dropouts']\n",
    "        DSNNr = PFN(input_dim=5,\n",
    "                    Phi_sizes=Phi_sizes, F_sizes=F_sizes,\n",
    "                    summary=False,\n",
    "                    mask_val=-99,\n",
    "#                     latent_dropout=latent_dropout,\n",
    "#                     F_dropouts=F_dropouts,\n",
    "                    compile_opts=compile_opts)\n",
    "        return DSNNr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b5a987f5-3d01-4e00-bacc-2d16c6ad76ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dir = tf.keras.callbacks.ModelCheckpoint('./saved_models/' + 'Raytest',\n",
    "                                 monitor='val_loss',\n",
    "                                 verbose=2,\n",
    "                                 save_best_only=True,\n",
    "                                 mode='min')\n",
    "earlyStopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                                  min_delta=0,\n",
    "                                  patience=500,\n",
    "                                  verbose=1,\n",
    "                                  restore_best_weights=True)\n",
    "callbacks = [checkpoint_dir, earlyStopping]\n",
    "\n",
    "def train_and_score(config,checkpoint_dir=None):\n",
    "        X_train = np.load(\"/home/jovyan/SplitArraysTest.npz\")['X_train']\n",
    "        Y_train = np.load(\"/home/jovyan/SplitArraysTest.npz\")['Y_train']\n",
    "        X_test = np.load(\"/home/jovyan/SplitArraysTest.npz\")['X_test']\n",
    "        Y_test = np.load(\"/home/jovyan/SplitArraysTest.npz\")['Y_test']\n",
    "\n",
    "        model = compile_model(config)\n",
    "        print(\"start to train_and_score the compiling model...\")\n",
    "        model.fit(X_train, Y_train,\n",
    "                  batch_size=20000,\n",
    "                  epochs=5,  # using early stopping, so no real limit\n",
    "                  verbose=0,\n",
    "                  validation_data=(X_test, Y_test),\n",
    "                  callbacks=[callbacks])\n",
    "\n",
    "        score = model.evaluate(X_test, Y_test, verbose=0)\n",
    "        tune.report(accuracy=score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9b84942a-e7d7-40fa-90b1-0134a19e1fdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 23:22:36,774\tWARNING services.py:1748 -- WARNING: The object store is using /tmp instead of /dev/shm because /dev/shm has only 67108864 bytes available. This will harm performance! You may be able to free up space by deleting files in /dev/shm. If you are inside a Docker container, you can increase /dev/shm size by passing '--shm-size=10.24gb' to 'docker run' (or add it to the run_options list in a Ray cluster config). Make sure to set this to more than 30% of available RAM.\n",
      "2021-11-05 23:22:37,747\tERROR syncer.py:75 -- Log sync requires rsync to be installed.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:22:39 (running for 00:00:02.32)<br>Memory usage on this node: 2.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (3 PENDING, 1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status  </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>RUNNING </td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>PENDING </td><td>              </td><td>(125, 125, 125)</td><td>(120, 120, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>PENDING </td><td>              </td><td>(120, 120, 120)</td><td>(125, 125, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m compile_model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.301966: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.310440: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.311136: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.312027: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.312712: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.313399: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:40.314044: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:41.048088: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:41.048854: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:41.049620: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:41.050233: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13803 MB memory:  -> device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m start to train_and_score the compiling model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:41.645071: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:22:42 (running for 00:00:05.17)<br>Memory usage on this node: 4.7/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (3 PENDING, 1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status  </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>RUNNING </td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>PENDING </td><td>              </td><td>(125, 125, 125)</td><td>(120, 120, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>PENDING </td><td>              </td><td>(120, 120, 120)</td><td>(125, 125, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m Epoch 00001: val_loss improved from inf to 77.88187, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7692)\u001b[0m 2021-11-05 23:22:44.452113: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m Epoch 00002: val_loss improved from 77.88187 to 19.85150, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:22:48 (running for 00:00:11.13)<br>Memory usage on this node: 5.1/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (3 PENDING, 1 RUNNING)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status  </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>RUNNING </td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>PENDING </td><td>              </td><td>(125, 125, 125)</td><td>(120, 120, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>PENDING </td><td>              </td><td>(120, 120, 120)</td><td>(125, 125, 100)</td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m Epoch 00003: val_loss did not improve from 19.85150\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m Epoch 00004: val_loss did not improve from 19.85150\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7692)\u001b[0m Epoch 00005: val_loss improved from 19.85150 to 3.41382, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 23:22:53,458\tINFO logger.py:706 -- Removed the following hyperparameter values when logging to tensorboard: {'F_sizes': (120, 120, 120), 'Phi_sizes': (120, 120, 100)}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for train_and_score_4385f_00000:\n",
      "  accuracy: 0.104319728910923\n",
      "  date: 2021-11-05_23-22-53\n",
      "  done: false\n",
      "  experiment_id: 03b2c25cc8144665ba21eec5a10e1799\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 7692\n",
      "  time_since_restore: 13.526757955551147\n",
      "  time_this_iter_s: 13.526757955551147\n",
      "  time_total_s: 13.526757955551147\n",
      "  timestamp: 1636154573\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00000\n",
      "  \n",
      "Result for train_and_score_4385f_00000:\n",
      "  accuracy: 0.104319728910923\n",
      "  date: 2021-11-05_23-22-53\n",
      "  done: true\n",
      "  experiment_id: 03b2c25cc8144665ba21eec5a10e1799\n",
      "  experiment_tag: 0_F_sizes=(120, 120, 120),Phi_sizes=(120, 120, 100)\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 7692\n",
      "  time_since_restore: 13.526757955551147\n",
      "  time_this_iter_s: 13.526757955551147\n",
      "  time_total_s: 13.526757955551147\n",
      "  timestamp: 1636154573\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00000\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:22:55 (running for 00:00:18.37)<br>Memory usage on this node: 2.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (2 PENDING, 1 RUNNING, 1 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>RUNNING   </td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>PENDING   </td><td>              </td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING   </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">   0.10432</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m compile_model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.372819: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.382433: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.383102: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.384059: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.384624: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.385321: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:56.385950: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:57.100907: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:57.101660: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:57.102306: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:57.102891: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13803 MB memory:  -> device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m start to train_and_score the compiling model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:22:57.690385: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m Epoch 00001: val_loss improved from inf to 83.99094, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:22:59 (running for 00:00:22.39)<br>Memory usage on this node: 4.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (2 PENDING, 1 RUNNING, 1 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>RUNNING   </td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>PENDING   </td><td>              </td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING   </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">   0.10432</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7850)\u001b[0m 2021-11-05 23:23:00.502224: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m Epoch 00002: val_loss did not improve from 83.99094\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m Epoch 00003: val_loss improved from 83.99094 to 38.57863, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:05 (running for 00:00:27.40)<br>Memory usage on this node: 5.1/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (2 PENDING, 1 RUNNING, 1 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>RUNNING   </td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>PENDING   </td><td>              </td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING   </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">   0.10432</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m Epoch 00004: val_loss improved from 38.57863 to 7.59706, saving model to ./saved_models/Raytest\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7850)\u001b[0m Epoch 00005: val_loss did not improve from 7.59706\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 23:23:09,327\tINFO logger.py:706 -- Removed the following hyperparameter values when logging to tensorboard: {'F_sizes': (125, 125, 125), 'Phi_sizes': (120, 120, 100)}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for train_and_score_4385f_00001:\n",
      "  accuracy: 0.9428571462631226\n",
      "  date: 2021-11-05_23-23-09\n",
      "  done: false\n",
      "  experiment_id: 1b40d6e7774345e18f299ad2ec460b3a\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 7850\n",
      "  time_since_restore: 13.342462539672852\n",
      "  time_this_iter_s: 13.342462539672852\n",
      "  time_total_s: 13.342462539672852\n",
      "  timestamp: 1636154589\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00001\n",
      "  \n",
      "Result for train_and_score_4385f_00001:\n",
      "  accuracy: 0.9428571462631226\n",
      "  date: 2021-11-05_23-23-09\n",
      "  done: true\n",
      "  experiment_id: 1b40d6e7774345e18f299ad2ec460b3a\n",
      "  experiment_tag: 1_F_sizes=(125, 125, 125),Phi_sizes=(120, 120, 100)\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 7850\n",
      "  time_since_restore: 13.342462539672852\n",
      "  time_this_iter_s: 13.342462539672852\n",
      "  time_total_s: 13.342462539672852\n",
      "  timestamp: 1636154589\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00001\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:11 (running for 00:00:34.33)<br>Memory usage on this node: 2.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 PENDING, 1 RUNNING, 2 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>RUNNING   </td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING   </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m compile_model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.324340: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.333053: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.333813: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.334753: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.335284: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.336001: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:12.336673: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:13.046013: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:13.046779: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:13.047409: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:13.047983: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13803 MB memory:  -> device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m start to train_and_score the compiling model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:13.638826: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m Epoch 00001: val_loss improved from inf to 56.83202, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:15 (running for 00:00:38.34)<br>Memory usage on this node: 4.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 PENDING, 1 RUNNING, 2 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>RUNNING   </td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING   </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=7978)\u001b[0m 2021-11-05 23:23:16.481889: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m Epoch 00002: val_loss improved from 56.83202 to 54.75808, saving model to ./saved_models/Raytest\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m Epoch 00003: val_loss improved from 54.75808 to 9.93499, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:20 (running for 00:00:43.35)<br>Memory usage on this node: 5.1/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 PENDING, 1 RUNNING, 2 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>RUNNING   </td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>PENDING   </td><td>              </td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m Epoch 00004: val_loss did not improve from 9.93499\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=7978)\u001b[0m Epoch 00005: val_loss did not improve from 9.93499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 23:23:25,794\tINFO logger.py:706 -- Removed the following hyperparameter values when logging to tensorboard: {'F_sizes': (120, 120, 120), 'Phi_sizes': (125, 125, 100)}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for train_and_score_4385f_00002:\n",
      "  accuracy: 0.9428571462631226\n",
      "  date: 2021-11-05_23-23-25\n",
      "  done: false\n",
      "  experiment_id: 18e785a13a864f6794eda9d39fa7b65a\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 7978\n",
      "  time_since_restore: 13.856202125549316\n",
      "  time_this_iter_s: 13.856202125549316\n",
      "  time_total_s: 13.856202125549316\n",
      "  timestamp: 1636154605\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00002\n",
      "  \n",
      "Result for train_and_score_4385f_00002:\n",
      "  accuracy: 0.9428571462631226\n",
      "  date: 2021-11-05_23-23-25\n",
      "  done: true\n",
      "  experiment_id: 18e785a13a864f6794eda9d39fa7b65a\n",
      "  experiment_tag: 2_F_sizes=(120, 120, 120),Phi_sizes=(125, 125, 100)\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 7978\n",
      "  time_since_restore: 13.856202125549316\n",
      "  time_this_iter_s: 13.856202125549316\n",
      "  time_total_s: 13.856202125549316\n",
      "  timestamp: 1636154605\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00002\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:28 (running for 00:00:50.80)<br>Memory usage on this node: 2.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 RUNNING, 3 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>RUNNING   </td><td>10.64.1.4:8106</td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>TERMINATED</td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.8562</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m compile_model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.791869: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.800318: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.801031: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.801926: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.802508: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.803318: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:28.803954: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:29.519965: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:29.520792: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:29.521457: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:29.522046: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13803 MB memory:  -> device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m start to train_and_score the compiling model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:30.099287: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m Epoch 00001: val_loss improved from inf to 37.01227, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:32 (running for 00:00:54.81)<br>Memory usage on this node: 4.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 RUNNING, 3 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>RUNNING   </td><td>10.64.1.4:8106</td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>TERMINATED</td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.8562</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=8106)\u001b[0m 2021-11-05 23:23:32.914310: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m Epoch 00002: val_loss improved from 37.01227 to 33.55730, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:37 (running for 00:00:59.81)<br>Memory usage on this node: 5.1/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 RUNNING, 3 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>RUNNING   </td><td>10.64.1.4:8106</td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>TERMINATED</td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.8562</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m Epoch 00003: val_loss did not improve from 33.55730\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m Epoch 00004: val_loss improved from 33.55730 to 21.31331, saving model to ./saved_models/Raytest\n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m \n",
      "\u001b[2m\u001b[36m(ImplicitFunc pid=8106)\u001b[0m Epoch 00005: val_loss improved from 21.31331 to 18.32895, saving model to ./saved_models/Raytest\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:42 (running for 00:01:04.82)<br>Memory usage on this node: 4.9/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 1.0/1 CPUs, 1.0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (1 RUNNING, 3 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>RUNNING   </td><td>10.64.1.4:8106</td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>TERMINATED</td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.8562</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 23:23:43,973\tINFO logger.py:706 -- Removed the following hyperparameter values when logging to tensorboard: {'F_sizes': (125, 125, 125), 'Phi_sizes': (125, 125, 100)}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for train_and_score_4385f_00003:\n",
      "  accuracy: 0.9428571462631226\n",
      "  date: 2021-11-05_23-23-43\n",
      "  done: false\n",
      "  experiment_id: 91b9ab67b9d3408ba78675dc6078e446\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 8106\n",
      "  time_since_restore: 15.564718961715698\n",
      "  time_this_iter_s: 15.564718961715698\n",
      "  time_total_s: 15.564718961715698\n",
      "  timestamp: 1636154623\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00003\n",
      "  \n",
      "Result for train_and_score_4385f_00003:\n",
      "  accuracy: 0.9428571462631226\n",
      "  date: 2021-11-05_23-23-43\n",
      "  done: true\n",
      "  experiment_id: 91b9ab67b9d3408ba78675dc6078e446\n",
      "  experiment_tag: 3_F_sizes=(125, 125, 125),Phi_sizes=(125, 125, 100)\n",
      "  hostname: jupyter-fangying\n",
      "  iterations_since_restore: 1\n",
      "  node_ip: 10.64.1.4\n",
      "  pid: 8106\n",
      "  time_since_restore: 15.564718961715698\n",
      "  time_this_iter_s: 15.564718961715698\n",
      "  time_total_s: 15.564718961715698\n",
      "  timestamp: 1636154623\n",
      "  timesteps_since_restore: 0\n",
      "  training_iteration: 1\n",
      "  trial_id: 4385f_00003\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2021-11-05 23:23:43 (running for 00:01:06.38)<br>Memory usage on this node: 4.6/51.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/1 CPUs, 0/1 GPUs, 0.0/35.03 GiB heap, 0.0/9.31 GiB objects (0.0/1.0 accelerator_type:T4)<br>Result logdir: /home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37<br>Number of trials: 4/4 (4 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc           </th><th>F_sizes        </th><th>Phi_sizes      </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_and_score_4385f_00000</td><td>TERMINATED</td><td>10.64.1.4:7692</td><td>(120, 120, 120)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.5268</td><td style=\"text-align: right;\">  0.10432 </td></tr>\n",
       "<tr><td>train_and_score_4385f_00001</td><td>TERMINATED</td><td>10.64.1.4:7850</td><td>(125, 125, 125)</td><td>(120, 120, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.3425</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "<tr><td>train_and_score_4385f_00002</td><td>TERMINATED</td><td>10.64.1.4:7978</td><td>(120, 120, 120)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         13.8562</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "<tr><td>train_and_score_4385f_00003</td><td>TERMINATED</td><td>10.64.1.4:8106</td><td>(125, 125, 125)</td><td>(125, 125, 100)</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         15.5647</td><td style=\"text-align: right;\">  0.942857</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-05 23:23:44,083\tINFO tune.py:630 -- Total run time: 66.53 seconds (66.37 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best config:  {'Phi_sizes': (120, 120, 100), 'F_sizes': (125, 125, 125)}\n",
      "CPU times: user 1.36 s, sys: 413 ms, total: 1.78 s\n",
      "Wall time: 1min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "analysis = tune.run(\n",
    "    train_and_score,\n",
    "    config={\n",
    "        'Phi_sizes': tune.grid_search([ (120,120,100),(125,125,100)]),\n",
    "        'F_sizes':tune.grid_search([(120,120,120),(125,125,125)])\n",
    "#          'latent_dropout': [0.01,0.15, 0.2],\n",
    "#         'F_dropouts': [0.01,0.15,0.2]\n",
    "        #\"alpha\": tune.grid_search([0.001, 0.01, 0.1]),\n",
    "        #\"beta\": tune.choice([1, 2, 3])\n",
    "    },\n",
    "    resources_per_trial={'gpu': 1})\n",
    "print(\"Best config: \", analysis.get_best_config(metric=\"accuracy\", mode=\"max\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4d2b6b0e-b418-4744-bb17-e236cf4f6c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d1a6a509-1f29-4c8e-bcb1-5ffacecba9d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dd7026b2-cc91-4a64-8666-5f9f253bc399",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-e052615a4ff32083\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-e052615a4ff32083\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "log_path = \"/home/jovyan/ray_results/train_and_score_2021-11-05_23-22-37\"\n",
    "%tensorboard --logdir log_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "627e8bdb-d701-45a3-b09d-7e79b48a309a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d9419f9-ddd3-4f3d-98bc-72958ada978f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}