ParamSearchOptuna.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-08-02T01:22:19.728811Z",
     "start_time": "2024-08-02T01:22:18.554930Z"
    }
   },
   "source": [
    "#Importing all the different python modules needed\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import pandas as pd\n",
    "from torch.utils.data import TensorDataset, DataLoader, random_split\n",
    "from ffModel import FeedForward\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import optuna"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-08-02T01:22:19.734231Z",
     "start_time": "2024-08-02T01:22:19.729566Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def objective(trial):\n",
    "    # Setup Hyperparameters to be tested\n",
    "    learning_rate = trial.suggest_float(\"learning_rate\", 1e-5, 1e-1, log=True)\n",
    "    dropout_rate = trial.suggest_float(\"dropout_rate\", 1e-1, 1.0)\n",
    "    batch_size = trial.suggest_int(\"batch_size\", 20, 40)\n",
    "    epochs = trial.suggest_int(\"epochs\", 100, 200)\n",
    "    betaA = trial.suggest_float(\"betaA\", 1e-1, 1, log=False)\n",
    "    betaB = trial.suggest_float(\"betaB\", 1e-1, 1, log=False)\n",
    "    weight_decay = trial.suggest_float(\"weight_decay\", 1e-5, 1e-1, log=True)\n",
    "    patient = trial.suggest_int('patient', 1, 101, step = 10)\n",
    "    scale_factor = trial.suggest_float('scale_factor', 1e-6, 1e-1, log=True)\n",
    "    \n",
    "    # Create the model, initialize the weights, setup loss, and optimizer\n",
    "    model = FeedForward(dropout_rate).to('cuda')\n",
    "    model.init_weights()\n",
    "    criterion = nn.MSELoss()\n",
    "    optimizer = optim.AdamW(model.parameters(), lr=learning_rate, betas= (betaA,betaB), weight_decay=weight_decay)\n",
    "    \n",
    "    #importing data into pandas data frame.\n",
    "    train_data = pd.read_csv('train.csv')\n",
    "    test_data = pd.read_csv('test.csv')\n",
    "    \n",
    "    #Dropping non-data column from testing.\n",
    "    test_data.drop(['Id'], axis=1, inplace=True)\n",
    "    \n",
    "    # Separate target variable from training data and drop it as well as ID variable from training inputs.\n",
    "    Y_train = train_data['SalePrice']\n",
    "    X_train = train_data.drop(['SalePrice', 'Id'], axis=1)\n",
    "    \n",
    "    # Combine training and test data for consistent encoding(I was getting different sized inputs when using dummies separately).\n",
    "    combined_data = pd.concat([X_train, test_data], axis=0)\n",
    "    \n",
    "    # Handle categorical variables by applying one hot encoding with pandas\n",
    "    combined_data = pd.get_dummies(combined_data)\n",
    "    \n",
    "    # Fill missing values with column mean with pandas\n",
    "    combined_data = combined_data.fillna(combined_data.mean())\n",
    "    \n",
    "    # Separate the combined data back into training and test sets\n",
    "    X_train = combined_data.iloc[:len(X_train), :]\n",
    "    X_test = combined_data.iloc[len(X_train):, :]\n",
    "    \n",
    "    # Standardize the data using scikit-learn(This is also to help the problem of exploding gradients w/this dataset).\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    \n",
    "    # Convert to tensors\n",
    "    X_train = torch.tensor(X_train, dtype=torch.float32)\n",
    "    Y_train = torch.tensor(Y_train.values, dtype=torch.float32).view(-1, 1)\n",
    "    X_test = torch.tensor(X_test, dtype=torch.float32)\n",
    "    \n",
    "    #create actual testing dataset which we will split into a testing dataset and validation data set\n",
    "    dataset = TensorDataset(X_train, Y_train)\n",
    "    train_size = int(.8 * len(dataset))\n",
    "    val_size = len(dataset) - train_size\n",
    "    train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
    "    \n",
    "    #create final training data loader.\n",
    "    trainValuesFinal = DataLoader(train_dataset,batch_size= batch_size,  shuffle=True)\n",
    "    \n",
    "    #set up plateau scheduler(I found this scheduler to work the best consistently via trial and error)\n",
    "    scheduler = ReduceLROnPlateau(optimizer, 'min', patience=patient, factor=scale_factor)\n",
    "    \n",
    "    # Setup average loss variable for scheduler and then set model to training mode\n",
    "    average_loss = 0\n",
    "    model.train()\n",
    "    \n",
    "    #Actual training loop, I have a dedicated GPU for ML training, so I use cuda for all of my mats.\n",
    "    for epoch in range(epochs):\n",
    "        for inputs, targets in trainValuesFinal:\n",
    "            inputs = inputs.to('cuda')\n",
    "            targets = targets.to('cuda')\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            loss.backward()\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)\n",
    "            optimizer.step()\n",
    "            average_loss += loss.item()\n",
    "        scheduler.step(average_loss/len(trainValuesFinal))\n",
    "        \n",
    "    # Set model to eval mode then use the val dataset to evaluate, use torch.no_grade so more efficient as not calculating gradients.\n",
    "    model.eval()\n",
    "    val_loss = 0\n",
    "    with torch.no_grad():\n",
    "        for inputs, targets in val_dataset:\n",
    "            inputs = inputs.to('cuda')\n",
    "            targets = targets.to('cuda')\n",
    "            outputs = model(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            val_loss += loss.item()\n",
    "    # Actual value which optuna then tries to minimize or maximize based on your choices.\n",
    "    return val_loss / len(val_dataset) "
   ],
   "id": "361ef2e5e260a8cb",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-08-02T01:22:49.203904Z",
     "start_time": "2024-08-02T01:22:19.734736Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#create the study and optimize will 300 trials\n",
    "study = optuna.create_study(direction='minimize', study_name='HousingPrice')\n",
    "study.optimize(objective, n_trials=300)"
   ],
   "id": "70109b86f870cc68",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-01 18:22:19,735] A new study created in memory with name: HousingPrice\n",
      "[W 2024-08-01 18:22:27,590] Trial 0 failed with parameters: {'learning_rate': 0.07317406651190207, 'dropout_rate': 0.5532144617861772, 'batch_size': 32, 'epochs': 175, 'betaA': 0.9429396737083805, 'betaB': 0.6372050438272204, 'weight_decay': 0.054673428377584425, 'patient': 71, 'scale_factor': 3.4559533154700992e-06} because of the following error: The value nan is not acceptable.\n",
      "[W 2024-08-01 18:22:27,590] Trial 0 failed with value nan.\n",
      "[I 2024-08-01 18:22:34,694] Trial 1 finished with value: 26004294668.328766 and parameters: {'learning_rate': 0.06940715432116638, 'dropout_rate': 0.7618598269490245, 'batch_size': 38, 'epochs': 199, 'betaA': 0.52953211512859, 'betaB': 0.36043321988689425, 'weight_decay': 3.522444707791381e-05, 'patient': 51, 'scale_factor': 0.00040523825869060267}. Best is trial 1 with value: 26004294668.328766.\n",
      "[I 2024-08-01 18:22:43,828] Trial 2 finished with value: 38226083916.712326 and parameters: {'learning_rate': 0.003289379885318159, 'dropout_rate': 0.9540705477998936, 'batch_size': 26, 'epochs': 186, 'betaA': 0.7535848731271653, 'betaB': 0.6751528124692794, 'weight_decay': 0.00011985762741698462, 'patient': 91, 'scale_factor': 0.028762474275049656}. Best is trial 1 with value: 26004294668.328766.\n",
      "[W 2024-08-01 18:22:48,944] Trial 3 failed with parameters: {'learning_rate': 0.00017780574990646034, 'dropout_rate': 0.8084505248765608, 'batch_size': 22, 'epochs': 177, 'betaA': 0.6781107736545887, 'betaB': 0.9699070544342354, 'weight_decay': 0.000296661186368999, 'patient': 41, 'scale_factor': 0.004852725167866951} because of the following error: KeyboardInterrupt().\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/abog/global_env/lib/python3.12/site-packages/optuna/study/_optimize.py\", line 196, in _run_trial\n",
      "    value_or_values = func(trial)\n",
      "                      ^^^^^^^^^^^\n",
      "  File \"/tmp/ipykernel_5013/3144668794.py\", line 77, in objective\n",
      "    loss.backward()\n",
      "  File \"/home/abog/global_env/lib/python3.12/site-packages/torch/_tensor.py\", line 521, in backward\n",
      "    torch.autograd.backward(\n",
      "  File \"/home/abog/global_env/lib/python3.12/site-packages/torch/autograd/__init__.py\", line 289, in backward\n",
      "    _engine_run_backward(\n",
      "  File \"/home/abog/global_env/lib/python3.12/site-packages/torch/autograd/graph.py\", line 768, in _engine_run_backward\n",
      "    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "KeyboardInterrupt\n",
      "[W 2024-08-01 18:22:48,946] Trial 3 failed with value None.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[3], line 3\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;66;03m#create the study and optimize will 300 trials\u001B[39;00m\n\u001B[1;32m      2\u001B[0m study \u001B[38;5;241m=\u001B[39m optuna\u001B[38;5;241m.\u001B[39mcreate_study(direction\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mminimize\u001B[39m\u001B[38;5;124m'\u001B[39m, study_name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mHousingPrice\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[0;32m----> 3\u001B[0m \u001B[43mstudy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43moptimize\u001B[49m\u001B[43m(\u001B[49m\u001B[43mobjective\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mn_trials\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m300\u001B[39;49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/optuna/study/study.py:451\u001B[0m, in \u001B[0;36mStudy.optimize\u001B[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001B[0m\n\u001B[1;32m    348\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21moptimize\u001B[39m(\n\u001B[1;32m    349\u001B[0m     \u001B[38;5;28mself\u001B[39m,\n\u001B[1;32m    350\u001B[0m     func: ObjectiveFuncType,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    357\u001B[0m     show_progress_bar: \u001B[38;5;28mbool\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[1;32m    358\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m    359\u001B[0m \u001B[38;5;250m    \u001B[39m\u001B[38;5;124;03m\"\"\"Optimize an objective function.\u001B[39;00m\n\u001B[1;32m    360\u001B[0m \n\u001B[1;32m    361\u001B[0m \u001B[38;5;124;03m    Optimization is done by choosing a suitable set of hyperparameter values from a given\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    449\u001B[0m \u001B[38;5;124;03m            If nested invocation of this method occurs.\u001B[39;00m\n\u001B[1;32m    450\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m--> 451\u001B[0m     \u001B[43m_optimize\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    452\u001B[0m \u001B[43m        \u001B[49m\u001B[43mstudy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m    453\u001B[0m \u001B[43m        \u001B[49m\u001B[43mfunc\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfunc\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    454\u001B[0m \u001B[43m        \u001B[49m\u001B[43mn_trials\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mn_trials\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    455\u001B[0m \u001B[43m        \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    456\u001B[0m \u001B[43m        \u001B[49m\u001B[43mn_jobs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mn_jobs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    457\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcatch\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mtuple\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m)\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;28;43misinstance\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mIterable\u001B[49m\u001B[43m)\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01melse\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43m(\u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m,\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    458\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcallbacks\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcallbacks\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    459\u001B[0m \u001B[43m        \u001B[49m\u001B[43mgc_after_trial\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mgc_after_trial\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    460\u001B[0m \u001B[43m        \u001B[49m\u001B[43mshow_progress_bar\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mshow_progress_bar\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    461\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/optuna/study/_optimize.py:62\u001B[0m, in \u001B[0;36m_optimize\u001B[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001B[0m\n\u001B[1;32m     60\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m     61\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m n_jobs \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m---> 62\u001B[0m         \u001B[43m_optimize_sequential\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     63\u001B[0m \u001B[43m            \u001B[49m\u001B[43mstudy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     64\u001B[0m \u001B[43m            \u001B[49m\u001B[43mfunc\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     65\u001B[0m \u001B[43m            \u001B[49m\u001B[43mn_trials\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     66\u001B[0m \u001B[43m            \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     67\u001B[0m \u001B[43m            \u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     68\u001B[0m \u001B[43m            \u001B[49m\u001B[43mcallbacks\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     69\u001B[0m \u001B[43m            \u001B[49m\u001B[43mgc_after_trial\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     70\u001B[0m \u001B[43m            \u001B[49m\u001B[43mreseed_sampler_rng\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m     71\u001B[0m \u001B[43m            \u001B[49m\u001B[43mtime_start\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m     72\u001B[0m \u001B[43m            \u001B[49m\u001B[43mprogress_bar\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mprogress_bar\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     73\u001B[0m \u001B[43m        \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     74\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m     75\u001B[0m         \u001B[38;5;28;01mif\u001B[39;00m n_jobs \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m:\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/optuna/study/_optimize.py:159\u001B[0m, in \u001B[0;36m_optimize_sequential\u001B[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001B[0m\n\u001B[1;32m    156\u001B[0m         \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[1;32m    158\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 159\u001B[0m     frozen_trial \u001B[38;5;241m=\u001B[39m \u001B[43m_run_trial\u001B[49m\u001B[43m(\u001B[49m\u001B[43mstudy\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfunc\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcatch\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    160\u001B[0m \u001B[38;5;28;01mfinally\u001B[39;00m:\n\u001B[1;32m    161\u001B[0m     \u001B[38;5;66;03m# The following line mitigates memory problems that can be occurred in some\u001B[39;00m\n\u001B[1;32m    162\u001B[0m     \u001B[38;5;66;03m# environments (e.g., services that use computing containers such as GitHub Actions).\u001B[39;00m\n\u001B[1;32m    163\u001B[0m     \u001B[38;5;66;03m# Please refer to the following PR for further details:\u001B[39;00m\n\u001B[1;32m    164\u001B[0m     \u001B[38;5;66;03m# https://github.com/optuna/optuna/pull/325.\u001B[39;00m\n\u001B[1;32m    165\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m gc_after_trial:\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/optuna/study/_optimize.py:247\u001B[0m, in \u001B[0;36m_run_trial\u001B[0;34m(study, func, catch)\u001B[0m\n\u001B[1;32m    240\u001B[0m         \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mShould not reach.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    242\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\n\u001B[1;32m    243\u001B[0m     frozen_trial\u001B[38;5;241m.\u001B[39mstate \u001B[38;5;241m==\u001B[39m TrialState\u001B[38;5;241m.\u001B[39mFAIL\n\u001B[1;32m    244\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m func_err \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m    245\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(func_err, catch)\n\u001B[1;32m    246\u001B[0m ):\n\u001B[0;32m--> 247\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m func_err\n\u001B[1;32m    248\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m frozen_trial\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/optuna/study/_optimize.py:196\u001B[0m, in \u001B[0;36m_run_trial\u001B[0;34m(study, func, catch)\u001B[0m\n\u001B[1;32m    194\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m get_heartbeat_thread(trial\u001B[38;5;241m.\u001B[39m_trial_id, study\u001B[38;5;241m.\u001B[39m_storage):\n\u001B[1;32m    195\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 196\u001B[0m         value_or_values \u001B[38;5;241m=\u001B[39m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtrial\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    197\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m exceptions\u001B[38;5;241m.\u001B[39mTrialPruned \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m    198\u001B[0m         \u001B[38;5;66;03m# TODO(mamu): Handle multi-objective cases.\u001B[39;00m\n\u001B[1;32m    199\u001B[0m         state \u001B[38;5;241m=\u001B[39m TrialState\u001B[38;5;241m.\u001B[39mPRUNED\n",
      "Cell \u001B[0;32mIn[2], line 77\u001B[0m, in \u001B[0;36mobjective\u001B[0;34m(trial)\u001B[0m\n\u001B[1;32m     75\u001B[0m outputs \u001B[38;5;241m=\u001B[39m model(inputs)\n\u001B[1;32m     76\u001B[0m loss \u001B[38;5;241m=\u001B[39m criterion(outputs, targets)\n\u001B[0;32m---> 77\u001B[0m \u001B[43mloss\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackward\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     78\u001B[0m torch\u001B[38;5;241m.\u001B[39mnn\u001B[38;5;241m.\u001B[39mutils\u001B[38;5;241m.\u001B[39mclip_grad_norm_(model\u001B[38;5;241m.\u001B[39mparameters(), \u001B[38;5;241m5.0\u001B[39m)\n\u001B[1;32m     79\u001B[0m optimizer\u001B[38;5;241m.\u001B[39mstep()\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/torch/_tensor.py:521\u001B[0m, in \u001B[0;36mTensor.backward\u001B[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001B[0m\n\u001B[1;32m    511\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m has_torch_function_unary(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m    512\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m handle_torch_function(\n\u001B[1;32m    513\u001B[0m         Tensor\u001B[38;5;241m.\u001B[39mbackward,\n\u001B[1;32m    514\u001B[0m         (\u001B[38;5;28mself\u001B[39m,),\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    519\u001B[0m         inputs\u001B[38;5;241m=\u001B[39minputs,\n\u001B[1;32m    520\u001B[0m     )\n\u001B[0;32m--> 521\u001B[0m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mautograd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackward\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    522\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mgradient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mretain_graph\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcreate_graph\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minputs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minputs\u001B[49m\n\u001B[1;32m    523\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/torch/autograd/__init__.py:289\u001B[0m, in \u001B[0;36mbackward\u001B[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001B[0m\n\u001B[1;32m    284\u001B[0m     retain_graph \u001B[38;5;241m=\u001B[39m create_graph\n\u001B[1;32m    286\u001B[0m \u001B[38;5;66;03m# The reason we repeat the same comment below is that\u001B[39;00m\n\u001B[1;32m    287\u001B[0m \u001B[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001B[39;00m\n\u001B[1;32m    288\u001B[0m \u001B[38;5;66;03m# calls in the traceback and some print out the last line\u001B[39;00m\n\u001B[0;32m--> 289\u001B[0m \u001B[43m_engine_run_backward\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    290\u001B[0m \u001B[43m    \u001B[49m\u001B[43mtensors\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    291\u001B[0m \u001B[43m    \u001B[49m\u001B[43mgrad_tensors_\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    292\u001B[0m \u001B[43m    \u001B[49m\u001B[43mretain_graph\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    293\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcreate_graph\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    294\u001B[0m \u001B[43m    \u001B[49m\u001B[43minputs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    295\u001B[0m \u001B[43m    \u001B[49m\u001B[43mallow_unreachable\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m    296\u001B[0m \u001B[43m    \u001B[49m\u001B[43maccumulate_grad\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m    297\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/global_env/lib/python3.12/site-packages/torch/autograd/graph.py:768\u001B[0m, in \u001B[0;36m_engine_run_backward\u001B[0;34m(t_outputs, *args, **kwargs)\u001B[0m\n\u001B[1;32m    766\u001B[0m     unregister_hooks \u001B[38;5;241m=\u001B[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001B[1;32m    767\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 768\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mVariable\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_execution_engine\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrun_backward\u001B[49m\u001B[43m(\u001B[49m\u001B[43m  \u001B[49m\u001B[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001B[39;49;00m\n\u001B[1;32m    769\u001B[0m \u001B[43m        \u001B[49m\u001B[43mt_outputs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\n\u001B[1;32m    770\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m  \u001B[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001B[39;00m\n\u001B[1;32m    771\u001B[0m \u001B[38;5;28;01mfinally\u001B[39;00m:\n\u001B[1;32m    772\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m attach_logging_hooks:\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "print(f\"Best parameters found: {study.best_params}\")\n",
    "print(f\"Best validation loss: {study.best_value}\")"
   ],
   "id": "7705d5edb0bcdd8c",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}