Hyperparameter_Tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# 📚 **Install & Import Libraries**"
      ],
      "metadata": {
        "id": "D7V_6OVa0a6l"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install scikit-optimize"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8DVVQxKervLj",
        "outputId": "0be30c6c-ca4c-4755-86d5-a726b190628c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting scikit-optimize\n",
            "  Downloading scikit_optimize-0.9.0-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/100.3 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m81.9/100.3 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.3/100.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.10/dist-packages (from scikit-optimize) (1.3.2)\n",
            "Collecting pyaml>=16.9 (from scikit-optimize)\n",
            "  Downloading pyaml-23.9.7-py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.10/dist-packages (from scikit-optimize) (1.23.5)\n",
            "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.10/dist-packages (from scikit-optimize) (1.11.3)\n",
            "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from scikit-optimize) (1.2.2)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from pyaml>=16.9->scikit-optimize) (6.0.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->scikit-optimize) (3.2.0)\n",
            "Installing collected packages: pyaml, scikit-optimize\n",
            "Successfully installed pyaml-23.9.7 scikit-optimize-0.9.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install hyperopt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EkVfumnjsrkU",
        "outputId": "d922b070-4c3c-4d6f-bf44-c40dd6c337b8"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: hyperopt in /usr/local/lib/python3.10/dist-packages (0.2.7)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from hyperopt) (1.23.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from hyperopt) (1.11.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from hyperopt) (1.16.0)\n",
            "Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.10/dist-packages (from hyperopt) (3.2.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from hyperopt) (0.18.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from hyperopt) (4.66.1)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from hyperopt) (2.2.1)\n",
            "Requirement already satisfied: py4j in /usr/local/lib/python3.10/dist-packages (from hyperopt) (0.10.9.7)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install deap"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0trt_ZM8tCH0",
        "outputId": "b89b382b-1094-4b41-99c0-bbabdd15c108"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting deap\n",
            "  Downloading deap-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (135 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/135.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m \u001b[32m133.1/135.4 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.4/135.4 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from deap) (1.23.5)\n",
            "Installing collected packages: deap\n",
            "Successfully installed deap-1.4.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install hpbandster-sklearn"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P8xDwsOZwRXN",
        "outputId": "781db4aa-397d-4c06-d474-70dcfef845f9"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting hpbandster-sklearn\n",
            "  Downloading hpbandster_sklearn-2.0.2-py3-none-any.whl (27 kB)\n",
            "Requirement already satisfied: scikit-learn>=0.24.0 in /usr/local/lib/python3.10/dist-packages (from hpbandster-sklearn) (1.2.2)\n",
            "Requirement already satisfied: hpbandster in /usr/local/lib/python3.10/dist-packages (from hpbandster-sklearn) (0.7.4)\n",
            "Requirement already satisfied: ConfigSpace in /usr/local/lib/python3.10/dist-packages (from hpbandster-sklearn) (0.7.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from hpbandster-sklearn) (1.3.2)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.0->hpbandster-sklearn) (1.23.5)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.0->hpbandster-sklearn) (1.11.3)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.0->hpbandster-sklearn) (3.2.0)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.10/dist-packages (from ConfigSpace->hpbandster-sklearn) (3.1.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from ConfigSpace->hpbandster-sklearn) (4.5.0)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from ConfigSpace->hpbandster-sklearn) (10.1.0)\n",
            "Requirement already satisfied: Pyro4 in /usr/local/lib/python3.10/dist-packages (from hpbandster->hpbandster-sklearn) (4.82)\n",
            "Requirement already satisfied: serpent in /usr/local/lib/python3.10/dist-packages (from hpbandster->hpbandster-sklearn) (1.41)\n",
            "Requirement already satisfied: statsmodels in /usr/local/lib/python3.10/dist-packages (from hpbandster->hpbandster-sklearn) (0.14.0)\n",
            "Requirement already satisfied: netifaces in /usr/local/lib/python3.10/dist-packages (from hpbandster->hpbandster-sklearn) (0.11.0)\n",
            "Requirement already satisfied: pandas>=1.0 in /usr/local/lib/python3.10/dist-packages (from statsmodels->hpbandster->hpbandster-sklearn) (1.5.3)\n",
            "Requirement already satisfied: patsy>=0.5.2 in /usr/local/lib/python3.10/dist-packages (from statsmodels->hpbandster->hpbandster-sklearn) (0.5.3)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.10/dist-packages (from statsmodels->hpbandster->hpbandster-sklearn) (23.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0->statsmodels->hpbandster->hpbandster-sklearn) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0->statsmodels->hpbandster->hpbandster-sklearn) (2023.3.post1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from patsy>=0.5.2->statsmodels->hpbandster->hpbandster-sklearn) (1.16.0)\n",
            "Installing collected packages: hpbandster-sklearn\n",
            "Successfully installed hpbandster-sklearn-2.0.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install optuna"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t2uUgA33z5ND",
        "outputId": "2ab50385-1244-48be-9266-087cb4e1d4c4"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting optuna\n",
            "  Downloading optuna-3.4.0-py3-none-any.whl (409 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m409.6/409.6 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting alembic>=1.5.0 (from optuna)\n",
            "  Downloading alembic-1.12.1-py3-none-any.whl (226 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m226.8/226.8 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting colorlog (from optuna)\n",
            "  Downloading colorlog-6.7.0-py2.py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from optuna) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from optuna) (23.2)\n",
            "Requirement already satisfied: sqlalchemy>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from optuna) (2.0.23)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from optuna) (4.66.1)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from optuna) (6.0.1)\n",
            "Collecting Mako (from alembic>=1.5.0->optuna)\n",
            "  Downloading Mako-1.3.0-py3-none-any.whl (78 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4 in /usr/local/lib/python3.10/dist-packages (from alembic>=1.5.0->optuna) (4.5.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy>=1.3.0->optuna) (3.0.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from Mako->alembic>=1.5.0->optuna) (2.1.3)\n",
            "Installing collected packages: Mako, colorlog, alembic, optuna\n",
            "Successfully installed Mako-1.3.0 alembic-1.12.1 colorlog-6.7.0 optuna-3.4.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import necessary libraries\n",
        "from sklearn import datasets\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score\n",
        "from sklearn.svm import SVC\n",
        "from skopt import BayesSearchCV\n",
        "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n",
        "from deap import base, creator, tools, algorithms\n",
        "import random\n",
        "from hpbandster_sklearn import HpBandSterSearchCV\n",
        "import optuna"
      ],
      "metadata": {
        "id": "k3M5HeqorJqZ"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the Iris dataset\n",
        "iris = datasets.load_iris()\n",
        "X, y = iris.data, iris.target\n",
        "\n",
        "# Split the dataset into training and testing sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Define the SVM classifier\n",
        "svm = SVC()"
      ],
      "metadata": {
        "id": "NCQNOKUOrMIs"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **1. Grid Search**"
      ],
      "metadata": {
        "id": "2CmOSiNDq_Qv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yo6k45OGq1XS",
        "outputId": "1f27a2b4-0163-4e15-d281-898ab58a56ce"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Hyperparameters: {'C': 1, 'gamma': 1}\n",
            "Accuracy on Test Set: 1.0\n"
          ]
        }
      ],
      "source": [
        "# Define the hyperparameters to tune and their possible values\n",
        "param_grid = {'C': [0.1, 1, 10, 100], 'gamma': [0.01, 0.1, 1, 10]}\n",
        "\n",
        "# Use GridSearchCV to find the best hyperparameters\n",
        "grid_search = GridSearchCV(svm, param_grid, cv=5)\n",
        "grid_search.fit(X_train, y_train)\n",
        "\n",
        "# Print the best hyperparameters\n",
        "print(\"Best Hyperparameters:\", grid_search.best_params_)\n",
        "\n",
        "# Evaluate the model with the best hyperparameters on the test set\n",
        "best_svm = grid_search.best_estimator_\n",
        "accuracy = best_svm.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set:\", accuracy)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **2. Random Search**"
      ],
      "metadata": {
        "id": "tOGYR6cwrU5I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the hyperparameters and their search spaces\n",
        "param_dist = {'C': [0.1, 1, 10, 100], 'gamma': [0.01, 0.1, 1, 10]}\n",
        "\n",
        "# Use RandomizedSearchCV for Random Search\n",
        "random_search = RandomizedSearchCV(svm, param_distributions=param_dist, n_iter=10, cv=5, random_state=42)\n",
        "random_search.fit(X_train, y_train)\n",
        "\n",
        "# Print the best hyperparameters from Random Search\n",
        "print(\"Best Hyperparameters (Random Search):\", random_search.best_params_)\n",
        "\n",
        "# Evaluate the models with the best hyperparameters on the test set\n",
        "best_svm_random = random_search.best_estimator_\n",
        "accuracy_random = best_svm_random.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set (Random Search):\", accuracy_random)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "njGOB9lLrZU_",
        "outputId": "8d51b326-7b3b-45f9-a132-e3368e73f87b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Hyperparameters (Random Search): {'gamma': 0.1, 'C': 1}\n",
            "Accuracy on Test Set (Random Search): 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **3. Bayesian Optimization**"
      ],
      "metadata": {
        "id": "CReh0IN5rlz7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the hyperparameters and their search spaces\n",
        "param_dist = {'C': [0.1, 1, 10, 100], 'gamma': [0.01, 0.1, 1, 10]}\n",
        "\n",
        "# Use BayesSearchCV for Bayesian Optimization\n",
        "bayesian_opt = BayesSearchCV(svm, search_spaces={'C': (0.1, 100), 'gamma': (0.01, 10)}, n_iter=10, cv=5)\n",
        "bayesian_opt.fit(X_train, y_train)\n",
        "\n",
        "# Print the best hyperparameters from Bayesian Optimization\n",
        "print(\"Best Hyperparameters (Bayesian Optimization):\", bayesian_opt.best_params_)\n",
        "\n",
        "best_svm_bayesian = bayesian_opt.best_estimator_\n",
        "accuracy_bayesian = best_svm_bayesian.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set (Bayesian Optimization):\", accuracy_bayesian)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ET-80OUVrpf5",
        "outputId": "674e0bd2-8570-4780-f843-7ea7c340c560"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Hyperparameters (Bayesian Optimization): OrderedDict([('C', 86.43262829720466), ('gamma', 2.039010901369773)])\n",
            "Accuracy on Test Set (Bayesian Optimization): 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **4. Gradient-Based Optimization / Tree-structured Parzen Estimators (TPE)**"
      ],
      "metadata": {
        "id": "cOLqWNV8sbEs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define objective function to minimize (negative accuracy)\n",
        "def objective(params):\n",
        "    clf = SVC(C=params['C'], gamma=params['gamma'])\n",
        "    score = cross_val_score(clf, X_train, y_train, cv=5).mean()\n",
        "    return {'loss': -score, 'status': STATUS_OK}\n",
        "\n",
        "# Define search space for hyperparameters\n",
        "space = {'C': hp.loguniform('C', -3, 2), 'gamma': hp.loguniform('gamma', -3, 2)}\n",
        "\n",
        "# Perform optimization using Tree of Parzen Estimators (TPE)\n",
        "trials = Trials()\n",
        "best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=10, trials=trials)\n",
        "\n",
        "# Print the best hyperparameters\n",
        "print(\"Best Hyperparameters (Gradient-Based Optimization):\", best)\n",
        "\n",
        "# Evaluate the model with the best hyperparameters on the test set\n",
        "best_svm_gradient = SVC(C=best['C'], gamma=best['gamma'])\n",
        "best_svm_gradient.fit(X_train, y_train)\n",
        "accuracy_gradient = best_svm_gradient.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set (Gradient-Based Optimization):\", accuracy_gradient)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9q_6eVIVsbxn",
        "outputId": "2dc7678f-6078-40d6-e4bb-42d1d1bffcc1"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "100%|██████████| 10/10 [00:00<00:00, 68.97trial/s, best loss: -0.9583333333333334]\n",
            "Best Hyperparameters (Gradient-Based Optimization): {'C': 0.8192182137468389, 'gamma': 0.9243140186047749}\n",
            "Accuracy on Test Set (Gradient-Based Optimization): 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **5. Genetic Algorithms**"
      ],
      "metadata": {
        "id": "YhuhcYWMs9JC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define objective function to maximize (accuracy)\n",
        "def objective_model(params):\n",
        "    # The 'gamma' parameter of SVC must be a str among {'scale', 'auto'} or a float in the range [0.0, inf)\n",
        "    clf = SVC(C=max(0.1, params[0]), gamma=params[1])\n",
        "    score = cross_val_score(clf, X_train, y_train, cv=5).mean()\n",
        "    return score,\n",
        "\n",
        "# Define genetic algorithm parameters\n",
        "creator.create(\"FitnessMax\", base.Fitness, weights=(1.0,))\n",
        "creator.create(\"Individual\", list, fitness=creator.FitnessMax)\n",
        "\n",
        "toolbox = base.Toolbox()\n",
        "toolbox.register(\"attr_float\", random.uniform, 0.1, 10)\n",
        "toolbox.register(\"individual\", tools.initRepeat, creator.Individual, toolbox.attr_float, n=2)\n",
        "toolbox.register(\"population\", tools.initRepeat, list, toolbox.individual)\n",
        "\n",
        "toolbox.register(\"evaluate\", objective_model)\n",
        "toolbox.register(\"mate\", tools.cxBlend, alpha=0.5)\n",
        "toolbox.register(\"mutate\", tools.mutGaussian, mu=0, sigma=1, indpb=0.2)\n",
        "toolbox.register(\"select\", tools.selTournament, tournsize=3)\n",
        "\n",
        "# Perform genetic algorithm optimization\n",
        "population = toolbox.population(n=10)\n",
        "algorithms.eaMuPlusLambda(population, toolbox, mu=10, lambda_=50, cxpb=0.7, mutpb=0.2, ngen=10, stats=None, halloffame=None, verbose=True)\n",
        "\n",
        "# Print the best individual's hyperparameters\n",
        "best_individual = tools.selBest(population, k=1)[0]\n",
        "print(\"Best Hyperparameters (Genetic Algorithm):\", best_individual)\n",
        "\n",
        "# Evaluate the model with the best hyperparameters on the test set\n",
        "best_svm_genetic = SVC(C=best_individual[0], gamma=best_individual[1])\n",
        "best_svm_genetic.fit(X_train, y_train)\n",
        "accuracy_genetic = best_svm_genetic.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set (Genetic Algorithm):\", accuracy_genetic)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nn_5Peu-s-V3",
        "outputId": "944bbda4-c791-449b-9c54-695c48c5114e"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/deap/creator.py:185: RuntimeWarning: A class named 'FitnessMax' has already been created and it will be overwritten. Consider deleting previous creation of that class or rename it.\n",
            "  warnings.warn(\"A class named '{0}' has already been created and it \"\n",
            "/usr/local/lib/python3.10/dist-packages/deap/creator.py:185: RuntimeWarning: A class named 'Individual' has already been created and it will be overwritten. Consider deleting previous creation of that class or rename it.\n",
            "  warnings.warn(\"A class named '{0}' has already been created and it \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "gen\tnevals\n",
            "0  \t10    \n",
            "1  \t46    \n",
            "2  \t43    \n",
            "3  \t44    \n",
            "4  \t42    \n",
            "5  \t44    \n",
            "6  \t44    \n",
            "7  \t44    \n",
            "8  \t48    \n",
            "9  \t43    \n",
            "10 \t50    \n",
            "Best Hyperparameters (Genetic Algorithm): [3.97166904498928, 7.492085209589785]\n",
            "Accuracy on Test Set (Genetic Algorithm): 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **6. Hyperband**"
      ],
      "metadata": {
        "id": "JA585PIcwM3c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the hyperparameters to tune and their possible values\n",
        "space = {'C': [0.1, 1, 10, 100], 'gamma': [0.01, 0.1, 1, 10]}\n",
        "\n",
        "# Use GridSearchCV to find the best hyperparameters\n",
        "hb_search = HpBandSterSearchCV(svm, space, cv=5, random_state=0, n_jobs=1, n_iter=10, verbose=1, optimizer = 'hyperband')\n",
        "hb_search.fit(X_train, y_train)\n",
        "\n",
        "# Print the best hyperparameters\n",
        "print(\"Best Hyperparameters:\", hb_search.best_params_)\n",
        "\n",
        "# Evaluate the model with the best hyperparameters on the test set\n",
        "best_svm = SVC(C=hb_search.best_params_['C'], gamma=hb_search.best_params_['gamma'])\n",
        "accuracy = hb_search.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set:\", accuracy)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Db_hX8Uyx_0v",
        "outputId": "2b686d08-552d-42b3-f96d-fb7318bf3524"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WORKER: start listening for jobs\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start listening for jobs\n",
            "HBMASTER: adjusted queue size to (0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:HBMASTER: adjusted queue size to (0, 1)\n",
            "HBMASTER: starting run at 1700250044.600166\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:HBMASTER: starting run at 1700250044.600166\n",
            "WORKER: start processing job (0, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (0, 0, 0)\n",
            "WORKER: registered result for job (0, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (0, 0, 0) with dispatcher\n",
            "WORKER: start processing job (0, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (0, 0, 1)\n",
            "WORKER: registered result for job (0, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (0, 0, 1) with dispatcher\n",
            "WORKER: start processing job (0, 0, 2)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (0, 0, 2)\n",
            "WORKER: registered result for job (0, 0, 2) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (0, 0, 2) with dispatcher\n",
            "WORKER: start processing job (0, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (0, 0, 0)\n",
            "WORKER: registered result for job (0, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (0, 0, 0) with dispatcher\n",
            "WORKER: start processing job (1, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (1, 0, 0)\n",
            "WORKER: registered result for job (1, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (1, 0, 0) with dispatcher\n",
            "WORKER: start processing job (1, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (1, 0, 1)\n",
            "WORKER: registered result for job (1, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (1, 0, 1) with dispatcher\n",
            "WORKER: start processing job (2, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (2, 0, 0)\n",
            "WORKER: registered result for job (2, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (2, 0, 0) with dispatcher\n",
            "WORKER: start processing job (2, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (2, 0, 1)\n",
            "WORKER: registered result for job (2, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (2, 0, 1) with dispatcher\n",
            "WORKER: start processing job (2, 0, 2)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (2, 0, 2)\n",
            "WORKER: registered result for job (2, 0, 2) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (2, 0, 2) with dispatcher\n",
            "WORKER: start processing job (2, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (2, 0, 1)\n",
            "WORKER: registered result for job (2, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (2, 0, 1) with dispatcher\n",
            "WORKER: start processing job (3, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (3, 0, 0)\n",
            "WORKER: registered result for job (3, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (3, 0, 0) with dispatcher\n",
            "WORKER: start processing job (3, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (3, 0, 1)\n",
            "WORKER: registered result for job (3, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (3, 0, 1) with dispatcher\n",
            "WORKER: start processing job (4, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (4, 0, 0)\n",
            "WORKER: registered result for job (4, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (4, 0, 0) with dispatcher\n",
            "WORKER: start processing job (4, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (4, 0, 1)\n",
            "WORKER: registered result for job (4, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (4, 0, 1) with dispatcher\n",
            "WORKER: start processing job (4, 0, 2)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (4, 0, 2)\n",
            "WORKER: registered result for job (4, 0, 2) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (4, 0, 2) with dispatcher\n",
            "WORKER: start processing job (4, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (4, 0, 1)\n",
            "WORKER: registered result for job (4, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (4, 0, 1) with dispatcher\n",
            "WORKER: start processing job (5, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (5, 0, 0)\n",
            "WORKER: registered result for job (5, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (5, 0, 0) with dispatcher\n",
            "WORKER: start processing job (5, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (5, 0, 1)\n",
            "WORKER: registered result for job (5, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (5, 0, 1) with dispatcher\n",
            "WORKER: start processing job (6, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (6, 0, 0)\n",
            "WORKER: registered result for job (6, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (6, 0, 0) with dispatcher\n",
            "WORKER: start processing job (6, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (6, 0, 1)\n",
            "WORKER: registered result for job (6, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (6, 0, 1) with dispatcher\n",
            "WORKER: start processing job (6, 0, 2)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (6, 0, 2)\n",
            "WORKER: registered result for job (6, 0, 2) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (6, 0, 2) with dispatcher\n",
            "WORKER: start processing job (6, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (6, 0, 0)\n",
            "WORKER: registered result for job (6, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (6, 0, 0) with dispatcher\n",
            "WORKER: start processing job (7, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (7, 0, 0)\n",
            "WORKER: registered result for job (7, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (7, 0, 0) with dispatcher\n",
            "WORKER: start processing job (7, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (7, 0, 1)\n",
            "WORKER: registered result for job (7, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (7, 0, 1) with dispatcher\n",
            "WORKER: start processing job (8, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (8, 0, 0)\n",
            "WORKER: registered result for job (8, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (8, 0, 0) with dispatcher\n",
            "WORKER: start processing job (8, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (8, 0, 1)\n",
            "WORKER: registered result for job (8, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (8, 0, 1) with dispatcher\n",
            "WORKER: start processing job (8, 0, 2)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (8, 0, 2)\n",
            "WORKER: registered result for job (8, 0, 2) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (8, 0, 2) with dispatcher\n",
            "WORKER: start processing job (8, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (8, 0, 0)\n",
            "WORKER: registered result for job (8, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (8, 0, 0) with dispatcher\n",
            "WORKER: start processing job (9, 0, 0)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (9, 0, 0)\n",
            "WORKER: registered result for job (9, 0, 0) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (9, 0, 0) with dispatcher\n",
            "WORKER: start processing job (9, 0, 1)\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: start processing job (9, 0, 1)\n",
            "WORKER: registered result for job (9, 0, 1) with dispatcher\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:WORKER: registered result for job (9, 0, 1) with dispatcher\n",
            "\n",
            "Best score: 0.9583333333333334\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:\n",
            "Best score: 0.9583333333333334\n",
            "Best found configuration: {'C': 100, 'gamma': 0.01}\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:Best found configuration: {'C': 100, 'gamma': 0.01}\n",
            "A total of 25 unique configurations where sampled.\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:A total of 25 unique configurations where sampled.\n",
            "A total of 30 runs where executed.\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:A total of 30 runs where executed.\n",
            "Total budget of resource 'n_samples' corresponds to 20.0 full function evaluations.\n",
            "INFO:hpbandster_sklearn.HpBandSterSearchCV:Total budget of resource 'n_samples' corresponds to 20.0 full function evaluations.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Hyperparameters: {'C': 100, 'gamma': 0.01}\n",
            "Accuracy on Test Set: 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **7. Optuna**\n",
        "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAWgAAABMCAYAAABAry1BAAAEDmlDQ1BrQ0dDb2xvclNwYWNlR2VuZXJpY1JHQgAAOI2NVV1oHFUUPpu5syskzoPUpqaSDv41lLRsUtGE2uj+ZbNt3CyTbLRBkMns3Z1pJjPj/KRpKT4UQRDBqOCT4P9bwSchaqvtiy2itFCiBIMo+ND6R6HSFwnruTOzu5O4a73L3PnmnO9+595z7t4LkLgsW5beJQIsGq4t5dPis8fmxMQ6dMF90A190C0rjpUqlSYBG+PCv9rt7yDG3tf2t/f/Z+uuUEcBiN2F2Kw4yiLiZQD+FcWyXYAEQfvICddi+AnEO2ycIOISw7UAVxieD/Cyz5mRMohfRSwoqoz+xNuIB+cj9loEB3Pw2448NaitKSLLRck2q5pOI9O9g/t/tkXda8Tbg0+PszB9FN8DuPaXKnKW4YcQn1Xk3HSIry5ps8UQ/2W5aQnxIwBdu7yFcgrxPsRjVXu8HOh0qao30cArp9SZZxDfg3h1wTzKxu5E/LUxX5wKdX5SnAzmDx4A4OIqLbB69yMesE1pKojLjVdoNsfyiPi45hZmAn3uLWdpOtfQOaVmikEs7ovj8hFWpz7EV6mel0L9Xy23FMYlPYZenAx0yDB1/PX6dledmQjikjkXCxqMJS9WtfFCyH9XtSekEF+2dH+P4tzITduTygGfv58a5VCTH5PtXD7EFZiNyUDBhHnsFTBgE0SQIA9pfFtgo6cKGuhooeilaKH41eDs38Ip+f4At1Rq/sjr6NEwQqb/I/DQqsLvaFUjvAx+eWirddAJZnAj1DFJL0mSg/gcIpPkMBkhoyCSJ8lTZIxk0TpKDjXHliJzZPO50dR5ASNSnzeLvIvod0HG/mdkmOC0z8VKnzcQ2M/Yz2vKldduXjp9bleLu0ZWn7vWc+l0JGcaai10yNrUnXLP/8Jf59ewX+c3Wgz+B34Df+vbVrc16zTMVgp9um9bxEfzPU5kPqUtVWxhs6OiWTVW+gIfywB9uXi7CGcGW/zk98k/kmvJ95IfJn/j3uQ+4c5zn3Kfcd+AyF3gLnJfcl9xH3OfR2rUee80a+6vo7EK5mmXUdyfQlrYLTwoZIU9wsPCZEtP6BWGhAlhL3p2N6sTjRdduwbHsG9kq32sgBepc+xurLPW4T9URpYGJ3ym4+8zA05u44QjST8ZIoVtu3qE7fWmdn5LPdqvgcZz8Ww8BWJ8X3w0PhQ/wnCDGd+LvlHs8dRy6bLLDuKMaZ20tZrqisPJ5ONiCq8yKhYM5cCgKOu66Lsc0aYOtZdo5QCwezI4wm9J/v0X23mlZXOfBjj8Jzv3WrY5D+CsA9D7aMs2gGfjve8ArD6mePZSeCfEYt8CONWDw8FXTxrPqx/r9Vt4biXeANh8vV7/+/16ffMD1N8AuKD/A/8leAvFY9bLAAAAXGVYSWZNTQAqAAAACAAEAQYAAwAAAAEAAgAAARIAAwAAAAEAAQAAASgAAwAAAAEAAgAAh2kABAAAAAEAAAA+AAAAAAACoAIABAAAAAEAAAFooAMABAAAAAEAAABMAAAAAHYuy50AAAILaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPHRpZmY6UmVzb2x1dGlvblVuaXQ+MjwvdGlmZjpSZXNvbHV0aW9uVW5pdD4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgICAgPHRpZmY6Q29tcHJlc3Npb24+MTwvdGlmZjpDb21wcmVzc2lvbj4KICAgICAgICAgPHRpZmY6UGhvdG9tZXRyaWNJbnRlcnByZXRhdGlvbj4yPC90aWZmOlBob3RvbWV0cmljSW50ZXJwcmV0YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgqWqErQAAAsw0lEQVR4Ae1dCZgVxbWu7r7LzJ2NGRUQFRcw6sM96suLRpOACghGYQZUFLcgGleCmqgxGhMxLqjgirgLLjPiBiqoGPcX43PXuCGfCwIuzDD73KW733+qb/Xt7c7cYebemTFV3/R01+lTVafO7f779KlT1QrrRyl21B3D2hLtQ1nK+ClLJvZhirIr09lgpphVzGRRLqrCmkBfwwxzNYuE32GG8ipTo1/WFFV+U1c3Re9H3ZGiSA1IDUgN9EgDSo9K96DwqJrayKpo0/D4ho3nAWR/zTR1G8ZMC4S7W69pMAB2CsD9PYD8JVYavXSUOmT1h3VTEt2tSvJLDUgNSA30Fw0UHqDHXVfNFGMWY8rPmeppXvHks2nJNDNnDMcxUQmsKRlgUtX/YyF1Dls66zGLKP9LDUgNSA0MHA3kiIg97NDkG3ZmrfEbAJhjeE1OYCZQVtP1dxegCYuzgXUGqKlyk4W0xRUVQ85qvH9aQ7o1uZMakBqQGujXGsgvQB85byxrSz4M90WJbS0LQHaCsRewhcoE+Dp5BY14nNYz0Z2A7TrHXSDpWtVVLFI6mi2d+ZVoRu6lBqQGpAb6owbyA9CTbtybtXU8C6u1infaC8oEyE7QdWpGURoBtB1A2yaY1uuYYTSA1/Ilm2YFrO1hKIy9Ug7ruQybsL8zgO0E62xArSjPDRte9pu1t81sczYvj6UGpAakBvqLBnoVoGtqayN1d339CizmfXkHCYQ1RxN+YDYAxktYKnV/tKLsg5Hx8q+6PbA3ccFwFur4L5bQp7Gi8DE2YAtg1mFZC6ubaNaAotB/ioXDJ7Nl59wrCHIvNSA1IDXQXzTgQM8eijRu7tWwdM/lrgwnMHtB2TQ/YAnj6lHFwx7sNhjnIGJNTa1W1752GouofwL7jtwNEmRRu4Ba+bjm5OF71E2RUR85qFiySA1IDRRIAz0H6Jo7t2DN9R9jALAqKzibyreIbb6YPXHWwgL1izfDwTq5bi4yp2MLMWFNC+vaBdIYSCwq2oM9fub7hZRRtiU1IDUgNZBNAz0D6MPm/4KZ+rPwC1vxy+TO4P7m9N40P0P+cLbkzI+zCVAIOsVcf9j81elwgVwFd4cF1NSw3+WB0LzQFPb0rIcLIZdsQ2pAakBqoDMNbDpAj5v7N4DvRS6rWbgzDNYG3/LPumuN/vzqFYNfe/er0UzHTMKQuhfT2CCX8CnWigfC+2j3vSjTVsYX/XaV63wXGQ7UbWvuYGHtWJc17bak6SHze7b83Ou6qE6elhqQGpAayKsGNg2gx14DqxkxzQTIXqs5aZ6Ssyvj2Funs0hoGiI1DsLAIqzwtDjZIjzEYB+pRLgpDEwGN8wXAdz31MSrFucy3Tt67O0j463t77KUHuMDiKIuJ1BLkM7rhScrlxqQGuhaA90H6HFz/wULc1+f5czUl0Zpgw/ubOCP+4SjjVMwk/BaFtGG+gCZAD+XZANqehahM68b6yMVZaclbjym69mDR8xfAWv9EBuknQBNcqihGunuyOUHkTxSA1ID+dBAjohIkWqmooy/9nUbnJ2Wc8I4vrNQtYpjFlc2htpuQdlqFlI03hHhq6aM02LuCqQFGFM5YVETzXnMzxlxFon8uaYhNrdTq/rw+TMQenebDdJUVrfXXEqx8s2GsbqTvieyTFIDUgNSA4XUQM4AzZxuDQJUK745xUx172y+Zm4xFzcutoFZALHwVVNPnYCssCSQ9kOWYg8Ux0KvGj90fKullLY2XUnGNDNsbFlUpLckNtNV9jNDU06Ca2MXZirWACUPpfMANQdtM85Syols0YwHsiqWBjtDxkvcL83B3jnzUGlFCF6VDMHLqj15QmpAaiBPGsgNoMfOnQcgPcvl1lDUtoriqq2zrm1xyt3ns3hqDreYvcDsBeWkfkWs2VzQ9uDJa7vdTwrzG8TOYJp2EaxgzW0J09xvJAJdw/hiVEflTlldMDT7UU+9GQjSmvoae2r2/lZl8r/UgNSA1EBhNNA1QPPV58w6Lo6GWdXctWHWD9uieJugadJVZy4qr9/Y+i6meW/HF0FSUEYAMgG1fWy+E21K1XQ3EqMztYRn3LVvUmN1sKq35UCtpy1hsqTxB38zFT+VLZq5ILAesqRVsqTTwJ5xdTAMZh4pV8UL1JokSg1IDeRJA50CNPcdN6yrZwJkQwBbWM41oSHlgX7d4xccBgR/HMCs2dY2AbITmE32eWxj6sBNspZzVELxOYv2b+9IrkSoX5Rbz3zwD4WFr1pXlrHFMyYGVjfh+ulA83s4LzFkQNqsKd82HNjvwIokUWpAakBqoGcaAOJmT4316z60wdnyOWM+Xsl2gSA17dabwLvMBc7c4k5b0PAvRyLaUezW40fmE5ypN+3XH/vqqA0l5XhI4Gsr9IBIyyAeFpo5gU1b8M/AntO6HKa5yLb0hcWPWuqavr4wsIwkSg1IDUgN5EED2S1oMRGFGrVdG2wce/Ts5T45pi1cionUEzioEaARv9tqbmYb9BF9Eg1x7MKjWSx0v209c390ejDRUF7G4OGBvv4QYew1a+Gm2ZKXc1jRbMV5nT7UAuuSRKkBqQGpgU3QQDBA08BbS/13tuVphdTNY4+dfY6vjeMWrsQ3A+mTVZZ/WQAz5XlSnme3TB/tKxdAoKiPf+5s7N2SYnurirK7qag7CDZTN74Lq9oHRVHtrf/eJflyd6IqSmbV7tba1voOBgBVexCRgJp8zYr6PLtvhk8+/n3E1uZvOI8zPlpOYBE/idxLDUgN5FkDwQBN1qOqbmm5B8CisnXs8XOwDrMneS1nLzibyi3s1um/85RyZXe4dOnw5kTH7zQAn6Zh8U+qA0nBHrHXLt700B1w1iTXcDKu6w9UFsUuXn3pxK9cjAEZPnuwRPuEg7Q9CIgaOVCzueyBmef6ih0x73osY3o2p9tWNMLuyodXBLp5fBVIgtSA1IDUwKZrwA/QFMlgJF5yWs8VJVtU+cLpjl5wDQsrs7nlTO0LC1pYzl2AM1m1RUXskaKwOhLgjOAQxeH2VbiHxNktwmr6zCAljqkE0tgMYGxCN1ZVRItHdwXUPkuagFq4PNqSP2WPnPGWs00ex93+TYrzOK3o0qJd+noBKCHnU089FW1pafkZ8udBabthTx8xCPPzioK4ctaMPK3Qd3Vpaek/x48fH+fnCvzv4Ycf3h1NTs61WfwyumKaa/GgfqeoqOjzCRMm5P1TZXV1dYegvbyFU1ZXV1+Sa/9z4at9+OE/4z3VmviVLoDpZKHy0tI5hx56aGtOddTWaoqmXQZdp5z8VE/N5MkXOWn5OK6trZ2rqmqLs25quygSWTBxYteGlyj30EMPXalpWofI0x7XUP2U6up5Tlohj5ctW1bZ1tFxFO6/GWiXDNwINro3SddxgO8P2C+urKy8bcyYMRtw7EshH0VPPAbr2XJXcNeG+owXnHk4Wzw520ZR8juLATheIdwaWSxnWtT/H++w16MhdU8nMGsoD7cGL01VBSXDVDhI0xWpQvsa8riLIa46sj0V/7L8/Idebbpq6gFBZYnWet2U99WZd51jqMp8+vW4/LwtAH9R+EVQyohPJG4lj5t7Ofgu4g8sLPvBE31fkbGDBV9f7QEotS2trfgIb1pxQhB3thLnh+PUYeBlKFNXU1MzRbAWao+3od2g5T93pz16HNNbVFt7u1lbV9eKn+oR5E+aMmWKPdWzO/V1yauqBxmGkZ+BYNNsR/u9CtBQzjm4jCtd/YK+GhsbbwYtJ4DGQzuE6+IC6Np915kmgWbeARrX5iwYXt62WXtHx8loH8tB5JYUVT3PWw8q/RClCw7QjzzyyOBUKvUcrttd0b67b47uQOdDkJ1T39AwBw/bT4uj0YO9DyUgsSOR9aww6zNVRMaNPmxw9EgHB6MV4ZLtiX9xmhOYCRQskP46m895iwvr/v7Ke2o8Ftb2jOAhEAZ/GA+BMKzuCLYQ8pEQbcgHbjiX5qMyvCyVobpQT0lE23+zC+tMCrNzyuw8NhaciI/XIrqDEslrb6yU0VuBJ40q3eYyKMKyOomXkmn+kqxrK1P4/7BG96l7+OEkfuAatJ71AgiSjMoA7AyqI+h8P6VRH0sh+3Rck0n0/cZ+KqcUq/c0MATW9UG9V11haoL9c0FK17/FdUpvs7nfm6b5EzyUvvRe226ATiVuc3VDUeu8k1E+jDS8xHmElSaAmYgIpRuWat/ZVUc6s/mFtW8DgP9AQEqWM4ExgawA5VAaqDWExMHBgQAK90Y0OsfBnIM3AXOmDqqT6iZaWUn0lbLzHzo+SA5Ou/WEA1AZ2dDupJizh52yIOYk8pmHWCnPScNxqC7x7X95aAXJ0kULoHoDlqT/7Sd3CRRYG28MxBsAXcTbuHk6LI5Pcu+u5BygGnh6IMlN7hrcm3N6IjO/tmtrl4k6bIDmFqHKdnb6nmPRmCtqg1wbwMn/zlid6QdEGqSLI9qvvIBO9VZdULsurKp7CouX7wmAAaYExAS8fA+bVNWUFtVkr8AonhQxI0NKN9u6smpwpLRi8NaVxWpkSHlMGw/XxiuaqsXBi5nk5P626hIgTXtY4HdXXvDgTNFR395ULqA3BN4XPChEn9a2mHf7eEur/sTPi3hqYkgkpvv48kwgfzNkfgE/Yq+0hNfC58hP1iuVFboSWBx4C1hR6GZlewXUgKIU4zfOedyigJL5muJvpBTh1RtJUQ4T/batsLqWr//uqlvV2rwTSpIdiScsoHIDMy9nmq/SBBFXHcisHMnWRDV1qLCcBZiSv5lAmVvGAGaMwdV/19w69uu/HvWGtw5Pnp6qTxPwv75r+JeDoupzIdjCwGiOt9xNnHY+tKfUWzEw+Br5nj11MHbbCVexmXddAX6gcxrwqBLdmExWtOtBQ6vZ/eZ6fGHc3NKuB75QHJ9n5wtw0Nrael8uzQiXdFdAjvMhDGLQNP4xudTb33jwFnAI3RgYfPu//iablKd3NIAxgbtR05LeqS1/teBafKar2sV9SXxd3puMPQi2sA3QKHESHxwUrST108Qh35N/2oTTnrCZNgIzSsIKveV43+AcuTUAxENpzFFYt2Q1EziTqwJhdahSaYmVRA986bSD37YqzO0/H8CrYyu/ggQH3fj0vA1NibNIJJqNTgOAOszwCB4ALGq+BzAPBYfFqdeCEx+6RRkap6CIDk1V1zakZoDiHlyI63UQ/CxbOtMst48LdADpqrM1hR9/NX7D0/DDrwRg2e4bANjB+I1uAX2HwLKmOXrFihUluY76B9bRAyK0fi9kp0FXX4LMO2E7GvJjCYHghBuDbt5tg892jwrfyX1o7/VspXAuBVkfgjylPh7T/I2PJgk91wB0jUG34ydNmnRPzyvLTw1pV2H2N1FFOQfhZjdicN4e3Ma1FKpbsmQGrv/5dOyVjGiITJlinaCJKU0brAYI5QCgFWVbLG10lqqK1nJDkwCZNkrESxuF1HlS+R/rLgGguCI1CJhpE+6MkqLQ/BfPGGfFGXvKdydLdRx0w1PLNzQnn6JyuGn5A8GAaOS9WPkTRj6dcd46K5pDcxrLjN9jOVJwQVqRwqG/4dAF0LFBg65sa2o8yxHNEeILQ91wbJMols89vfKgX2nFe1oyzWvx48/2UEWWnuwjyD+G3y3wFaypqelo8NwuChRyDwtp/dSpU7NZwERfjItVweDJOhzTqLc3DadRc9zA33lPdDePB9vHKENb1oQBVgqR8iVEQ6zoqxBGnzA/MgIiIm5B9NcD3ZmcVmAV/CWoPTzMU0XR6AhvZAbx0jnsboFxdC+ibr4HodhbB3hmk+3IWFPDqRw5BQcmprhC6yYuGI6g46EcjAVEOIC6ZkPRmaIo7Qm4MGXvUjJgM35n+JmRJ2uaADplqCf2BjiLdl88c/zTsSJ1Ej0vyI0iLHaSwdDNsYfcu6JE8Io976PCvuYPHEd/ANal1AfBR3vu7qHKKaX39d9sHGkR8v8fIDU1qBX8iKsRdpYNnO0ixEO8NsFxgLjToxzZfncIuc2K8vIREMzxFM2Iqev6npmcPPrRaQDgNVVV8x/yt6mKU5SDgoriYv1rEDg7eenNNRQKub0VgkFR9rMAmsKXnElRrTA6QStlmVd7AjJngu/Z6z5IFYefEiBJrGQ18z3QkgYEDaZc9M75h93NiZ38++Vdbw86ZOGKk8mFcciCFVfASh7HBzOzlPnX7yc+Wqxpr/DBxzSIElCTLK9/tHFRYDEdK9eJRGIK8G1q/YUg23uDxe1jOkgmh7vy+c0ER42Y5p9zbRbdOz+Q1zR3DaT3IyJ3wSDyJEgkAHjWsMogfknr9xrwPYhBuJAPkvdP0X3ykphVgwbdlIu4afdNYB1pgHYMflGNun63u2JjlgAuTicQS29h3Zzl5KU4aVVV9ifLlaxlinUmfObRFjwKgj331rkTOg1FIUAeu/C59uLUDw1Yt/T2WDR0FlzDf4wVhZ9qOXhQ4tDbnn05yCImOXbYoX00dYraJHC2/N/Im8oRJJtTVn68PnmH1Zd0ISJS3zr0gMkcxpu8jPinaFuIw3zv8ettHtRGSUnJw0H0INrkyZMfD6KDFuQ6yMLah2RVvTuodVjQw4LokjYwNYC77xSv5OSTbW1rm+ul93Wexm+yyNCSbXZgFv7WILpKiwLhzTHmOllUYUdjBIKaYMZs6+TCE11WzdoR5kJxmgCSEreg6VhR4iO3bT9MnPfuD7zl2b3G3faMToCMlTiKvOd5XkFwmMIO0DqUptE3PvcTLw/5qcIR7TnL3+0++8XWbCc3BfMvt8R0Sxoc5COfjrMR7RBHzjr0ugiSyW19PPkjBF4I48aNS+TaJILoqaO0uRIsUFe+32YM45t+K5sUrNc0UFxcTAO/G70VAqRP7wQQvewFyWN6OeFU0A1U3x0BUMGXQfxqW3Mr+fZg5sLuJMsRN+uwyqSN5h/q3+0KmmVpe29kla/x4Ko3ZSoTvO4Nqpb8zojYODWbo/9/5j91UEmIvWW35ao1IAOZIhHjoyCXB1a8+xu1SUk8JEimSNQ816Jm/vNwulB6pmCGjLcI+Ny9KWV+yklpdaBDZV4WmZcakBromQba2tpipmFMC6oFA9p3BdH7Gw1GT+BgcjY5YTUFGloq0xP7eAtVNlRmKo+Fgv2TAGtVN+90lqWBNYOZfKo4uRaEq0H4oN8+b/w9Tn5xTOtzVBRHlou82CMY4/1WLD1KGwbyfxB0ew+QbhhT6Rs8a3q55X+pbW5FYy9AGvmf22XdB+vtrPchZJ/AQTS8nj/EaOEkmQquAVz08oFYcK0XvkGsrbMZInsoIutbb+sAshpyd3jpP9Y8gi0wM9CTMBplx+thNaLd7dNpq5SsbErFsZIX+EH6X1tIcflJna/ONPsPed/rNRVt2li50uvSQEjZk8tPOXh3io+mrezZ+qGI8/Uho6aYxzhloGPxYVgS0ylDwnCsM+IqpKZEn1zkbBlhQSvBT71sxSS9xxrwu51Qpapp7sHbHjcjK+hLDQCA+dsrbt/pQXLAVfdYEP3HSIOhqezcaccM3T0QJnwHKNS6piVjeSJvJJXdaT1nGiD0JjOkPO2lUZ4G+/CUcFu2AGKzyB1WRpEicJF86K9DqfDT3BQxUOimOnKqSUtyOgg5HoZCG3LklGy9oAH8QscFVQP6Z0F0SRuYGkBs/CCSHLH9z2Dns6JhTB2GuP6tBmbvuic1ViYyt+m0iKZaAdQOYBb8o1ipay5LNKZlBcuyqPq2KOfc603aVlA4eSTshC+pJJ6ZHrSerZKZap3mBpa/Yhfs5ADonj0ZeHXuzLUhSprmZuKQ71PGKldeZvKmAUzUWYHKAx793JX2j7w1LCsuuAawPrV9n8EtOSFIAFNVX/TQA68ND8+Ay4bg4oh6pf433LagWW4O3Wjn0/Eo0oFAmk+HJl2YBrlCnCZtvE1vDJdoWJXar6sElvX1tpMtr5hGhCxrJ0hThAc8JC4XCpXXk9pd3noofvqHtV/aZFrUn6z6sJrVJUGL3Nv8WQ9Sye35/HTBoGr+p7s4J/c5awDWEF1vvlRRUVHU3Ny8C03nxjbcx2ARzFgs9kmWc5I8ADWAKffcgibRaZ0VzN6kiA6bRnTwjMBDe3ecf4/y/TBt1Z0VFwFP28K14+tGoLO9obKB6AnOrWrf25FZVF5gr4LFLjxp683L3vq2PRiH2zvM7cH+oqcIa2/XN0YiWPrTaUXjWE1ob4KXu1/GLHj2mLBi3pdp3KqFBhFXnjHmU2+dbQ1rfmVZxBkRyYKG6C1eXooCqTNbN/fye/l43mQ7uOhq9EtXXma6rQFaaB2FaPOlxiY+ix4/WyfJNF+SU6w70c8APAWgchlisKIPwgP6XW9XQCMXiD/aysvYB3n0gQxfXxhwNlEySOXmoMWX+V3gJLe2lGasak0loPQn09RerWzI8IFj44ZGetIhQg2GNqxW8USAIkE1xtA5b3rtvEO/M1XN50OEFb3TuIXPmhQXDct3sQvAUQn80R1GkfE/3voon0yYF1Gb1KyQgej4oZfQ3plWDk1Y8cVcRucZxQfmrDi0N+ewxirNmqJKGZfrVNmmHRMAd7Z1Wius7Kxx9Z0WlCf7rQYwsO8C6LSVHOQiHZCL+ndH8RifU77zFmhs3JAJrWvv8CvGQj62sSnkeu2ox8JBQPx6nU/8ACSjYoF7eIcd521H5MvKN+xOgCvyrr3Tsk6foJA7I6pv7nSBiDIUsqcz86ckAoE0yUDfLqTU3O75IAFo7RrbDWif8YELgQ3jBSrjSkmEEFLFlstc905xd/HKTN41ENK0E/pqFb68d+4/uwFr3Muhg1hx8WhkrRvZQcehFXxgfVLMfeZHkAMwmR/zfpBVSOBDABU2J9t9W3Lmx6ARzmXQNn0yWab6RlIjmvkgnRagyEESVcZTRpV3AaJ0NYwmr3SENtsyMNZZMNEecrTFU/OXnzJmiyBwJpbPvii+k6x3y2rPyEEPjYlrwj6XRHscq9kFpZBS5yS7ZOfqUOQAoVNBBT6GyX1hf16CssDq+FE1h7de98xm9I4+Gow34Gd9HcVCSrQsJ96wfZ4AH+8AJJCL43Wf3Koy3kdzEYC4ALykqk5xkZEZ+plir83Bv7rNWWHJAvi336riSS+/yL9w4l4bCXjbOpLjUfUr3CdsBTLTM+P9pMGmlQ7aWNzZCng0OIgQnWk2ONMDB4lcLqZiPhZo8ZrmAfzBlOblBei4UXmBH6f/1X+2/mRnHnq7zZXPfybZ0yYQtkSuhIGdYCnhRt0DfbliYHckf9JjokdZrrV3dHQER16JN8lcK+pdvtKg6jAYfDjo1k3tYMA4xh0ACZ/x5WAZsIchFit6kbW2uztgmNvTGhxiwge8t9cBxGZbs+igH77QMooobCb+uwZ4qAx9XVvHgkmEjJhIwr/AjZuKxQ3jgP2uXXokrTrnbjCTo2VDkQuMmc5w+Y/ItbH6izWfW9YzQBkgS2Z/ktAZSWtJHs8PHP+KZi3eoaMtOdgmiYuS3hiWnvqVTaeDWOQklkjP3yEAj5T7/Nku/t7PNKPKSm+1CNofBlpOvvAnn3ySblwfSMNi8fvbvQ3lMe+cTBTQTBzTfj+FjE/jBr2pq+UbA8r/mEmB1wQWj6LJZdabcRe9x1rLu8AY8l0ToPXYIOii6c5O+1wcxEyDwbje78PdN91TmAB9Nw+tT7NQ6erqyZNH5iIEeOmr9W+Bdy8vf4hpsa/hiSUUy/xImErJ1+BgWBsDKWaGrm1TkrO5pcmt2nQ1plHi+zwUToXak+P1kkijhtg2sqIxi5ADJkXpdSSMR7DI0U5B0RfpWru9ozCtK78oXpc0jCrLpUIWe8a9geaXk3/cW3FHh341med8o5OkBSpomIucvLyP3/JPqKfJSitbOtMN4M4C+ThWlM8g53Bv1QCuS0Gb4aUH5fHJrKtgbfhPmea//cTCUAC+V02ZOvUPhWntx9UKbqcXcLV6wQrWiDYHPa3NqbeqerF9/TsKYI3weke2oIcALGvgPqBVXO8n4eFBE5YyeEV8AQveBxQvKImAt6cNqoy+t8eUNg5MVBsHKNob9iskX6xeU76g0/aPyYGNqWtZ9AxOd/wjMETc3KUimsNydVigicXz2fctrZ8ErUTnqCLnQ7KcAc7fcXBG3RyY8Y+3CROaZNjyU//niPgiS4Z+hN1valFY0G3axU4B1q6PT+PmOFVOSWHPWweF+4+4z5VBrWGa8wm5fPiVvjpCvEF14KJ/IYguaf1eA4uDJEzHCO8TdM5Jo2sC1/yvnTRxDPS7Vxz3pz0+PKED+G7uTzLlUxZhTsGKRkqPBVpA5fnh6uPTbTAjIONAjr1qXhYkYNPfa/6CelYRUPKNQBNbiooCOAmk97zqyROCyuZK2+HSpcNXfxXbIMCZ6k7qBm+HXBvUbnNr/KiMqyZT85Kq1t/BYrb6L/pCpxX2hc86Vgzqi1WYdFQSPT9TU2GOKisrA33eANcQVv/6it9sWUShcyld/5x4g1iwZOJdQXRJ698awLKcb0DC9IXplhXE/8VEjqzLONBDHe6Nz1HKbYmmqymORu9211i4HKz3cGetlcRi9HWgzIJunTEP8HPWDRsOXYGvg9zj6gvdzEfOG8sePXs5pz951svsuAXrAcxDeZ6m5nGgVqKRsxdNTcw79iFXeWSGrmKj1o3kzvuh1txEA8XpesDnr/h1kbprryuXXlpWXnRkdz4a+/OrVwyO64klcIUfkEzp9EkrDvw2OKcfCvB5PxYkF8lpKPCrE9jawItLmoC6LZmJYCHG39wA35a+JR3yZLDWGnXIZ64QD3Euj3ta/Bt+qgY04fND4/WuFAD8Lb7btxrnn0YvPiRRoOlR6OFYnBtB+SzJTH+LL8tpSe6vGqDIBrj3PsDv7/O/0sMY18FHuGa+hRtpKaZPc3eluCba2ttHoFy2rn3Vn3393Be9ZMlVuHcvzNaBHwudA3TV9ps/Vv/pOvyejqcp7mz4su7GfwuQccDaEr9ixZGP6NBOALVEnIO7D6DJcoUrYeuVI8014B+q4bMq5JURcdI0F9FQjG2bmtrf2uuqZXGEF19WWhJ9esjzDe85Iy5wsSljblq5Y0NHR7WCwcp2I1GFqAzgKYDZ6daALEnQyELH43V54xVTj7TldB7MvId8zxoHZAJlAdKK+il75Ax+IdvsGnuSOYdLFOUOp2w2XyEOTPNI3FQvZGsKetoB504X590/qKC697hFx7opMjeQNIDv2Y2hh3MnMg/BuMNvxTWeyzWBAX23kdJJ5X11qnrSpEswYDgL90PggGJfydXb7fJXfGsATf2cAxZZlQK0DH2IZUGmm6WYaKa+zM/DlcD3dMrAeh6n3hvoFyIwG72KbQ0wfacjZbAE0JMsXdpSaIdv5I4w9KihG5c3N3W8tWqf4hQAu0Nse1+1zKhvbf/E1M3L4bao4mWpLvg0qLyoj8CZXBtx01xef3nNuCBl8XhmVZnNL1gBzNRf2jYaB7vKTLpxb6D9NpyXzkM3NWXbBMdNuwrmJwP/24u4eZ7ptdpN88X0imG9VqWsqLAaoK+Z43X/pl5rFcv80voXvVbfJlSEO03rqhi5OLJ+bLWrwgPovOWDJYHD2l+53AREIhGAaeb9Ikv7YTEjY3HRedo4wOmnlcyq9b1qURkC6R/mTNkLrFeSdcuBFGW4nxj7BMCawDYB0LXBNqVHCbThwojaNM5nAXsC5cgSpzqStEfllt/ZuDkbOJMs9Un9LXxzUbFkTstPJ1LGMp/vWU8t46OMdJ6SorzWZ9azJQHDdwXHwkdHvsOeJUX5FBWM7lklsnR/0EBNdfUZuDYDB5G7Ix9Aj74QP6E7ZfqSNz1RqU9DRPPd/wxALzvnXjQGxKL/aesYOwDZrk4rmn8iSmMTbIADOAqQbm1vfYPV3LkFryPg3/dzav7YuKpxq46ksYrANGNNE7gSOKctatRJlrGwkIWlTUDOATm9jxNgc4scYK2zJAYED9gwZ6r9iu8T4bR7l0DWEVx2erBQIvmx1eibHWER0v/HXVeNi35L0TeuE630aBdPH2RwE5lTq6tHwpK+bJObN81rp1RX70Qj4ptchyzYrzSA33MM3FWb7JOFn/p2AH1nYxX9qr9CGPT5VHH8Y9xnAJp6pyoLXZ0kEKNN1V9wfTz2nplPYjHSuZyXzhPI8T1cHVXKeoobdtXjyFDI3oYrpuwYUlL7EVDHBVCjDgJiTAlPg3XQPu3SAJ8TmIHT56+/vDrSfv2xrzqach/OuPMvcFdM4nK6ZIbszYndnZYxD8GLqg9wXrsW5Q2fhW2fK/wBXkMvKS0pKaIbC6Ad70oC4sHFfG9FeXkpgHl2V/zy/MDTAM2uBMiGcd1eC+m7tCzpmqDrh64jfGJqxsDrMV/UfzHkpsHzPkn4RmKX7picBMsSYYV7NpP47MGmLztAwWdR0K5YpD8EHI+G72UP/e74DDeOpi1cChfIBM5HI8Ia+KiMyuLDMJDHrW1XAX+G1n1+8+ONV2EccFJxWOUDkuIbgl5uCpujxN0khvFOa1vijAnfRP7pBFdvGZ6nQUGGD8YKYAai80R7wzyVLZq5wFVu0vz3Qd+VuzeIh94oSqsGWzHjLs5+k6FR+yVLloykWWQYsedRJ4iH/QE34Ztwi6zCvk/DkmigF4M6boMA2gOoGJDN+mH7jTazC5Jt7er++DZC1wTC6YbF4/F9cQVvTb3qT9cEyUMy4rpw/f51CJHq8p6mwumU7Tfpbj2ivu7us7XfnWuCjMKaGn/LLoDmpw+95kGobaoFumnApZA6AmBT3Z09fub7rmqOXfgSYqF/YYMz8XGQVpsrmkLbNt4/LeenW8UxiyvLR2gjG5PaeEUx9sTns7YDppaiyhb4m9/D1fVae4f+7Ig14TVBsc0uuUSG3BrCciZLnwMu9nQcT/kfOjU3nc2Sqes5OAtAN43b2NPn0rR2maQGpAakBgqmAaCpO/FpzV82NYKKaeBp650Al6xoil577OywuwRyxy1cCfD8dQBI65GoNi1bLLKvnl4kENg3lqdoJT6aLZV2w5DFjEY4SCvPs/tmjHY1STHPIfYOYvfwtfN0GcOoH1U2fMucHwiuCmVGakBqQGpg0zXge+Xkbolw+GReJb3ai0SARaA9aT6N/rsTAZ1u3MytUrJMBSAahpboSD3ITr37Fe7XdZfKXw4hf40V+gY3OBPgokluRSvLvOAcO+qOYRycdUPlfEK6kuj+EpyFMuReakBqoJAa8FnQduOHzn0HaLZHoKuDKY+zx85yRz1QwcPm/4JtHvsHfNCW6S180uT2gCMXT4NzJteX3Nwd/5ItTxcH5D//qKp1pqFq89CWFUbndGdQeQLnlH4zW3yqK9KDvzVsSDTjIYMPGIDHtp7N+Wz57LO7aFqelhqQGpAayIsGsgJ0F64OEmYe3B3neKWiiSD1G1vfhbtjO+63JveI8EvTXjERcaDdWdGkXdQd/7S3HZEny7etMnwfgPUgGtrkICwseOfeVHTW1DrBnrqeroDLu6ZhHZ4fMTc4G+vY8nOHiXbkXmpAakBqoNAayArQXBCyiFPxl/ix8EdzqxgU2htmIEhzfhpsK4lcb4MzEZ1ArWKit2muRuT1o0Wx0C0d101bzcvl8C9yxv1HJPTUcSh/MOosxSCg1Q8nIFM9ZDHzzVjPOsK7+6IwKGY71fqFD5xRtGrE4IqgJUpzEE+ySA1IDUgN9IoGOgdoauLQuRcD5S5zuTqIToOGZB0jeoo9euYUInkTD9tTf1jIisPTbXAmJgHU4pj2BNiUTPN7RIWsRcSItR6taURRYCjcJsPRVno1trQLw+LnxWzLmebakK+ZwJqsZl0/3RdGRyVoGrdivuFza+h6Covxj+hPMc9WB+V/qQGpgf80DXQN0KSRoNA7AlkKvyOQZuwD9shZgdO86SRPx9xay6IhK9JPuD3ohFXeAm2L0/ov6IJGlrBIBL4iCbqwnomOFUcxf/x69sDMcwWbaz9x3h9ZRL2CgziVJ5+zKF8U+alvwSRXYZmRGpAakBoojAZyA2iSZew1TwDEJrosaQHSdD6kNbI43AidfGmE+7Xb2HHwFV8KfmuVPAHEVJdIgibyYi/AmPICpJ20lL4eBvSlo+KVdwVFXvD2f4i/Dgt+VxuQneCshA5ktKyqTFIDUgNSA/1AAw5UzEGabJY0vB3cbaGptDjGRWzp2X/vqjYa3OuoUCcbzYk5LKrRN8WsxMGZLGSvaGmr2RH5xwuYRpyFQ1ey9ck7Ons4MFpbIxZ6CMCMMLp0XZhWzhPlo9o47wCidVL+lxqQGpAa6BsNeFGwaymy+aSFNc0BVvmaJSMHdAqY3pZowC6W3I9FwvuxuL4TfMPbwS9dCYvdCtlTzEasy7wWA4KfYIDyzShT3ogv+u0qbzW+/MQFw1k4ga+Es8yyoU6rmWEdi3BoX98MSV9FkiA1IDUgNVBYDXQfoEk+K7rjeYAnPoxC5jMSAbMTpDFXG4N0r+CjtNW+6AmrRH7/0yAgMx6ETDu63CECnKn1lF5ftePQ7WW0Rn5/Clm71IDUwKZpYNMAGm3x+OFPv30TERcjA0Ga5LFAG0Bt/jsaix2Zk8W7af3gpWi2Yl1q3cGYov4g2qzgRDH4x/egiDw+A4Q458N70JwsKjUgNSA1kFcNbDJA21JRREQCfmQsUNw1UCvfs/b4fcO2Kb0kl5Xu7DY6OeCgrH+3B1iuQ9jcAdhbJr0AYi8w0/T1kHYkWzrrsU6qlaekBqQGpAb6XAM9B2h0gS9M1LB+BczTfV0gTd0jt4cYRKS8cIXoBpa/xIJFYe0pDCw+n6sPmM8cDBkH4SO3Y7Fw9K+4b5nqpSRA2T720GA1DxtedlRvPRyoGZmkBqQGpAbypYFeAWghXPTY20fGv9+INaLVnQXNBmQiUNy0SHwwERkCcHeyQisMRutSE7gXpU+nnd3pnNgRKFMSkRkiT75mQdeNj2OVlaPpYwEWUf6XGpAakBro/xrwoWOviEyRE4mWJwDI5HqwkheQnWBNHOJ8mr3TnQBjwSTcGJQXAE08uvkui5Qe3q1oElGn3EsNSA1IDfSxBvID0KJTFDrX3PAnhMthsXsTU7bTyQnGwoIW9rHIC17nXoAv0UQ8tItGoIyp2iHtwVhZxR+kxexUnjyWGpAaGGgayC9AO7VBi+En4mfAFXESoizSa2o4GJyg7SBnPXRa0fwjt+whForcJGcCZtWYPCE1IDUwwDRQOID2KoZiqfVkNXzM+wC0d4aVXeVlCc5jYgl8yiyk4LuByuOjSrd6Imhad3BZSZUakBqQGhg4Gvh/wvBK0jMLWd8AAAAASUVORK5CYII=)"
      ],
      "metadata": {
        "id": "0kK7NNQuz1i8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define objective function to minimize (negative accuracy)\n",
        "def objective_optuna(trial):\n",
        "    C = trial.suggest_loguniform('C', 0.1, 100)\n",
        "    gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
        "\n",
        "    clf = SVC(C=C, gamma=gamma)\n",
        "    score = cross_val_score(clf, X_train, y_train, cv=5).mean()\n",
        "    return -score  # Negative accuracy since Optuna minimizes the objective\n",
        "\n",
        "# Perform hyperparameter optimization using Optuna\n",
        "study = optuna.create_study(direction='minimize')\n",
        "study.optimize(objective_optuna, n_trials=10)\n",
        "\n",
        "# Get the best hyperparameters\n",
        "best_params = study.best_params\n",
        "print(\"Best Hyperparameters (Optuna):\", best_params)\n",
        "\n",
        "# Evaluate the model with the best hyperparameters on the test set\n",
        "best_svm_optuna = SVC(C=best_params['C'], gamma=best_params['gamma'])\n",
        "best_svm_optuna.fit(X_train, y_train)\n",
        "accuracy_optuna = best_svm_optuna.score(X_test, y_test)\n",
        "print(\"Accuracy on Test Set (Optuna):\", accuracy_optuna)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3leM6AC1z4WX",
        "outputId": "646c75ed-c805-4968-a9a2-839fa3f037d2"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-11-17 19:45:29,171] A new study created in memory with name: no-name-5ca55f16-037a-4928-9f43-58969b90e047\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,230] Trial 0 finished with value: -0.9416666666666667 and parameters: {'C': 73.66181373496023, 'gamma': 0.20768075189963264}. Best is trial 0 with value: -0.9416666666666667.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,271] Trial 1 finished with value: -0.95 and parameters: {'C': 0.8833288130427942, 'gamma': 0.2090969058221804}. Best is trial 1 with value: -0.95.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,303] Trial 2 finished with value: -0.9583333333333334 and parameters: {'C': 1.8599519901400818, 'gamma': 2.411628808928544}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,360] Trial 3 finished with value: -0.95 and parameters: {'C': 4.553040321355101, 'gamma': 9.120460796224547}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,396] Trial 4 finished with value: -0.6416666666666666 and parameters: {'C': 0.11910105714956665, 'gamma': 0.011289883440459487}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,423] Trial 5 finished with value: -0.925 and parameters: {'C': 0.4981030278838294, 'gamma': 0.0435683115411594}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,449] Trial 6 finished with value: -0.9083333333333334 and parameters: {'C': 0.2132664365095729, 'gamma': 0.058259308220709106}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,496] Trial 7 finished with value: -0.9166666666666667 and parameters: {'C': 0.7279296793776932, 'gamma': 0.01679601347029499}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,523] Trial 8 finished with value: -0.9166666666666667 and parameters: {'C': 0.6038250353194058, 'gamma': 0.023338296397415238}. Best is trial 2 with value: -0.9583333333333334.\n",
            "<ipython-input-39-f06d85e7428d>:3: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  C = trial.suggest_loguniform('C', 0.1, 100)\n",
            "<ipython-input-39-f06d85e7428d>:4: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n",
            "  gamma = trial.suggest_loguniform('gamma', 0.01, 10)\n",
            "[I 2023-11-17 19:45:29,558] Trial 9 finished with value: -0.7333333333333332 and parameters: {'C': 0.14160197243617356, 'gamma': 0.019980710313020206}. Best is trial 2 with value: -0.9583333333333334.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best Hyperparameters (Optuna): {'C': 1.8599519901400818, 'gamma': 2.411628808928544}\n",
            "Accuracy on Test Set (Optuna): 1.0\n"
          ]
        }
      ]
    }
  ]
}