tp1_hinge.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TP1: Classifieur linéaire, fonction de perte **Hinge**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "from utils.data_utils import load_CIFAR10\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "# Pour automatiquement recharger les modules externes\n",
    "# voir http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CIFAR-10 Chargement des données et prétraitement\n",
    "\n",
    "### **TODO** assurez-vous d'exécuter le script *./get_datasets.sh* au moins une fois dans un terminal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000, num_dev=500, num_batch=200):\n",
    "    \"\"\"\n",
    "    Charger la banque de données CIFAR-10, prétraiter les images et ajouter une dimension pour le biais.\n",
    "    \n",
    "    Input :\n",
    "    - num_training : nombre d'images à mettre dans l'ensemble d'entrainement\n",
    "    - num_validation : nombre d'images à mettre dans l'ensemble de validation\n",
    "    - num_test : nombre d'images à mettre dans l'ensemble de test\n",
    "    - num_dev : d'images à mettre dans l'ensemble dev\n",
    "    \n",
    "    Output :\n",
    "    - X_train, y_train : données et cibles d'entrainement\n",
    "    - X_val, y_val: données et cibles de validation\n",
    "    - X_test y_test: données et cibles de test \n",
    "    - X_dev, y_dev: données et cibles dev\n",
    "    - X_batch, y_batch: batch de données et de cibles \n",
    "    \"\"\"\n",
    "    # Charger les données CIFAR-10\n",
    "    cifar10_dir = 'datasets/cifar-10-batches-py'\n",
    "    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)\n",
    "  \n",
    "    # Séparer en ensembles d'entraînement, de validation, de test et de dev\n",
    "    mask = range(num_training, num_training + num_validation)\n",
    "    X_val = X_train[mask]\n",
    "    y_val = y_train[mask]\n",
    "    mask = range(num_training)\n",
    "    X_train = X_train[mask]\n",
    "    y_train = y_train[mask]\n",
    "    mask = range(num_test)\n",
    "    X_test = X_test[mask]\n",
    "    y_test = y_test[mask]\n",
    "    mask = np.random.choice(num_training, num_dev, replace=False)\n",
    "    X_dev = X_train[mask]\n",
    "    y_dev = y_train[mask]\n",
    "    mask = range(num_batch)\n",
    "    X_batch = X_train[mask]\n",
    "    y_batch = y_train[mask]\n",
    "    \n",
    "    X_train = np.reshape(X_train, (X_train.shape[0], -1))\n",
    "    X_val = np.reshape(X_val, (X_val.shape[0], -1))\n",
    "    X_test = np.reshape(X_test, (X_test.shape[0], -1))\n",
    "    X_dev = np.reshape(X_dev, (X_dev.shape[0], -1))\n",
    "    X_batch = np.reshape(X_batch, (X_batch.shape[0], -1))\n",
    "\n",
    "    # Normalisation\n",
    "    X_train -= np.mean(X_train, axis = 0)\n",
    "    X_val -= np.mean(X_val, axis = 0)\n",
    "    X_test -= np.mean(X_test, axis = 0)\n",
    "    X_dev -= np.mean(X_dev, axis = 0)\n",
    "    X_batch -= np.mean(X_batch, axis = 0)\n",
    "\n",
    "    # Ajout du biais\n",
    "    X_train = np.hstack([X_train, np.ones((X_train.shape[0], 1))])\n",
    "    X_val = np.hstack([X_val, np.ones((X_val.shape[0], 1))])\n",
    "    X_test = np.hstack([X_test, np.ones((X_test.shape[0], 1))])\n",
    "    X_dev = np.hstack([X_dev, np.ones((X_dev.shape[0], 1))])\n",
    "    X_batch = np.hstack([X_batch, np.ones((X_batch.shape[0], 1))])\n",
    "    \n",
    "    return X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev, X_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train, X_val, y_val, X_test, y_test, X_dev, y_dev, X_batch, y_batch = get_CIFAR10_data()\n",
    "print('Train data shape: ', X_train.shape)\n",
    "print('Train labels shape: ', y_train.shape)\n",
    "print('Validation data shape: ', X_val.shape)\n",
    "print('Validation labels shape: ', y_val.shape)\n",
    "print('Test data shape: ', X_test.shape)\n",
    "print('Test labels shape: ', y_test.shape)\n",
    "print('dev data shape: ', X_dev.shape)\n",
    "print('dev labels shape: ', y_dev.shape)\n",
    "print('batch data shape: ', X_batch.shape)\n",
    "print('batch labels shape: ', y_batch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifieur linéaire avec fonction de perte de type \"Hinge loss\"\n",
    "\n",
    "Le code pour cette section est dans le fichier **utils/loss.py**. \n",
    "\n",
    "La fonction `hinge_naive_forward_backward` estime la perte (et le gradient) à l'aide de boucles `for` qui itèrent sur chaque donnée de la mini-batch.  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# TODO: Implémenter le calcul de perte de façon naive avec des boucles dans  #\n",
    "#  la fonction hinge_naive_forward_backward située dans le fichier           #\n",
    "#  utils.loss.                                                               #\n",
    "#  On commence par UNE image et UNE cible                                    #\n",
    "##############################################################################\n",
    "\n",
    "from utils.loss import hinge_naive_fb_npdl\n",
    "import time\n",
    "\n",
    "# Matrice de poids aléatoires + 1 donnée à tester\n",
    "np.random.seed(0)\n",
    "W = np.random.randn(3073, 10) * 5e-4\n",
    "X_rnd = np.random.randn(1, 3073) * 5\n",
    "y_rnd = np.uint32(np.ones(1))\n",
    "loss, _ = hinge_naive_fb_npdl(X_rnd, W, y_rnd, 0.0)\n",
    "\n",
    "# La loss suivante est celle que vous devriez obtenir\n",
    "target_loss = 1.28220758164\n",
    "loss_error = np.abs(loss - target_loss)\n",
    "if loss_error < 1e-6:\n",
    "    print(\"Bravo!\")\n",
    "else:\n",
    "    print(\"Il y a un bug...\")\n",
    "print('loss error: %f' % loss_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# TODO: Implémenter le calcul de perte de façon naive avec des boucles dans  #\n",
    "#  la fonction hinge_naive_forward_backward située dans le fichier           #\n",
    "#  utils.loss.                                                               #\n",
    "#  Maintenant on test avec N=500 images et autant de cibles                  #\n",
    "##############################################################################\n",
    "\n",
    "from utils.loss import hinge_naive_fb_npdl\n",
    "import time\n",
    "\n",
    "# Matrice de poids aléatoires + 500 données \n",
    "np.random.seed(0)\n",
    "W = np.random.randn(3073, 10) * 1e-4\n",
    "loss, _ = hinge_naive_fb_npdl(X_batch, W, y_batch, 0.0)\n",
    "\n",
    "target_loss = 1.53644079\n",
    "loss_error = np.abs(loss - target_loss)\n",
    "if loss_error < 1e-6:\n",
    "    print(\"Bravo!\")\n",
    "else:\n",
    "    print(\"Il y a un bug...\")\n",
    "print('loss error: %f' % loss_error)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1:\n",
    "   \n",
    "Expliquez pourquoi lorsque la matrice de poids W est initialisée avec des valeurs proches de zéro, la **hinge loss** tend vers 1.0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rétro-propagation\n",
    "\n",
    "Maintenant, passons à la **rétro-propagation**!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# TODO: Implémenter le calcul de perte ET DE LA RÉTROPROPAGATION de façon    #\n",
    "#       naive avec des boucles dans la fonction                              #\n",
    "#       hinge_naive_forward_backward située dans le fichier utils.loss       #\n",
    "#                                                                            #\n",
    "#  On commence par UNE image et UNE cible                                    #\n",
    "##############################################################################\n",
    "\n",
    "from utils.loss import hinge_naive_fb_npdl\n",
    "import time\n",
    "\n",
    "# Matrice de poids aléatoires + une donnée\n",
    "np.random.seed(0)\n",
    "W = np.random.randn(3073, 10) * 1e-4\n",
    "X_rnd = np.random.randn(1, 3073)\n",
    "y_rnd = np.uint32(np.ones(1))\n",
    "loss, dW = hinge_naive_fb_npdl(X_rnd, W, y_rnd, 0.0)\n",
    "\n",
    "# La loss suivante est celle que vous devriez obtenir\n",
    "target_loss = 1.011288303265\n",
    "loss_error = np.abs(loss - target_loss)\n",
    "if loss_error < 1e-5:\n",
    "    print(\"Bravo pour la loss!\")\n",
    "else:\n",
    "    print(\"Il y a un bug au niveau de la loss...\")\n",
    "print('loss error: %f' % loss_error)\n",
    "\n",
    "# Le gradient suivant est celui que vous devriez obtenir pour les 3 premiers poids\n",
    "target_dW = np.array([0.0, 1.28672636, 0.0])\n",
    "dW_error = np.mean(np.abs(dW[0,0:3]-target_dW))\n",
    "if dW_error < 1e-7:\n",
    "    print(\"Bravo pour le gradient!\")\n",
    "else:\n",
    "    print(\"Il y a un bug au niveau du gradient...\")\n",
    "print('gradient error %f' % dW_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# TODO: Implémenter le calcul de perte ET DE LA RÉTROPROPAGATION de façon    #\n",
    "#       naive avec des boucles dans la fonction                              #\n",
    "#       hinge_naive_forward_backward située dans le fichier utils.loss       #\n",
    "#                                                                            #\n",
    "#  Maintenant on test avec N=200 images et autant de cibles                  #\n",
    "##############################################################################\n",
    "\n",
    "from utils.loss import hinge_naive_fb_npdl\n",
    "import time\n",
    "\n",
    "# Matrice de poids aléatoires + 500 données\n",
    "np.random.seed(1)\n",
    "W = np.random.randn(3073, 10) * 1e-4\n",
    "loss, dW = hinge_naive_fb_npdl(X_batch, W, y_batch, 0.0)\n",
    "\n",
    "# La loss suivante est celle que vous devriez obtenir\n",
    "target_loss = 1.5373299967\n",
    "loss_error = np.abs(loss - target_loss)\n",
    "if loss_error < 1e-5:\n",
    "    print(\"Bravo pour la loss!\")\n",
    "else:\n",
    "    print(\"Il y a un bug au niveau de la loss...\")\n",
    "print('loss error: %f' % loss_error)\n",
    "\n",
    "# Le gradient suivant est celui que vous devriez obtenir pour les 3 premiers poids\n",
    "target_dW = np.array([-2.395425, -0.840625, -0.32925, 7.256375])\n",
    "dW_error = np.mean(np.abs(dW[0,0:4]-target_dW))\n",
    "if dW_error < 1e-7:\n",
    "    print(\"Bravo pour le gradient!\")\n",
    "else:\n",
    "    print(\"Il y a un bug au niveau du gradient...\")\n",
    "print('gradient error %f' % dW_error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encore quelques vérifications d'usage\n",
    "En principe, à ce point-ci, le calcul de la hinge loss (et de son gradient) via la fonction *hinge_naive_forward_backward* devrait fonctionner.  Mais avant de passer à la prochaine étape il nous reste deux vérifications à faire : s'assurer que la **régularisation** fonctionne et passer le teste du **gradient numérique**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# En principe, le fait d'augmenter le terme de régularisation L2 devrait     #\n",
    "# augmenter la loss...                                                       #\n",
    "##############################################################################\n",
    "success = True\n",
    "\n",
    "# Matrice de poids aléatoires + 500 données\n",
    "np.random.seed(1)\n",
    "W = np.random.randn(3073, 10) * 1e-4\n",
    "prev_loss, _ = hinge_naive_fb_npdl(X_dev, W, y_dev, 0.0)\n",
    "\n",
    "reg = 1e2\n",
    "for i in range(10):\n",
    "    loss, _ = hinge_naive_fb_npdl(X_dev, W, y_dev, reg)\n",
    "    print(loss)\n",
    "    if loss < prev_loss:\n",
    "        success = False\n",
    "    prev_loss = loss\n",
    "    reg *= 1.5\n",
    "    \n",
    "if success:\n",
    "    print(\"Bravo!\")\n",
    "else:\n",
    "    print('Erreur!')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Maintenant testons le gradient numérique avec et sans régularisation       #\n",
    "# Les erreurs relatives devraient être inférieures à 1e-6                    #\n",
    "##############################################################################\n",
    "from utils.gradients import check_gradient_sparse\n",
    "\n",
    "print(\"\\n------------\\nGradient check : reg=0\\n------------\\n\")\n",
    "check_gradient_sparse(hinge_naive_forward_backward, W, X_dev, y_dev, 0.0)\n",
    "\n",
    "# Faire un autre test de gradients avec régularisation \n",
    "print(\"\\n------------\\nGradient check : reg=1e-2\\n------------\\n\")\n",
    "check_gradient_sparse(hinge_naive_forward_backward, W, X_dev, y_dev, 1e2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hinge loss et gradients vectorisés\n",
    "Passons maintenant aux choses sérieuses. Vous devez ici coder la version vectorisée de l'entropie croisée et du gradient dans la fonction **hinge_naive_forward_backward**.  Ce code s'apparente à la réponse que vous avec donné au début."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 2:  \n",
    "Les prochaines étapes consistent à implanter le calcul de la loss et du gradient de la loss de façon **linéarisée**.  Avant de commencer à coder, veuillez donner ici la formule de la loss et du gradient pour une mini-batch de 500 données contenue dans le tableau \n",
    "\n",
    "$$X \\in R^{500\\times 3073}$$\n",
    "\n",
    "et une matrice de poids $$W \\in R^{3073\\times 10}$$ \n",
    "\n",
    "où 3073 est la dimensionnalité des données et 10 est le nombre de classes.\n",
    "\n",
    "**Votre Réponse:** \n",
    "\n",
    "$$Loss = ....$$\n",
    "\n",
    "$$dW = ....$$\n",
    "\n",
    "**NOTE IMPORTANT** : la réponse à cette question ne contient aucune boucle, seulement des multiplications matricielles et ventorielles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# TODO: Implémenter le calcul de perte et du gradient de façon vectorielle   #\n",
    "# dans la fonction hinge_naive_forward_backward située dans le fichier        #\n",
    "# utils.loss.                                                                #\n",
    "# Les deux versions devraient calculer les mêmes résultats, mais la version  #\n",
    "# vectorielle devrait être BEAUCOUP PLUS RAPIDE.                             #\n",
    "##############################################################################\n",
    "start = time.time()\n",
    "loss_naive, grad_naive = hinge_naive_fb_npdl(X_dev, W, y_dev, 0.00001)\n",
    "end = time.time()\n",
    "print('naive loss: %e computed in %fs' % (loss_naive, end - start))\n",
    "\n",
    "from utils.loss import hinge_fb_npdl\n",
    "start = time.time()\n",
    "loss_vectorized, grad_vectorized = hinge_fb_npdl(X_dev, W, y_dev, 0.00001)\n",
    "end = time.time()\n",
    "print('vectorized loss: %e computed in %fs' % (loss_vectorized, end - start))\n",
    "\n",
    "grad_difference = np.linalg.norm(grad_naive - grad_vectorized, ord='fro')\n",
    "loss_diff = np.abs(loss_naive - loss_vectorized)\n",
    "if loss_diff < 1e-7:\n",
    "    print('bravo pour la loss!')\n",
    "else:\n",
    "    print('il y a un bug au niveau de la loss')\n",
    "print('Loss difference: %f' % np.abs(loss_naive - loss_vectorized))\n",
    "\n",
    "if grad_difference < 1e-7:\n",
    "    print('bravo pour le gradient !')\n",
    "else:\n",
    "    print('il y a un bug au niveau du gradient')\n",
    "print('Gradient difference: %f' % grad_difference)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Entraînement\n",
    "À l'aide de la classe **LinearClassifier** ainsi que de la fonction vectorisée **hinge_forward_backward** que vous venez de coder, vous devez maintenant entraîner un réseau de neurones multiclasses linéaire à l'aide d'une **descente de gradient**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.LinearClassifier import LinearClassifier\n",
    "from visualization.utils import visualize_loss\n",
    "import itertools as it\n",
    "lr = 1e-7\n",
    "reg = 100\n",
    "\n",
    "classifier = LinearClassifier(hinge_forward_backward)\n",
    "#\n",
    "# TODO : ajouter code à la fonction train.  Si tout fonctionne bien, la courbe de la loss devrait décroitre\n",
    "#\n",
    "train_loss_history = classifier.train_npdl(X_train, y_train, learning_rate=lr, reg=reg, num_iter=5000, verbose = True)\n",
    "\n",
    "y_train_pred = classifier.predict(X_train)\n",
    "y_val_pred = classifier.predict(X_val)\n",
    "\n",
    "acc_train = np.mean(y_train == y_train_pred)\n",
    "acc_val = np.mean(y_val == y_val_pred)\n",
    "\n",
    "print('train vs val acc %f / %f' %(acc_train, acc_val))\n",
    "\n",
    "visualize_loss(train_loss_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recherche d'hyper-paramètres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.LinearClassifier import LinearClassifier\n",
    "from visualization.utils import visualize_loss\n",
    "import itertools as it\n",
    "results = {}\n",
    "best_val = -1\n",
    "best_reg = -1\n",
    "best_lr = -1\n",
    "learning_rates = np.linspace(1e-8, 1e-6, 5)\n",
    "regularization_strengths = np.linspace(1, 1e3, 3)\n",
    "best_loss_history = None\n",
    "\n",
    "##############################################################################\n",
    "# TODO: Utilisez l'ensemble de validation pour régler les hyper-paramètres   #\n",
    "#  (force de régularisation et vitesse d'apprentissage). Vous devez          #\n",
    "#  expérimenter différentes plages de valeurs pour les taux d'apprentissage  #\n",
    "#  et les forces de régularisation; si vous êtes prudent, vous devriez       #\n",
    "#  pouvoir obtenir une précision de classification supérieur à 0.30 sur      #\n",
    "#  l'ensemble de validation, et de 0.36 sur l'ensemble de test.              #\n",
    "##############################################################################\n",
    "for lr, reg in it.product(learning_rates, regularization_strengths):\n",
    "    classifier = LinearClassifier(hinge_forward_backward)\n",
    "    loss_history = classifier.train_npdl(X_train, y_train, learning_rate=lr, reg=reg, num_iter=3000)\n",
    "    y_train_pred = classifier.predict(X_train)\n",
    "    y_val_pred = classifier.predict(X_val)\n",
    "    acc_train = np.mean(y_train == y_train_pred)\n",
    "    acc_val = np.mean(y_val == y_val_pred)\n",
    "    results[(lr, reg)] = (acc_train, acc_val)\n",
    "    \n",
    "    if acc_val > best_val:\n",
    "        print('best val acc %f : lr=%f reg=%f' %(acc_val, lr, reg))\n",
    "        best_lr = lr\n",
    "        best_reg = reg\n",
    "        best_val = acc_val\n",
    "        best_classifier = classifier\n",
    "        best_loss_history = loss_history\n",
    "################################################################################\n",
    "#                             FIN DE VOTRE CODE                                #\n",
    "################################################################################\n",
    "\n",
    "for lr, reg in sorted(results):\n",
    "    train_accuracy, val_accuracy = results[(lr, reg)]\n",
    "    print('lr %e reg %e train accuracy: %f val accuracy: %f' % (lr, reg, train_accuracy, val_accuracy))\n",
    "    \n",
    "print('best validation accuracy achieved during cross-validation: %f' % best_val)\n",
    "\n",
    "visualize_loss(best_loss_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# En réentraine avec plus d'itérations\n",
    "classifier = LinearClassifier(hinge_forward_backward)\n",
    "train_loss_history = classifier.train_npdl(X_train, y_train, learning_rate=best_lr, reg=best_reg, num_iter=6000, verbose = True)\n",
    "\n",
    "# On évalue la performance sur l'ensemble de test\n",
    "y_test_pred = best_classifier.predict(X_test)\n",
    "test_accuracy = np.mean(y_test == y_test_pred)\n",
    "print('Linear model with Hinge loss : CIFAR-10 final test accuracy: %f' % (test_accuracy) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualisation des poids appris pour chaque classe\n",
    "w = best_classifier.W[:-1,:] # retire le biais\n",
    "w = w.reshape(32, 32, 3, 10)\n",
    "\n",
    "w_min, w_max = np.min(w), np.max(w)\n",
    "\n",
    "classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n",
    "for i in range(10):\n",
    "    plt.subplot(2, 5, i + 1)\n",
    "\n",
    "    # Redimensionne les poids pour qu'ils soient entre 0 et 255\n",
    "    wimg = 255.0 * (w[:, :, :, i].squeeze() - w_min) / (w_max - w_min)\n",
    "    plt.imshow(wimg.astype('uint8'))\n",
    "    plt.axis('off')\n",
    "    plt.title(classes[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}