tp1_simple_neural_net.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TP1: Réseau de neurones à trois couches, fonction de perte **Erreur quadratique** (Mean square error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "from utils.data_utils import load_CIFAR10\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "# pour automatiquement recharger les modules externes\n",
    "# voir http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font size=5>Code simple pour un MLP à trois couches</font>\n",
    "\n",
    "Le but de ce notebook est de vous familiariser avec le code pour effectuer une **propagation avant**, une **propagation arrière** puis une **descente de gradient**.  Vous verrez du code \"naif\" ainsi qu'un implémentation plus \"réelle\" avec l'utilisation d'une *cache*.\n",
    "\n",
    "Bien qu'aucune note ne soit attribée à ce notebook, prenez le temps de **bien comprendre les notions qui suivent**.\n",
    "\n",
    "Il est d'ailleurs recommandé de \"jouer\" avec le code afin de :\n",
    "\n",
    "* implanter d'autres fonctions d'activation (relu, tanh, etc)\n",
    "* implanter un réseau à 2 couche puis à 4 couches \n",
    "* tester une autre loss (entropie croisée)\n",
    "* etc.\n",
    "\n",
    "Le but de ce notebook est de vous préparer au *tp1_neural_net.ipynb* dans lequel les concepts abordés ici seront encapsulés dans des **classes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Les variables déterminant la forme du réseau                               #\n",
    "##############################################################################\n",
    "\n",
    "num_input_neurons = 4\n",
    "num_hidden_neurons_1 = 5\n",
    "num_hidden_neurons_2 = 6\n",
    "num_output_neurons = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Création de données jouets aléatoires                                      #\n",
    "##############################################################################\n",
    "\n",
    "def create_random_toy_data(nb_data, nb_input_neurons, num_output_neurons):\n",
    "    np.random.seed(1)\n",
    "    \n",
    "    X = 10 * np.random.randn(nb_data, nb_input_neurons)\n",
    "    y = 0.1 * np.random.randn(nb_data, num_output_neurons)\n",
    "        \n",
    "    return X, y\n",
    "\n",
    "# Pour simplifier les choses, commençons avec une seule donnée\n",
    "X, y = create_random_toy_data(1, num_input_neurons, num_output_neurons)\n",
    "print(X.shape)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Création + initialisation des matrices de poids                            #\n",
    "##############################################################################\n",
    "def create_random_weights(in_size, num_1, num_2, nb_out):\n",
    "    Weight_1 = np.random.normal(loc=0.0, scale=0.01, size=(in_size, num_1))\n",
    "    Weight_2 = np.random.normal(loc=0.0, scale=0.01, size=(num_1, num_2))\n",
    "    Weight_3 = np.random.normal(loc=0.0, scale=0.01, size=(num_2, nb_out))\n",
    "    W = {\"W1\": Weight_1, \"W2\": Weight_2, \"W3\": Weight_3}\n",
    "    return W\n",
    "\n",
    "W = create_random_weights(num_input_neurons, num_hidden_neurons_1, num_hidden_neurons_2, num_output_neurons)\n",
    "\n",
    "print(\"Poids de la premiere couche = \\n\", W['W1'])\n",
    "print(\"Poids de la 2e couche = \\n\", W['W2'])\n",
    "print(\"Poids de la 3e couche = \\n\", W['W3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Fonction d'activation + sa dérivée                                         #\n",
    "##############################################################################\n",
    "def sigmoid(x):\n",
    "    return 1.0 / (1.0 + np.exp(-x))\n",
    "\n",
    "def derivative_sigmoid(x):\n",
    "    return sigmoid(x)*(1-sigmoid(x))\n",
    "\n",
    "print('sig(0) = ', sigmoid(0), '  sig(1) = ', sigmoid(1), '  sig(-1) = ', sigmoid(-1))\n",
    "print('dsig(0) = ', derivative_sigmoid(0), '  dsig(1) = ', derivative_sigmoid(1), '  dsig(-1) = ', derivative_sigmoid(-1))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code naif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Une simple propagation avant                                               #\n",
    "##############################################################################\n",
    "\n",
    "def forward_pass(W, X, activation):\n",
    "    H1 = X.dot(W['W1'])\n",
    "    L1 = activation(H1)\n",
    "    \n",
    "    H2 = L1.dot(W['W2'])\n",
    "    L2 = activation(H2)\n",
    "    \n",
    "    H3 = L2.dot(W['W3'])\n",
    "    score = H3\n",
    "    \n",
    "    return score\n",
    "\n",
    "score = forward_pass(W, X, sigmoid)\n",
    "\n",
    "print('score = ', score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Une simple propagation + loss quadratique                                  #\n",
    "##############################################################################\n",
    "def forward_pass_with_loss_MSE(W, X, y, activation):\n",
    "    H1 = X.dot(W['W1'])\n",
    "    L1 = activation(H1)\n",
    "    \n",
    "    H2 = L1.dot(W['W2'])\n",
    "    L2 = activation(H2)\n",
    "    \n",
    "    H3 = L2.dot(W['W3'])\n",
    "    score = H3\n",
    "    \n",
    "    loss = np.sum(np.power(score - y, 2))\n",
    "    \n",
    "    return score, loss\n",
    "\n",
    "score, loss = forward_pass_with_loss_MSE(W, X, y, sigmoid)\n",
    "\n",
    "print('score = ', score)\n",
    "print('loss MSE = ', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Fonction effectuant une propagation AVANT + ARRIERE                        #\n",
    "#                                                                            #\n",
    "# WARNING-WARNING-WARNING!                                                   #\n",
    "# Portez attention à la dimension de chaque opération, c'est ce qui est le   #\n",
    "# plus mélangeant! Pour comprendre ce code, il est préférable d'effectuer    #\n",
    "# la dérivation manuellement sur papier.                                     #\n",
    "#                                                                            #\n",
    "##############################################################################\n",
    "\n",
    "def forward_naive_backward_pass_with_loss_MSE(W, X, y, activation, derivative_activation):\n",
    "    # X : 1x4\n",
    "    # W1 : 4x5\n",
    "    # W2 : 5x6\n",
    "    # W3 : 6x3\n",
    "    H1 = X.dot(W['W1']) #  H1 : 1x4.4x5 => 1x5\n",
    "    L1 = activation(H1) #  L1 : 1x5\n",
    "    \n",
    "    H2 = L1.dot(W['W2']) # H2 : 1x5.5x6 => 1x6\n",
    "    L2 = activation(H2)  # L2 : 1x6\n",
    "    \n",
    "    H3 = L2.dot(W['W3']) # H3 : 1x6.6x3 => 1x3\n",
    "    score = H3           # score : 1x3\n",
    "    \n",
    "    loss = np.sum(np.power(score - y, 2)) # loss : 1x3\n",
    "    \n",
    "    dLoss_dScore = 2 * (score - y)        # dLoss_dScore : 1x3\n",
    "    \n",
    "    dH3_dW3 = L2                          # dH3_dW3 : 1x6\n",
    "    dH3_dL2 = W['W3']                     # dH3_dL2 : 6x3\n",
    "    dL2_dH2 = derivative_activation(H2)   # dL2_dH2 : 1x6\n",
    "    dH2_dW2 = L1                          # dH2_dW2 : 1x5\n",
    "    dH2_dL1 = W['W2']                     # dH3_dL2 : 5x6\n",
    "    dL1_dH1 = derivative_activation(H1)   # dL1_dH1 : 1x5\n",
    "    dH1_dW1 = X                           # dH1_dW1 : 1x4\n",
    "    \n",
    "    # dLoss_dScore * dH3_dW3 \n",
    "    #      1x3         1x6   => 6x3\n",
    "    dW3 = dLoss_dScore.T.dot(dH3_dW3).T\n",
    "    \n",
    "    # dLoss_dScore * dH3_dL2 * dL2_dH2 * dH2_dW2 \n",
    "    #      1x3         6x3      ----       1x5   => 5x6\n",
    "    dW2 = dLoss_dScore.dot(dH3_dL2.T)    # 1x3.3x6 : 1x6\n",
    "    dW2 = dW2 * dL2_dH2                  # 1x6\n",
    "    dW2 = dW2.T.dot(dH2_dW2).T           # 6x1 . 1x5.\n",
    "    \n",
    "    # dLoss_dScore * dH3_dL2 * dL2_dH2 * dH2_dL1 * dL1_dH1 * dH1_dW1  \n",
    "    #      1x3         6x3      ----       5x6      ----       1x4  => 4x5\n",
    "    dW1 = dLoss_dScore.dot(dH3_dL2.T)    # 1x3.3x6 : 1x6\n",
    "    dW1 = dW1 * dL2_dH2                  # 1x6\n",
    "    dW1 = dW1.dot(dH2_dL1.T)             # 1x6.6x5 : 1x5\n",
    "    dW1 = dW1 * dL1_dH1                  # 1x5.\n",
    "    dW1 = dW1.T.dot(dH1_dW1).T         # 5x1.1x4.t : 4x5 \n",
    "    \n",
    "    dW = {'dW1':dW1, 'dW2':dW2, 'dW3':dW3}\n",
    "    return score, loss, dW\n",
    "\n",
    "score, loss, dW = forward_naive_backward_pass_with_loss_MSE(W, X, y, sigmoid, derivative_sigmoid)\n",
    "\n",
    "# Meme dimensions?\n",
    "print(dW['dW1'].shape,  ' vs ', W['W1'].shape)\n",
    "print(dW['dW2'].shape,  ' vs ', W['W2'].shape)\n",
    "print(dW['dW3'].shape,  ' vs ', W['W3'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Descente de gradient sur des données aléatoires                            #\n",
    "# Le but est de montrer que le réseau \"apprend\" quelque chose au bout de     #\n",
    "# quelques epochs                                                            #\n",
    "##############################################################################\n",
    "train_size = 1000\n",
    "lr = 0.1\n",
    "history_loss = []\n",
    "X, y = create_random_toy_data(train_size, num_input_neurons, num_output_neurons)\n",
    "\n",
    "W = create_random_weights(num_input_neurons, num_hidden_neurons_1, num_hidden_neurons_2, num_output_neurons)\n",
    "for iter in range(100):\n",
    "    if np.mod(iter,10)==0:\n",
    "        print(iter)\n",
    "    \n",
    "    loss_epoch = 0\n",
    "    for i in range(train_size):\n",
    "        \n",
    "        x_data = np.expand_dims(X[i,:], axis=0)\n",
    "        score, loss, dW = forward_naive_backward_pass_with_loss_MSE(W, x_data, y[i,:], sigmoid, derivative_sigmoid)\n",
    "        W['W1'] = W['W1'] - lr*dW['dW1']\n",
    "        W['W2'] = W['W2'] - lr*dW['dW2']\n",
    "        W['W3'] = W['W3'] - lr*dW['dW3']\n",
    "        \n",
    "        loss_epoch += loss\n",
    "        \n",
    "    history_loss.append(loss_epoch/train_size)\n",
    "\n",
    "# En principe, la loss devrait décroitre\n",
    "plt.plot(history_loss)\n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code plus réaliste utilisant une \"cache\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def real_forward_pass_with_loss_MSE(W, X, y, activation, derivative_activation):\n",
    "    # X : 1x4\n",
    "    # W1 : 4x5\n",
    "    # W2 : 5x6\n",
    "    # W3 : 6x3\n",
    "    H1 = X.dot(W['W1']) #  H1 : 1x4.4x5 => 1x5\n",
    "    L1 = activation(H1) #  L1 : 1x5\n",
    "    \n",
    "    H2 = L1.dot(W['W2']) # H2 : 1x5.5x6 => 1x6\n",
    "    L2 = activation(H2)  # L2 : 1x6\n",
    "    \n",
    "    H3 = L2.dot(W['W3']) # H3 : 1x6.6x3 => 1x3\n",
    "    score = H3           # score : 1x3\n",
    "    \n",
    "    loss = np.sum(np.power(score - y, 2)) # loss : 1x3\n",
    "    \n",
    "    cache = {'H1':H1, 'L1':L1, 'H2':H2, 'L2':L2, 'score':score}\n",
    "    return loss, cache\n",
    "\n",
    "def real_backprop_pass_with_loss_MSE(W, X, y, activation, derivative_activation, cache):\n",
    "    dLoss_dScore = 2 * (cache['score'] - y)\n",
    "    dW3 = dLoss_dScore.T.dot(cache['L2']).T\n",
    "    \n",
    "    dW2 = dLoss_dScore.dot(W['W3'].T)              \n",
    "    dW2 = dW2 * derivative_activation(cache['H2'])\n",
    "    dW2 = dW2.T.dot(cache['L1']).T\n",
    "    \n",
    "    dW1 = dLoss_dScore.dot(W['W3'].T)\n",
    "    dW1 = dW1 * derivative_activation(cache['H2'])\n",
    "    dW1 = dW1.dot(W['W2'].T)\n",
    "    dW1 = dW1 * derivative_activation(cache['H1'])\n",
    "    dW1 = dW1.T.dot(X).T\n",
    "    \n",
    "    dW = {'dW1':dW1, 'dW2':dW2, 'dW3':dW3}\n",
    "    return dW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################################\n",
    "# Descente de gradient sur des données aléatoires                            #\n",
    "# Le but est de montrer que le réseau \"apprend\" quelque chose au bout de     #\n",
    "# quelques epochs                                                            #\n",
    "##############################################################################\n",
    "X, y = create_random_toy_data(1, num_input_neurons, num_output_neurons)\n",
    "\n",
    "loss, cache = real_forward_pass_with_loss_MSE(W, X, y, sigmoid, derivative_sigmoid)\n",
    "dW = real_backprop_pass_with_loss_MSE(W, X, y, sigmoid, derivative_sigmoid, cache)\n",
    "\n",
    "# Meme dimensions? oui!\n",
    "print(dW['dW1'].shape,  ' vs ', W['W1'].shape)\n",
    "print(dW['dW2'].shape,  ' vs ', W['W2'].shape)\n",
    "print(dW['dW3'].shape,  ' vs ', W['W3'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 1000\n",
    "lr = 0.1\n",
    "history_loss = []\n",
    "X, y = create_random_toy_data(train_size, num_input_neurons, num_output_neurons)\n",
    "W = create_random_weights(num_input_neurons, num_hidden_neurons_1, num_hidden_neurons_2, num_output_neurons)\n",
    "\n",
    "# 100 epochs\n",
    "for iter in range(100):\n",
    "    if np.mod(iter,10)==0:\n",
    "        print(iter)\n",
    "    \n",
    "    loss_epoch = 0\n",
    "    for i in range(train_size):\n",
    "        \n",
    "        x_data = np.expand_dims(X[i,:], axis=0)\n",
    "        loss, cache = real_forward_pass_with_loss_MSE(W, x_data, y[i,:], sigmoid, derivative_sigmoid)\n",
    "        dW = real_backprop_pass_with_loss_MSE(W, x_data, y[i,:], sigmoid, derivative_sigmoid, cache)\n",
    "\n",
    "        W['W1'] = W['W1'] - lr*dW['dW1']\n",
    "        W['W2'] = W['W2'] - lr*dW['dW2']\n",
    "        W['W3'] = W['W3'] - lr*dW['dW3']\n",
    "        \n",
    "        loss_epoch += loss\n",
    "        \n",
    "    history_loss.append(loss_epoch/train_size)\n",
    "\n",
    "plt.plot(history_loss)\n",
    "plt.show() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}