Mathematics_for_Machine_Learning_Examples.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Mathematics for Machine Learning Examples",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "ST8QJlxSdvMU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7eae7c44-6007-47b5-bd59-cb4ca1df832c"
      },
      "cell_type": "code",
      "source": [
        "#@title Imports { display-mode: \"form\" }\n",
        "from __future__ import print_function\n",
        "from __future__ import division\n",
        "from __future__ import absolute_import\n",
        "\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "try:\n",
        "  tf.enable_eager_execution()\n",
        "  print('Eager execution enabled')\n",
        "except ValueError:\n",
        "  print('Already running in Eager mode')\n",
        "\n",
        "tfe = tf.contrib.eager\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Already running in Eager mode\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RgFS1eZOdws_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Matrix Multiplication"
      ]
    },
    {
      "metadata": {
        "id": "E0TdsagndySr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "535c541c-da43-4154-dae9-7198ba1e9ed0"
      },
      "cell_type": "code",
      "source": [
        "# Define matrix A\n",
        "A = np.array(\n",
        "  [[1.0, 3.0],\n",
        "   [2.0, 1.0],\n",
        "   [4.0, 2.0]]\n",
        ")\n",
        "\n",
        "# Define matrix B\n",
        "B = np.array(\n",
        "  [[6.0, 2.0, 1.0],\n",
        "   [3.0, 4.0, 5.0]]\n",
        ")\n",
        "\n",
        "# Define vector x\n",
        "x = np.array([3.0, 2.0])\n",
        "\n",
        "print('A.shape is:', A.shape, 'B.shape is:', B.shape, 'x.shape is:', x.shape)\n",
        "A"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "A.shape is: (3, 2) B.shape is: (2, 3) x.shape is: (2,)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1., 3.],\n",
              "       [2., 1.],\n",
              "       [4., 2.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "metadata": {
        "id": "4xVzrzTskHqK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Matrix-vector multiplication"
      ]
    },
    {
      "metadata": {
        "id": "ySFMEkRrkJ51",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "f52ddad3-c34a-4a07-8ec4-8682e1abcc4e"
      },
      "cell_type": "code",
      "source": [
        "# Using numpy dot\n",
        "y = A.dot(x)\n",
        "\n",
        "print('Using dot:\\t y =', y, '\\t y.shape =', y.shape)\n",
        "\n",
        "# Using einsum\n",
        "y = np.einsum('ij, j', A, x)\n",
        "\n",
        "print('Using einsum:\\t y =', y, '\\t y.shape =', y.shape)\n",
        "\n",
        "# Manual version 1\n",
        "y = np.array([\n",
        "    A[0,0] * x[0] + A[0,1] * x[1],\n",
        "    A[1,0] * x[0] + A[1,1] * x[1],\n",
        "    A[2,0] * x[0] + A[2,1] * x[1],\n",
        "    ])\n",
        "print('Manual 1:\\t y =', y, '\\t y.shape =', y.shape)\n",
        "\n",
        "# Manual version 2: \n",
        "# Matrix-vector multiplication can be thought of as a linear combination of the columns of of the matrix\n",
        "y = x[0] * A[:,0]  +  x[1] * A[:, 1]\n",
        "\n",
        "print('Manual 2:\\t y =', y, '\\t y.shape =', y.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using dot:\t y = [ 9.  8. 16.] \t y.shape = (3,)\n",
            "Using einsum:\t y = [ 9.  8. 16.] \t y.shape = (3,)\n",
            "Manual 1:\t y = [ 9.  8. 16.] \t y.shape = (3,)\n",
            "Manual 2:\t y = [ 9.  8. 16.] \t y.shape = (3,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "IXfRCQfglg7Y",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Matrix-matrix multiplication"
      ]
    },
    {
      "metadata": {
        "id": "CCw3o6GMeD1o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        },
        "outputId": "8f963951-8502-4b13-b858-fce70e4c6538"
      },
      "cell_type": "code",
      "source": [
        "# Using numpy dot\n",
        "C = A.dot(B)\n",
        "\n",
        "print('Using DOT: C= \\n\\n', C, '\\n\\nC.shape =', C.shape)\n",
        "\n",
        "# Using einsum\n",
        "C = np.einsum('ik, kj', A, B)\n",
        "print('\\n\\nUsing einsum: C= \\n\\n', C, '\\n\\nC.shape =', C.shape)\n",
        "\n",
        "# Note, the above einsum notation is equivalent to the following\n",
        "C = np.einsum('ik, kj -> ij', A, B)\n",
        "\n",
        "# And in Tensorflow\n",
        "C = tf.matmul(A, B)\n",
        "print('\\n\\nUsing Tensorflow: C= \\n\\n', C, '\\n\\nC.shape =', C.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using DOT: C= \n",
            "\n",
            " [[15. 14. 16.]\n",
            " [15.  8.  7.]\n",
            " [30. 16. 14.]] \n",
            "\n",
            "C.shape = (3, 3)\n",
            "\n",
            "\n",
            "Using einsum: C= \n",
            "\n",
            " [[15. 14. 16.]\n",
            " [15.  8.  7.]\n",
            " [30. 16. 14.]] \n",
            "\n",
            "C.shape = (3, 3)\n",
            "\n",
            "\n",
            "Using Tensorflow: C= \n",
            "\n",
            " tf.Tensor(\n",
            "[[15. 14. 16.]\n",
            " [15.  8.  7.]\n",
            " [30. 16. 14.]], shape=(3, 3), dtype=float64) \n",
            "\n",
            "C.shape = (3, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "o3U0Fc7mlxSd",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Matrix multiplication is not commutative"
      ]
    },
    {
      "metadata": {
        "id": "iiI6J7ZDj9Fl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "1c0e575f-1b2c-4da6-aab1-7f1e5ed60f19"
      },
      "cell_type": "code",
      "source": [
        "# Matrix multiplication is not commutative:\n",
        "C = B.dot(A)\n",
        "print('C: \\n', C)\n",
        "print()\n",
        "print('C.shape:', C.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C: \n",
            " [[14. 22.]\n",
            " [31. 23.]]\n",
            "\n",
            "C.shape: (2, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "WxJ0muOj34SB",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Computing gradients with TensorFlow\n",
        "$y = Ax$\n",
        "\n",
        "In the code below, we use Tensorflow to calculate the following derivatives:\n",
        "\n",
        "$\\frac{dy}{dx}$ \n",
        "\n",
        "and \n",
        "\n",
        "$\\frac{\\partial y}{\\partial A}$ "
      ]
    },
    {
      "metadata": {
        "id": "5wCGTn1s3zlV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "2b250cbe-4c07-481d-ecf9-ed8d449cf921"
      },
      "cell_type": "code",
      "source": [
        "A_tensor = tfe.Variable(A)\n",
        "x_tensor = tfe.Variable(x)\n",
        "\n",
        "with tf.GradientTape() as tape:\n",
        "  y = tf.einsum('ij,j', A_tensor, x_tensor)\n",
        "\n",
        "dydx, dydA = tape.gradient(y, [x_tensor, A_tensor])\n",
        "\n",
        "print('dy/dx =', dydx)\n",
        "print()\n",
        "print('dy/dA =', dydA)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dy/dx = tf.Tensor([7. 6.], shape=(2,), dtype=float64)\n",
            "\n",
            "dy/dA = tf.Tensor(\n",
            "[[3. 2.]\n",
            " [3. 2.]\n",
            " [3. 2.]], shape=(3, 2), dtype=float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "btAykJ_oEZ-i",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Neural Network Gradient Example\n",
        "In the following example, we compute the output of a 1 layer neural network and the gradients with respect to its parameters. We define an example input vector and parameters, but keep the computation generic. You can change the values and shapes of x, A and b below and run the rest of the code to compute the output and gradients for your own example."
      ]
    },
    {
      "metadata": {
        "id": "WRnif1ZP4kYQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x = np.array([[-1.], [0.1], [2.1]])  # X has shape (3, 1)\n",
        "A = np.array([              # A has shape (2, 3)\n",
        "    [ 1.1, -2.5,  0.3],\n",
        "    [-2.1,  0.2, -1.1]\n",
        "])  \n",
        "b = np.array([[-1.0], [2.0]])      # b has shape (2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XaTm7-r9FBxl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Compute the neural network output\n",
        "$\\mathbf{f} = \\operatorname{tanh}(A\\mathbf{x} + \\mathbf{b})$"
      ]
    },
    {
      "metadata": {
        "id": "1HR6uCTjFAvV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "ff5beb67-d370-4a70-a8ee-b6c4b64a585e"
      },
      "cell_type": "code",
      "source": [
        "M, N = A.shape\n",
        "z = A.dot(x) + b\n",
        "f = np.tanh(z)\n",
        "\n",
        "print('f =', f)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "f = [[-0.93786303]\n",
            " [ 0.94783185]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wQKVQ13jGkIH",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Compute the partial derivatives:\n",
        "\n",
        "\\begin{align}\n",
        "\\frac{d\\mathbf{f}}{d\\mathbf{z}} ; \\frac{\\partial\\mathbf{z}}{\\partial\\mathbf{x}} ; \\frac{\\partial\\mathbf{z}}{\\partial\\mathbf{b}} ; \\frac{\\partial\\mathbf{z}}{\\partial\\mathbf{A}}\n",
        "\\end{align}"
      ]
    },
    {
      "metadata": {
        "id": "sJF9p5lHE4Z_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "outputId": "3900b59e-ab3d-4f9d-caa4-dd79c7fd49c6"
      },
      "cell_type": "code",
      "source": [
        "# partial derivatives\n",
        "dfdz = 1-f**2       # (derivative of tanh is 1-tanh^2)\n",
        "print('df/dz =', dfdz, '\\nshape:', dfdz.shape)\n",
        "print()\n",
        "\n",
        "dzdx = A\n",
        "print('dz/dx =\\n', dzdx, '\\n\\nshape:', dzdx.shape)\n",
        "print()\n",
        "\n",
        "dzdb = np.eye(M)\n",
        "print('dz/db =\\n', dzdb, '\\n\\nshape:', dzdb.shape)\n",
        "print()\n",
        "\n",
        "dzdA = np.zeros((M, M, N))  # Start with a tensor of zeros of the correct shape\n",
        "for i in range(M):          # Then set the diagonal elements of dzdA\n",
        "  dzdA[i,i,:] = x.T  \n",
        "\n",
        "print('dz/dA =\\n', dzdA, '\\n\\nshape:', dzdA.shape)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "df/dz = [[0.12041293]\n",
            " [0.10161478]] \n",
            "shape: (2, 1)\n",
            "\n",
            "dz/dx =\n",
            " [[ 1.1 -2.5  0.3]\n",
            " [-2.1  0.2 -1.1]] \n",
            "\n",
            "shape: (2, 3)\n",
            "\n",
            "dz/db =\n",
            " [[1. 0.]\n",
            " [0. 1.]] \n",
            "\n",
            "shape: (2, 2)\n",
            "\n",
            "dz/dA =\n",
            " [[[-1.   0.1  2.1]\n",
            "  [ 0.   0.   0. ]]\n",
            "\n",
            " [[ 0.   0.   0. ]\n",
            "  [-1.   0.1  2.1]]] \n",
            "\n",
            "shape: (2, 2, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Gcw6bNirweZl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Finally, we compute the gradients of the neural network output $f$ with respect to the parameters $A$ and $\\mathbf{b}$ and the input $\\mathbf{x}$ using the chain rule:\n",
        "\n",
        "\\begin{align}\n",
        "\\frac{\\partial \\mathbf{f}}{\\partial \\mathbf{x}} &= \\frac{d \\mathbf{f}}{d \\mathbf{z}} \\frac{\\partial \\mathbf{z}}{\\partial \\mathbf{x}} \\ ; \\ \n",
        "\\frac{\\partial \\mathbf{f}}{\\partial \\mathbf{b}} = \\frac{d \\mathbf{f}}{d \\mathbf{z}} \\frac{\\partial \\mathbf{z}}{\\partial \\mathbf{b}} \\ ; \\ \n",
        "\\frac{\\partial \\mathbf{f}}{\\partial A} = \\frac{d \\mathbf{f}}{d \\mathbf{z}} \\frac{\\partial \\mathbf{z}}{\\partial A} \n",
        "\\end{align}"
      ]
    },
    {
      "metadata": {
        "id": "o22q4uljzbpR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "c07b47af-c680-4e0a-ef47-f9bf921a8346"
      },
      "cell_type": "code",
      "source": [
        "dfdx = np.einsum('il, lj', dfdz, dzdx)\n",
        "print('df/dx =\\n', dfdx, '\\n\\nshape:', dfdx.shape)\n",
        "print()\n",
        "\n",
        "dfdb = np.einsum('il, lj', dfdz, dzdb)\n",
        "print('df/db =\\n', dfdb, '\\n\\nshape:', dfdb.shape)\n",
        "print()\n",
        "\n",
        "dfdA = np.einsum('il, ljk', dfdz, dzdA)\n",
        "print('df/dA =\\n', dfdA, '\\n\\nshape:', dfdA.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "df/dx =\n",
            " [[-0.12041293 -0.27694975 -0.09633035]\n",
            " [-0.10161478 -0.233714   -0.08129183]] \n",
            "\n",
            "shape: (2, 3)\n",
            "\n",
            "df/db =\n",
            " [[0.12041293 0.12041293]\n",
            " [0.10161478 0.10161478]] \n",
            "\n",
            "shape: (2, 2)\n",
            "\n",
            "df/dA =\n",
            " [[[-0.12041293  0.01204129  0.25286716]\n",
            "  [-0.12041293  0.01204129  0.25286716]]\n",
            "\n",
            " [[-0.10161478  0.01016148  0.21339105]\n",
            "  [-0.10161478  0.01016148  0.21339105]]] \n",
            "\n",
            "shape: (2, 2, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}