LinearRegression_Regularization_Dropout_Adam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "LinearRegression_Regularization_Dropout_Adam.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "FsbkPqpuK62s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#from google.colab import drive\n",
        "#drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpI6WIAdK6Yh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#cd /content/drive/My Drive/Walmart_Competition"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_cxhRKP0wCUY",
        "colab": {}
      },
      "source": [
        "# Imports\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pickle\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mviedDwSwXAt",
        "colab": {}
      },
      "source": [
        "def initialize_parameters_deep(layer_dims, n):\n",
        "    \"\"\"\n",
        "    This function takes the numbers of layers to be used to build our model as\n",
        "    input and otputs a dictonary containing weights and biases as parameters\n",
        "    to be learned during training\n",
        "    The number in the layer_dims corresponds to number of neurons in \n",
        "    corresponding layer\n",
        "\n",
        "    @params\n",
        "\n",
        "    Input to this function is layer dimensions\n",
        "    layer_dims = List contains number of neurons in one respective layer\n",
        "                 and [len(layer_dims) - 1] gives L Layer Neural Network\n",
        "    \n",
        "    Returns:\n",
        "    \n",
        "    parameters = Dictionary containing parameters \"W1\", \"b1\", ., \"WL\", \"bL\"\n",
        "                 where Wl = Weight Matrix of shape (layer_dims[l-1],layer_dims[l])\n",
        "                       bl = Bias Vector of shape (1,layer_dims[l])\n",
        "    \"\"\"\n",
        "    # layers_dims = [250, 128, 128, 5] #  3-layer model\n",
        "    np.random.seed(3)\n",
        "    parameters = {}\n",
        "    L = len(layer_dims)            # Number of layers in the network\n",
        "\n",
        "    for l in range(1, L):          # It starts with 1 hence till len(layer_dims)\n",
        "        # Initialize weights randomly according to Xavier initializer in order to avoid linear model\n",
        "        parameters['W' + str(l)] = np.random.randn(layer_dims[l-1],layer_dims[l])*np.sqrt(n / layer_dims[l-1])*0.7\n",
        "        #parameters['W' + str(l)] = np.random.randn(layer_dims[l-1],layer_dims[l])*np.sqrt(2 / (layer_dims[l-1]+layer_dims[l]))\n",
        "        #parameters['W' + str(l)] = np.random.randn(layer_dims[l-1],layer_dims[l])*0.005\n",
        "        # Initialize bias vector with zeros\n",
        "        parameters['b' + str(l)] = np.zeros((1,layer_dims[l]))\n",
        "        # Making sure the shape is correct\n",
        "        assert(parameters['W' + str(l)].shape == (layer_dims[l-1], layer_dims[l]))\n",
        "        assert(parameters['b' + str(l)].shape == (1,layer_dims[l]))\n",
        "\n",
        "    # parameters = {\"W [key]\": npnp.random.randn(layer_dims[l-1],layer_dims[l]) [value]}\n",
        "    return parameters"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_XI3msnwwhux",
        "colab": {}
      },
      "source": [
        "# Activation functions and their derivaties:\n",
        "\n",
        "def sigmoid(Z):\n",
        "    \"\"\"\n",
        "    This function takes the forward matrix Z (Output of the linear layer) as the\n",
        "    input and applies element-wise Sigmoid activation\n",
        "\n",
        "    @params\n",
        "\n",
        "    Z = numpy array of any shape\n",
        "    \n",
        "    Returns:\n",
        "\n",
        "    A = Output of sigmoid(Z), same shape as Z, for the last layer this A is the\n",
        "        output value from our model\n",
        "\n",
        "    cache = Z is cached, this is useful during backpropagation\n",
        "    \"\"\"\n",
        "    \n",
        "    A = 1/(1+np.exp(-Z)) # Using numpy apply sigmoid to Z \n",
        "    cache = Z           # Cache the matrix Z\n",
        "    \n",
        "    return A, cache\n",
        "\n",
        "def relu(Z):\n",
        "    \"\"\"\n",
        "    This function takes the forward matrix Z as the input and applies element \n",
        "    wise Relu activation\n",
        "\n",
        "    @params\n",
        "\n",
        "    Z = Output of the linear layer, of any shape\n",
        "\n",
        "    Returns:\n",
        "\n",
        "    A = Post-activation parameter, of the same shape as Z\n",
        "    cache = Z is cached, this is useful during backpropagation\n",
        "    \"\"\"\n",
        "    \n",
        "    A = np.maximum(0,Z) # Element-wise maximum of array elements\n",
        "    # Making sure shape of A is same as shape of Z\n",
        "    assert(A.shape == Z.shape)\n",
        "    \n",
        "    cache = Z      # Cache the matrix Z\n",
        "\n",
        "    return A, cache\n",
        "\n",
        "def relu_backward(dA, cache):\n",
        "    \"\"\"\n",
        "    This function implements the backward propagation for a single Relu unit\n",
        "\n",
        "    @params\n",
        "\n",
        "    dA = post-activation gradient, of any shape\n",
        "    cache = Retrieve cached Z for computing backward propagation efficiently\n",
        "\n",
        "    Returns:\n",
        "\n",
        "    dZ = Gradient of the cost with respect to Z\n",
        "    \"\"\"\n",
        "    \n",
        "    #Z = cache\n",
        "    dZ = np.array(dA) # Just converting dz to a correct object.\n",
        "    #print(dZ.all()==dA.all())\n",
        "    #print(dZ.shape, Z.shape)\n",
        "    # When z <= 0, you set dz to 0 as well, as relu sets negative values to 0 \n",
        "    dZ[cache <= 0] = 0\n",
        "    # Making sure shape of dZ is same as shape of Z\n",
        "    assert (dZ.shape == cache.shape)\n",
        "    \n",
        "    return dZ\n",
        "\n",
        "def sigmoid_backward(dA, cache):\n",
        "    \"\"\"\n",
        "    This function implements the backward propagation for a single Sigmoid unit\n",
        "\n",
        "    @params\n",
        "\n",
        "    dA = post-activation gradient, of any shape\n",
        "    cache = Retrieve cached Z for computing backward propagation efficiently\n",
        "\n",
        "    Returns:\n",
        "    dZ = Gradient of the cost with respect to Z\n",
        "    \"\"\"\n",
        "    \n",
        "    Z = cache\n",
        "    \n",
        "    s = 1/(1+np.exp(-Z)) # Using numpy apply Sigmoid to Z \n",
        "    dZ = dA * s * (1-s)  # This is derivatie of Sigmoid function\n",
        "\n",
        "    # Making sure shape of dZ is same as shape of Z\n",
        "    assert (dZ.shape == Z.shape)\n",
        "    \n",
        "    return dZ\n",
        "\n",
        "# Softmax\n",
        "def softmax(Z):\n",
        "    \"\"\"\n",
        "    This fucntion caculates the softmax values element wise.\n",
        "    Here I've implemented a stable softmax function\n",
        "      \n",
        "    @params\n",
        "\n",
        "    Z = Output of the linear layer, of any shape\n",
        "\n",
        "    Returns:\n",
        "\n",
        "    Elementwise exponential values of the matriz Z\n",
        "\n",
        "     \"\"\"\n",
        "    exp_val = np.exp(Z - np.max(Z, axis=1,keepdims=True))\n",
        "    return exp_val/np.sum(exp_val,axis=1,keepdims=True)\n",
        "\n",
        "def softmax_loss(Z,y,act_cache):\n",
        "    \"\"\"\n",
        "    This function takes the forward matrix Z as the input and applies element \n",
        "    wise Softmax activation. It even calculates the cross entropy loss and\n",
        "    derivative of cross entropy loss function\n",
        "\n",
        "    @params\n",
        "\n",
        "    Z = Output of the linear layer, of any shape\n",
        "    Y  = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "         shape  = (number of examples, 1)\n",
        "\n",
        "    Returns:\n",
        "\n",
        "    log_loss = Returns cross entropy loss: −∑ylog(probs)\n",
        "    dZ = Gradient of the cost with respect to Z\n",
        "    \n",
        "    \"\"\"\n",
        "    # Forward Pass\n",
        "    # Here we'll implement a stable softmax\n",
        "    m = y.shape[0]\n",
        "    cache = Z\n",
        "    A = softmax(Z)\n",
        "    #Z,_ = act_cache\n",
        "    A_back = softmax(act_cache)\n",
        "    y = y.flatten()\n",
        "    log_loss = np.sum(-np.log(A[range(m), y]))/m\n",
        "    \n",
        "    # Backward Pass\n",
        "    \n",
        "    dZ = A_back.copy()\n",
        "    dZ[range(m), y] -= 1\n",
        "    dZ /= m\n",
        "    \n",
        "    #dZ = (A - y)/m\n",
        "    assert(A.shape == Z.shape)\n",
        "    assert (dZ.shape == Z.shape)\n",
        "\n",
        "    return A, cache, log_loss, dZ"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PSwKL70nwjzO",
        "colab": {}
      },
      "source": [
        "def linear_forward(A, W, b):\n",
        "    \"\"\"\n",
        "    This function implements the forward propagation equation Z = WX + b\n",
        "\n",
        "    @params\n",
        "\n",
        "    A = Activations from previous layer (or input data),\n",
        "        shape = (number of examples, size of previous layer)\n",
        "    W = Weight matrix of shape (size of previous layer,size of current layer)\n",
        "    b = Bias vector of shape (1, size of the current layer)\n",
        "\n",
        "    Returns:\n",
        "\n",
        "    Z = The input of the activation function, also called pre-activation \n",
        "        parameter, shape = (number of examples, size of current layer)\n",
        "    cache = Tuple containing \"A\", \"W\" and \"b\"; \n",
        "            stored for computing the backward pass efficiently\n",
        "    \"\"\"\n",
        "    # print(A.shape, W.shape)\n",
        "    Z = A.dot(W) + b # Here b gets broadcasted \n",
        "    #print(Z)\n",
        "    # Making sure shape of Z = (number of examples, size of current layer)\n",
        "    assert(Z.shape == (A.shape[0], W.shape[1]))\n",
        "\n",
        "    cache = (A, W, b)   # Cache all the three params \n",
        "    \n",
        "    return Z, cache"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "DKCA8Ps8wnT-",
        "colab": {}
      },
      "source": [
        "def linear_activation_forward(A_prev, W, b, y,keep_prob,predict_result,activation):\n",
        "    \"\"\"\n",
        "    This function implements forward propagation LINEAR -> ACTIVATION layer\n",
        "\n",
        "    @params\n",
        "\n",
        "    A_prev = Activations from previous layer (or input data), \n",
        "             shape = (number of examples, size of previous layer)\n",
        "    W = Weight matrix of shape (size of previous layer,size of current layer)\n",
        "    b = Bias vector of shape (1, size of the current layer)\n",
        "    Y  = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "         shape  = (number of examples, 1)\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    predict_result = False while training, True when predicting the ground truth \n",
        "                     values (False only when ground truth values are not present)\n",
        "                     Must be kept False if you have ground truth values\n",
        "                     while predicting\n",
        "    activation = The activation to be used in this layer, \n",
        "                 stored as a text string: \"sigmoid\" or \"relu\"\n",
        "\n",
        "    Returns:\n",
        "    \n",
        "    When activation is Sigmoid:\n",
        "    A = The output of the activation function, also called the post-activation \n",
        "        value \n",
        "    cache = Tuple containing \"linear_cache\" and \"activation_cache\";\n",
        "            stored for computing the backward pass efficiently\n",
        "\n",
        "    When activation is Softmax and Y is present during training and prediction:\n",
        "    A = The output of the activation function, also called the post-activation \n",
        "        value \n",
        "    cache = Tuple containing \"linear_cache\" and \"activation_cache\";\n",
        "            stored for computing the backward pass efficiently\n",
        "    log_loss = Cross ENtropy loss\n",
        "    dZ = Derivative of cross entropy softmax \n",
        "    \n",
        "    When activation is Softmax and Y is not present during prediction:\n",
        "    Z = The input of the activation function, also called pre-activation \n",
        "        parameter, shape = (number of examples, size of current layer) \n",
        "    \"\"\"\n",
        "    \n",
        "    if activation == \"sigmoid\":\n",
        "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\"\n",
        "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
        "        A, activate_cache = Z,0 #sigmoid(Z) # Z,0 #\n",
        "        D = np.ones((A.shape[0],A.shape[1]))\n",
        "        A = np.multiply(A,D)\n",
        "        activation_cache = (activate_cache,D)\n",
        "\n",
        "    elif activation == \"relu\":\n",
        "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\"\n",
        "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
        "        A, activate_cache = relu(Z)\n",
        "        D = np.random.rand(A.shape[0],A.shape[1])\n",
        "        #print(\"Relu Function \",(A.shape, D.shape))\n",
        "        D = (D < keep_prob).astype(int)\n",
        "        #print(\"Relu D\", D.shape)\n",
        "        A = np.multiply(A,D)\n",
        "        A /= keep_prob\n",
        "        activation_cache = (activate_cache,D)\n",
        "        #print(\"Relu Activation cache\", len(activation_cache))\n",
        "        \n",
        "    elif activation == \"softmax\":\n",
        "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\"\n",
        "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
        "        #print(\"Z values\",Z)\n",
        "        if predict_result: return Z\n",
        "        A, activate_cache, log_loss, dZ = softmax_loss(Z,y,Z.copy())\n",
        "        \n",
        "        D = np.ones((A.shape[0],A.shape[1]))\n",
        "        #print(\"Softmax D\", D.shape)\n",
        "        A = np.multiply(A,D)\n",
        "        activation_cache = (activate_cache,D)\n",
        "        #print(\"Softmax Activation cache\", len(activation_cache))\n",
        "        #print(\"A values\", A)\n",
        "        \n",
        "\n",
        "    # Making sure shape of A = (number of examples, size of current layer)\n",
        "    assert (A.shape == (A_prev.shape[0],W.shape[1]))\n",
        "    cache = (linear_cache, activation_cache)\n",
        "    #print(cache)\n",
        "    if activation==\"softmax\":\n",
        "        return A, cache,log_loss,dZ \n",
        "    else: \n",
        "        return A, cache"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZK9rCX2jwpXd",
        "colab": {}
      },
      "source": [
        "def L_model_forward(X, parameters, y,Output_classes,keep_prob,predict_result,activation_type):\n",
        "    #print(y.shape)\n",
        "    \"\"\"\n",
        "    This function implements forward propagation as following:\n",
        "    [LINEAR->RELU]*(L-1) -> LINEAR -> SIGMOID computation\n",
        "    So we apply Relu to all the hidden layers and Sigmoid to the output layer\n",
        "\n",
        "    @params\n",
        "\n",
        "    X = Data, numpy array of shape (number of examples, number of features)\n",
        "    parameters = Output of initialize_parameters_deep() function\n",
        "    Y  = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "         shape  = (number of examples, 1)\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    predict_result = False while training, True when predicting the ground truth \n",
        "                     values (False only when ground truth values are not present)\n",
        "                     Must be kept False if you have ground truth values\n",
        "                     while predicting\n",
        "    activation_type = The activation to be used in this layer, \n",
        "                      stored as a text string: \"bianry\" or \"multiclass\"\n",
        "    Returns:\n",
        "\n",
        "    When activation is Binary:\n",
        "    AL = last post-activation value, also rferred as prediction from model\n",
        "    caches = list of caches containing:\n",
        "             every cache of linear_activation_forward() function\n",
        "             (there are L-1 of them, indexed from 0 to L-1)\n",
        "\n",
        "    When activation is Mukticlass and Y is present during training and prediction:\n",
        "    A = The output of the activation function, also called the post-activation \n",
        "        value \n",
        "    cache = Tuple containing \"linear_cache\" and \"activation_cache\";\n",
        "            stored for computing the backward pass efficiently\n",
        "    log_loss = Cross ENtropy loss\n",
        "    dZ = Derivative of cross entropy softmax \n",
        "\n",
        "    When activation is Multiclass and Y is not present during prediction:\n",
        "    Z = The input of the activation function, also called pre-activation \n",
        "        parameter, shape = (number of examples, size of current layer) \n",
        "    \"\"\"\n",
        "    #print(np.unique(y).shape[0])\n",
        "    caches = []\n",
        "    A = X\n",
        "    L = len(parameters) // 2            # Number of layers in the neural network\n",
        "    # Implement [LINEAR -> RELU]*(L-1). Add \"cache\" to the \"caches\" list.\n",
        "    for l in range(1, L):\n",
        "        A_prev = A\n",
        "        # For hidden layers use Relu activation\n",
        "        A, cache = linear_activation_forward(A_prev, parameters['W' + str(l)], parameters['b' + str(l)],y,keep_prob, predict_result,activation='relu')\n",
        "        #print(\"Relu A\",A.shape)\n",
        "        caches.append(cache)\n",
        "    \n",
        "    # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n",
        "    # For output layer use Sigmoid activation\n",
        "    if activation_type == \"binary\":\n",
        "        AL, cache = linear_activation_forward(A, parameters['W' + str(L)], parameters['b' + str(L)],y, keep_prob,predict_result,activation='sigmoid')\n",
        "        caches.append(cache)\n",
        "        # Making sure shape of AL = (number of examples, 1)\n",
        "        assert(AL.shape == (X.shape[0],1))\n",
        "    \n",
        "    \n",
        "    # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n",
        "    # For output layer use Sigmoid activation\n",
        "    \n",
        "    elif activation_type == \"multiclass\":\n",
        "        if not predict_result:\n",
        "            AL, cache, log_loss, dZ = linear_activation_forward(A, parameters['W' + str(L)], parameters['b' + str(L)],y,keep_prob,predict_result,activation='softmax')\n",
        "        else:\n",
        "            Z = linear_activation_forward(A, parameters['W' + str(L)], parameters['b' + str(L)],y,keep_prob,predict_result,activation='softmax')\n",
        "            return Z\n",
        "        #print(\"AL\",AL.shape)\n",
        "        caches.append(cache)\n",
        "        # Making sure shape of AL = (number of examples, number of classes)\n",
        "        assert(AL.shape == (X.shape[0],Output_classes))\n",
        "        #print(\"Softmax A\", AL.shape)\n",
        "    \n",
        "    if activation_type==\"multiclass\":\n",
        "        return AL, caches, log_loss, dZ  \n",
        "    else:\n",
        "        return AL, caches"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xs4y9mvDwrfG",
        "colab": {}
      },
      "source": [
        "def compute_cost(AL, Y, parameters, lambd, log_loss, reg_type, activation_type):\n",
        "    \"\"\"\n",
        "    When activation is Sigmoid:\n",
        "    This function implements the Binary Cross-Entropy Cost along with l1/l2 \n",
        "    regularization\n",
        "    For l1:\n",
        "    J = -(1/m)*(ylog(predictions)+(1−y)log(1−predictions)) + (λ/2*m)∑absolute(W)\n",
        "    For l2:\n",
        "    J = -(1/m)*(ylog(predictions)+(1−y)log(1−predictions)) + (λ/2*m)∑(W**2)\n",
        "\n",
        "    When activation is Softmax:\n",
        "    This function implements the Cross-Entropy Softmax Cost along with L2 \n",
        "    regularization\n",
        "    For l1:\n",
        "    J = -(1/m)*(∑ylog(predictions)) + (λ/2*m)∑absolute(W)\n",
        "    For l2:\n",
        "    J = -(1/m)*(∑ylog(predictions)) + (λ/2*m)∑(W**2)\n",
        "    \n",
        "    @params\n",
        "\n",
        "    AL = Probability vector corresponding to our label predictions \n",
        "         shape =  (number of examples, 1)\n",
        "    Y  = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "         shape  = (number of examples, 1)\n",
        "    parameters = Dictionary containing parameters as follwoing:\n",
        "                    parameters[\"W\" + str(l)] = Wl\n",
        "                    parameters[\"b\" + str(l)] = bl\n",
        "    lambd = Regularization parameter, int\n",
        "    reg_type = Type of regularization to use \"l1\" or \"l2\"\n",
        "    activation_type = The activation to be used in this layer, \n",
        "                      stored as a text string: \"bianry\" or \"multiclass\"\n",
        "    Returns:\n",
        "\n",
        "    cost = Binary or Softmax Cross-Entropy Cost with l1/l2 Regularizaion \n",
        "    \"\"\"\n",
        "    \n",
        "    m = Y.shape[0]  # Number of training examples\n",
        "    if activation_type==\"binary\":\n",
        "    # Compute loss from aL and y\n",
        "        \n",
        "        cross_entropy_cost =  (1/2*m)*np.sum((Y - AL)**2) #-(1/m)*(np.dot(np.log(AL).T,Y) + np.dot(np.log(1-AL).T,(1-Y)))\n",
        "    #print(cost)\n",
        "    elif activation_type==\"multiclass\":\n",
        "        cross_entropy_cost = log_loss\n",
        "        \n",
        "    reg_cost = []\n",
        "    W = 0\n",
        "    L = len(parameters) // 2                  # number of layers in the neural network\n",
        "    if reg_type==\"l2\":\n",
        "        for l in range(1, L+1):\n",
        "            W = parameters[\"W\" + str(l)]\n",
        "            reg_cost.append(lambd*1./(2*m)*np.sum(W**2))\n",
        "    elif reg_type==\"l1\":\n",
        "        for l in range(1, L+1):\n",
        "            W = parameters[\"W\" + str(l)]\n",
        "            reg_cost.append(lambd*np.sum(abs(W)))\n",
        "        \n",
        "\n",
        "    cross_entropy_cost = np.squeeze(cross_entropy_cost) # To make sure cost's is scalar (e.g. this turns [[cost]] into cost)\n",
        "    assert(cross_entropy_cost.shape == ())\n",
        "    cost = cross_entropy_cost + np.sum(reg_cost)\n",
        "    #print(\"Cost\",(cost,log_loss))\n",
        "\n",
        "    return cost"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Fm9KD-TQwtbg",
        "colab": {}
      },
      "source": [
        "def linear_backward(dZ,l_cache,keep_prob,lambd,reg_type):\n",
        "    \"\"\"\n",
        "    This function implements the linear portion of backward propagation for a \n",
        "    single layer (layer l)\n",
        "\n",
        "    @params\n",
        "\n",
        "    dZ = Gradient of the cost with respect to the linear output of current \n",
        "         layer l, shape = (number of examples, size of current layer)\n",
        "    cache = Tuple of values (A_prev, W, b) coming from the forward propagation \n",
        "            in the current layer\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    lambd = Regularization parameter, int\n",
        "    reg_type = Type of regularization to use \"l1\" or \"l2\"\n",
        "    \n",
        "    Returns:\n",
        "\n",
        "    dA_prev = Gradient of the cost with respect to the activation of the \n",
        "              previous layer l-1, \n",
        "              same shape as A_prev(number of examples, size of previous layer)\n",
        "    dW = Gradient of the cost with respect to W of current layer l, \n",
        "         same shape as W(size of previous layer,size of current layer)\n",
        "    db = Gradient of the cost with respect to b of current layer l, \n",
        "         same shape as b(1,size of current layer)\n",
        "    \"\"\"\n",
        "    \n",
        "    \n",
        "    if reg_type==\"l2\":\n",
        "        A_prev, W, b = l_cache\n",
        "        #print(\"1 Softmax, 2 Relu W\", W.shape)\n",
        "        #print(\"Backward A_prev for cache\",A_prev.shape)\n",
        "        \n",
        "        m = A_prev.shape[0] # Number of training examples\n",
        "        dW = (1/m)*np.dot(A_prev.T,dZ) + (1/m)*lambd*W  # Derivative wrt Weights\n",
        "        db = (1/m)*np.sum(dZ, axis=0, keepdims=True)  # Derivative wrt Bias\n",
        "        dA_prev = np.dot(dZ,W.T)\n",
        "\n",
        "    elif reg_type==\"l1\":\n",
        "        A_prev, W, b = l_cache\n",
        "        m = A_prev.shape[0] # Number of training examples\n",
        "        dW_pos = (W > 0)*lambd # wherever weights are positive(+)lambd from weights\n",
        "        dW_neg = (W < 0)*-lambd # wherever weights are negative(-)lambd from weights\n",
        "        dW = (1/m)*np.dot(A_prev.T,dZ) + (dW_pos + dW_neg)\n",
        "        db = (1/m)*np.sum(dZ, axis=0, keepdims=True)  # Derivative wrt Bias\n",
        "        dA_prev = np.dot(dZ,W.T)\n",
        "        \n",
        "    assert (dA_prev.shape == A_prev.shape)\n",
        "    assert (dW.shape == W.shape)\n",
        "    assert (db.shape == b.shape)\n",
        "    \n",
        "    return dA_prev, dW, db"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "26MtorZtwvjo",
        "colab": {}
      },
      "source": [
        "def linear_activation_backward(dA, cache,keep_prob, lambd, y, reg_type,first_layer, activation):\n",
        "    \"\"\"\n",
        "    This function implements backward propagation for LINEAR -> ACTIVATION layer\n",
        "    \n",
        "    @params\n",
        "\n",
        "    dA = post-activation gradient for current layer l \n",
        "    cache = tuple of values (linear_cache, activation_cache) \n",
        "            we store for computing backward propagation efficiently\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    lambd = Regularization parameter, int\n",
        "    Y  = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "         shape  = (number of examples, 1)\n",
        "    reg_type = Type of regularization to use \"l1\" or \"l2\"\n",
        "    first_layer = True only for first layer i.e. the input layer. It is True \n",
        "                  because while unpacking the tuple cache it has only \"Two\" values\n",
        "                  \"linear\" and \"activation\" cache, cached durinng forward pass.\n",
        "                  For other layers it is False as it has to unpack \"Four\"values\n",
        "                  of \"linear\" and \"activation\" cache, from current and next\n",
        "                  layer during backward class (current and previous layer in \n",
        "                  terms of forward pass)\n",
        "    activation = the activation to be used in this layer, \n",
        "                 stored as a text string: \"sigmoid\" or \"relu\"\n",
        "    \n",
        "    Returns:\n",
        "\n",
        "    dA_prev = Gradient of the cost with respect to the activation of the \n",
        "              previous layer l-1, \n",
        "              same shape as A_prev(number of examples, size of previous layer)\n",
        "    dW = Gradient of the cost with respect to W of current layer l, \n",
        "         same shape as W(size of previous layer,size of current layer)\n",
        "    db = Gradient of the cost with respect to b of current layer l, \n",
        "         same shape as b(1,size of current layer)\n",
        "    \"\"\"\n",
        "    \n",
        "    if activation == \"relu\":\n",
        "        if not first_layer:\n",
        "            # Unpacking Four Tuple Values from Cache\n",
        "            curr_l_a_cache, next_l_a_cache = cache\n",
        "            curr_linear_cache, curr_activation_cache = curr_l_a_cache  \n",
        "            next_linear_cache, next_activation_cache = next_l_a_cache\n",
        "            Z,_ = curr_activation_cache\n",
        "            _,D = next_activation_cache \n",
        "            dZ = relu_backward(dA,Z)\n",
        "            dA_prev, dW, db = linear_backward(dZ, curr_linear_cache,keep_prob,lambd,reg_type)\n",
        "            dA_prev = np.multiply(dA_prev,D)\n",
        "            dA_prev /= keep_prob\n",
        "        else: #Unpacking Two Tuple Values from Cache\n",
        "            curr_linear_cache, curr_activation_cache = cache\n",
        "            Z,_ = curr_activation_cache\n",
        "            dZ = relu_backward(dA,Z)\n",
        "            dA_prev, dW, db = linear_backward(dZ, curr_linear_cache,keep_prob,lambd,reg_type)\n",
        "\n",
        "    elif activation == \"sigmoid\":\n",
        "        # Unpacking Four Tuple Values from Cache\n",
        "        curr_l_a_cache, next_l_a_cache = cache\n",
        "        curr_linear_cache, curr_activation_cache = curr_l_a_cache  \n",
        "        next_linear_cache, next_activation_cache = next_l_a_cache\n",
        "\n",
        "        Z,_ = curr_activation_cache\n",
        "        _,D = next_activation_cache\n",
        "        #print(\"D\",D.shape)\n",
        "        dZ = dA #sigmoid_backward(dA,Z) # dA #\n",
        "        #print(\"dZ shape\",(dZ.shape,D.shape))\n",
        "        dA_prev, dW, db = linear_backward(dZ, curr_linear_cache,keep_prob,lambd,reg_type)\n",
        "        dA_prev = np.multiply(dA_prev,D)\n",
        "        dA_prev /= keep_prob\n",
        "        #Z,_ = activation_cache\n",
        "        #dZ = sigmoid_backward(dA,Z)\n",
        "        #dA_prev, dW, db = linear_backward(dZ, linear_cache,activation_cache,keep_prob,lambd,reg_type)\n",
        "    \n",
        "    elif activation == \"softmax\":\n",
        "        # Unpacking Four Tuple Values from Cache\n",
        "        curr_l_a_cache, next_l_a_cache = cache\n",
        "        curr_linear_cache, curr_activation_cache = curr_l_a_cache  \n",
        "        next_linear_cache, next_activation_cache = next_l_a_cache\n",
        "\n",
        "        Z,_ = curr_activation_cache\n",
        "        _,D = next_activation_cache\n",
        "        #print(\"D\",D.shape)\n",
        "        _,_,_,dZ = softmax_loss(dA, y, Z)\n",
        "        #print(\"dZ shape\",(dZ.shape,D.shape))\n",
        "        dA_prev, dW, db = linear_backward(dZ, curr_linear_cache,keep_prob,lambd,reg_type)\n",
        "        dA_prev = np.multiply(dA_prev,D)\n",
        "        dA_prev /= keep_prob\n",
        "        #print(\"Softmax dA\", dA_prev.shape)\n",
        "\n",
        "    return dA_prev, dW, db"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IYqQp87awxvH",
        "colab": {}
      },
      "source": [
        "def L_model_backward(AL, Y, caches, keep_prob, lambd,reg_type, activation_type):\n",
        "    \"\"\"\n",
        "    This function implements the backward propagation as following: \n",
        "    [LINEAR->RELU] * (L-1) -> LINEAR -> SIGMOID group\n",
        "    \n",
        "    @params\n",
        "\n",
        "    AL = probability vector, output of the L_model_forward function\n",
        "    Y = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "        shape  = (number of examples, 1)\n",
        "    caches = list of caches containing:\n",
        "             every cache of linear_activation_forward function with \"relu\" \n",
        "             (it's caches[l], for l in range(L-1) i.e l = 0...L-2)\n",
        "             the cache of linear_activation_forward function with \"sigmoid\" \n",
        "             (it's caches[L-1])\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    lambd = Regularization parameter, int\n",
        "    reg_type = Type of regularization to use \"l1\" or \"l2\"\n",
        "    activation_type = The activation to be used in this layer, \n",
        "                      stored as a text string: \"bianry\" or \"multiclass\"\n",
        "    \n",
        "    Returns:\n",
        "\n",
        "    grads = Dictionary with the gradients\n",
        "            grads[\"dA\" + str(l)] = ... \n",
        "            grads[\"dW\" + str(l+1)] = ...\n",
        "            grads[\"db\" + str(l+1)] = ... \n",
        "    \"\"\"\n",
        "\n",
        "    grads = {}\n",
        "    L = len(caches) # the number of layers\n",
        "    #print(L)\n",
        "    m = AL.shape[0] # Number of training examples\n",
        "    \n",
        "    \n",
        "    # Initializing the backpropagation\n",
        "    # Derivative of Binary Cross Entropy function\n",
        "    if activation_type==\"binary\":\n",
        "        #Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n",
        "        dAL = -(Y - AL) #- (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
        "        # Lth layer (SIGMOID -> LINEAR) gradients. \n",
        "        # Inputs: \"dAL, current_cache\". Outputs: \"grads[\"dAL-1\"], grads[\"dWL\"], grads[\"dbL\"]\n",
        "        current_cache = (caches[L-1],caches[L-2]) # Grabbig correct dropout mask of the previous layer (wrt Forward pass)\n",
        "        grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(dAL, current_cache, keep_prob,lambd,Y,reg_type,first_layer=False,activation = \"sigmoid\")\n",
        "    \n",
        "    elif activation_type==\"multiclass\":\n",
        "        #Y = Y.reshape(AL.shape)\n",
        "        #curr_cache = caches[L-2]\n",
        "        current_cache = (caches[L-1],caches[L-2]) # Grabbig correct dropout mask of the previous layer (wrt Forward pass)\n",
        "        #print(\"Softmax CC\",len(current_cache))\n",
        "        grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(AL, current_cache, keep_prob,lambd, Y,reg_type,first_layer=False,activation = \"softmax\")\n",
        "        #print(\"Softmax_grad\",grads[\"dA\"+str(L-1)])\n",
        "        \n",
        "    # Loop from l=L-2 to l=0\n",
        "    for l in reversed(range(L-1)):\n",
        "        #print(\"l\",l) #l = 1,0\n",
        "        # lth layer: (RELU -> LINEAR) gradients\n",
        "        # Inputs: \"grads[\"dA\" + str(l + 1)], current_cache\". Outputs: \"grads[\"dA\" + str(l)] , grads[\"dW\" + str(l + 1)] , grads[\"db\" + str(l + 1)] \n",
        "        if l != 0:\n",
        "            first_layer = False\n",
        "            current_cache = (caches[l], caches[l-1]) # Grabbig correct dropout mask of the previous layer (wrt Forward pass)\n",
        "            #print(\"Relu CC\",len(current_cache))\n",
        "        elif l==0:\n",
        "            first_layer = True \n",
        "            current_cache = caches[l] # No dropout is appkied to the first/input layer\n",
        "            \n",
        "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\" + str(l + 1)], current_cache,keep_prob, lambd, Y,reg_type,first_layer,activation = \"relu\")\n",
        "        grads[\"dA\" + str(l)] = dA_prev_temp\n",
        "        grads[\"dW\" + str(l + 1)] = dW_temp\n",
        "        grads[\"db\" + str(l + 1)] = db_temp\n",
        "        #print(grads)\n",
        "    return grads"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "J8f1EEcBwzwj",
        "colab": {}
      },
      "source": [
        "def initialize_adam(parameters) :\n",
        "    \"\"\"\n",
        "    This function Initializes v and s as two python dictionaries with:\n",
        "                - keys: \"dW1\", \"db1\", ..., \"dWL\", \"dbL\" \n",
        "                - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters\n",
        "    \n",
        "    @param\n",
        "    \n",
        "    parameters = Dictionary containing parameters as follwoing:\n",
        "                    parameters[\"W\" + str(l)] = Wl\n",
        "                    parameters[\"b\" + str(l)] = bl\n",
        "    \n",
        "    Returns: \n",
        "    \n",
        "    v = Dictionary that will contain the exponentially weighted average of the gradient\n",
        "                    v[\"dW\" + str(l)] = ...\n",
        "                    v[\"db\" + str(l)] = ...\n",
        "    s = Dictionary that will contain the exponentially weighted average of the squared gradient\n",
        "                    s[\"dW\" + str(l)] = ...\n",
        "                    s[\"db\" + str(l)] = ...\n",
        "\n",
        "    \"\"\"\n",
        "    \n",
        "    L = len(parameters) // 2 # number of layers in the neural networks\n",
        "    v = {}\n",
        "    s = {}\n",
        "    \n",
        "    # Initialize v, s. Input: \"parameters\". Outputs: \"v, s\".\n",
        "    for l in range(L):\n",
        "        v[\"dW\" + str(l+1)] = np.zeros(parameters['W' + str(l+1)].shape)\n",
        "        v[\"db\" + str(l+1)] = np.zeros(parameters['b' + str(l+1)].shape)\n",
        "        s[\"dW\" + str(l+1)] = np.zeros(parameters['W' + str(l+1)].shape)\n",
        "        s[\"db\" + str(l+1)] = np.zeros(parameters['b' + str(l+1)].shape)\n",
        "    \n",
        "    return v, s"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pSWKDj5hw2cn",
        "colab": {}
      },
      "source": [
        "def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate = 0.01,\n",
        "                              beta1 = 0.9, beta2 = 0.999,  epsilon = 1e-8):\n",
        "    \"\"\"\n",
        "    This function updates our model parameters using Adam\n",
        "\n",
        "    @params\n",
        "    \n",
        "    parameters = Dictionary containing our parameters:\n",
        "                  parameters['W' + str(l)] = Wl\n",
        "                  parameters['b' + str(l)] = bl\n",
        "    grads = Dictionary containing our gradients for each parameters:\n",
        "                  grads['dW' + str(l)] = dWl\n",
        "                  grads['db' + str(l)] = dbl\n",
        "    v = Adam variable, moving average of the first gradient, python dictionary\n",
        "    s = Adam variable, moving average of the squared gradient, python dictionary\n",
        "    learning_rate = the learning rate, scalar.\n",
        "    beta1 = Exponential decay hyperparameter for the first moment estimates \n",
        "    beta2 = Exponential decay hyperparameter for the second moment estimates \n",
        "    epsilon = hyperparameter preventing division by zero in Adam updates\n",
        "\n",
        "    Returns:\n",
        "    \n",
        "    parameters = Dictionary containing our updated parameters \n",
        "    v = Adam variable, moving average of the first gradient, python dictionary\n",
        "    s = Adam variable, moving average of the squared gradient, python dictionary\n",
        "    \"\"\"\n",
        "\n",
        "    L = len(parameters) // 2                 # number of layers in the neural networks\n",
        "    v_corrected = {}                         # Initializing first moment estimate, python dictionary\n",
        "    s_corrected = {}                         # Initializing second moment estimate, python dictionary\n",
        "\n",
        "    # Perform Adam update on all parameters\n",
        "    for l in range(L):\n",
        "        # Moving average of the gradients. Inputs: \"v, grads, beta1\". Output: \"v\".\n",
        "        v[\"dW\" + str(l+1)] = beta1 * v['dW' + str(l+1)] + (1 - beta1) * grads['dW' + str(l+1)]\n",
        "        v[\"db\" + str(l+1)] = beta1 * v['db' + str(l+1)] + (1 - beta1) * grads['db' + str(l+1)]\n",
        "\n",
        "        # Compute bias-corrected first moment estimate. Inputs: \"v, beta1, t\". Output: \"v_corrected\".\n",
        "        v_corrected[\"dW\" + str(l+1)] = v['dW' + str(l+1)] / float(1 - beta1**t)\n",
        "        v_corrected[\"db\" + str(l+1)] = v['db' + str(l+1)] / float(1 - beta1**t)\n",
        "\n",
        "        # Moving average of the squared gradients. Inputs: \"s, grads, beta2\". Output: \"s\".\n",
        "        s[\"dW\" + str(l+1)] = beta2 * s['dW' + str(l+1)] + (1 - beta2) * (grads['dW' + str(l+1)]**2)\n",
        "        s[\"db\" + str(l+1)] = beta2 * s['db' + str(l+1)] + (1 - beta2) * (grads['db' + str(l+1)]**2)\n",
        "          ### END CODE HERE ###\n",
        "\n",
        "        # Compute bias-corrected second raw moment estimate. Inputs: \"s, beta2, t\". Output: \"s_corrected\".  \n",
        "        s_corrected[\"dW\" + str(l+1)] = s[\"dW\" + str(l+1)] / float(1 - beta2**t)\n",
        "        s_corrected[\"db\" + str(l+1)] = s[\"db\" + str(l+1)] / float(1 - beta2**t)\n",
        "\n",
        "        # Update parameters. Inputs: \"parameters, learning_rate, v_corrected, s_corrected, epsilon\". Output: \"parameters\".\n",
        "        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * v_corrected[\"dW\" + str(l+1)] / (np.sqrt(s_corrected[\"dW\" + str(l+1)]) + epsilon)\n",
        "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * v_corrected[\"db\" + str(l+1)] / (np.sqrt(s_corrected[\"db\" + str(l+1)]) + epsilon)\n",
        "    \n",
        "    return parameters, v, s"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "O_5mhO8-w4l5",
        "colab": {}
      },
      "source": [
        "def random_mini_batches(X, Y, mini_batch_size):\n",
        "    \"\"\"\n",
        "    This function creates a list of random minibatches from (X, Y)\n",
        "    \n",
        "    @params\n",
        "    \n",
        "    X = Data, numpy array of shape (number of examples, number of features)\n",
        "    Y = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "        shape = (number of examples, 1)\n",
        "    mini_batch_size = size of the mini-batches (suggested to use powers of 2)\n",
        "    \n",
        "    Returns:\n",
        "    \n",
        "    mini_batches = list of synchronous (mini_batch_X, mini_batch_Y)\n",
        "    \n",
        "    \"\"\"\n",
        "    \n",
        "    np.random.seed(0)            \n",
        "    m = X.shape[0]                  # Number of training examples\n",
        "    mini_batches = []               # List to return synchronous minibatches\n",
        "        \n",
        "    # Step 1: Shuffle (X, Y)\n",
        "    permutation = list(np.random.permutation(m))\n",
        "    shuffled_X = X[permutation,:]\n",
        "    #print(\"S_X\",shuffled_X.shape)\n",
        "    shuffled_Y = Y[permutation].reshape((m,1))\n",
        "    #print(\"S_Y\",shuffled_Y.shape)\n",
        "    \n",
        "    # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case.\n",
        "    num_complete_minibatches = (m//mini_batch_size) # number of mini batches of size mini_batch_size in our partitionning\n",
        "    for k in range(num_complete_minibatches):\n",
        "        mini_batch_X = shuffled_X[k*mini_batch_size : (k+1)*mini_batch_size,:]\n",
        "        #print(\"M_X\",mini_batch_X.shape)\n",
        "        mini_batch_Y = shuffled_Y[k*mini_batch_size : (k+1)*mini_batch_size,:]\n",
        "        mini_batch = (mini_batch_X, mini_batch_Y)   # Tuple for synchronous minibatches\n",
        "        mini_batches.append(mini_batch)\n",
        "    \n",
        "    # Handling the end case (last mini-batch < mini_batch_size)\n",
        "    if m % mini_batch_size != 0:\n",
        "        mini_batch_X = shuffled_X[num_complete_minibatches*mini_batch_size :,: ]\n",
        "        mini_batch_Y = shuffled_Y[num_complete_minibatches*mini_batch_size :,: ]\n",
        "        mini_batch = (mini_batch_X, mini_batch_Y)\n",
        "        mini_batches.append(mini_batch)\n",
        "    \n",
        "    return mini_batches"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KTjs1EN4w655",
        "colab": {}
      },
      "source": [
        "def L_layer_model(X, Y, Output_classes,layers_dims, activation_type, reg_type, keep_prob=0.5,learning_rate = 0.01, mini_batch_size = 128,n=1,lambd=0.7,\n",
        "          beta1 = 0.9, beta2 = 0.999,  epsilon = 1e-8, num_epochs = 10000, predict_result=False,print_cost = True): #lr was 0.009\n",
        "    \"\"\"\n",
        "    This function implements a L-layer neural network: \n",
        "    [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID\n",
        "    \n",
        "    Arguments:\n",
        "    X = Data, numpy array of shape (number of examples, number of features)\n",
        "    Y = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "        shape = (number of examples, 1)\n",
        "    layers_dims = List contains number of neurons in one respective layer\n",
        "                  and [len(layer_dims) - 1] gives L Layer Neural Network\n",
        "    activation_type = The activation to be used in this layer, \n",
        "        stored as a text string: \"bianry\" or \"multiclass\"\n",
        "    reg_type = Type of regularization to use \"l1\" or \"l2\"\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    learning_rate = learning rate of the gradient descent update rule\n",
        "    n = 1 or 2, used for random initialization of weights, when \n",
        "        n = 1, we get LeCun Initializer\n",
        "        n = 2, we get He Initializer\n",
        "    lambd = Regularization parameter, int\n",
        "    num_epochs = number of epochs\n",
        "    print_cost = if True, it prints the cost every 10 steps\n",
        "    \n",
        "    Returns:\n",
        "    parameters = parameters learnt by the model. They are used during prediction\n",
        "    \"\"\"\n",
        "    np.random.seed(1)\n",
        "    costs = []                         # keep track of cost\n",
        "    t = 0                              # Used in Adam\n",
        "    \n",
        "    # Parameters initialization.\n",
        "    parameters = initialize_parameters_deep(layers_dims,n)\n",
        "    v, s = initialize_adam(parameters)\n",
        "    \n",
        "    # MiniBatch Gradient Descent\n",
        "    for i in range(num_epochs):\n",
        "        minibatches = random_mini_batches(X, Y, mini_batch_size)\n",
        "        for minibatch in minibatches:\n",
        "            # Select a minibatch\n",
        "            (minibatch_X, minibatch_Y) = minibatch\n",
        "\n",
        "            # Forward propagation: [LINEAR -> RELU]*(L-1) -> LINEAR -> SIGMOID.\n",
        "            if activation_type == \"binary\":\n",
        "                AL, caches = L_model_forward(minibatch_X, parameters, minibatch_Y,Output_classes, keep_prob,predict_result, activation_type)\n",
        "                # Compute cost\n",
        "                cost = compute_cost(AL, minibatch_Y, parameters, lambd,0,reg_type,activation_type)\n",
        "                #print(cost)\n",
        "                # Backward propagation\n",
        "                grads = L_model_backward(AL, minibatch_Y, caches,keep_prob,lambd,reg_type,activation_type)\n",
        "                # Update parameters as per Adam\n",
        "                t += 1\n",
        "                parameters, v, s = update_parameters_with_adam(parameters, grads, v, s, t, learning_rate, beta1, beta2,  epsilon)\n",
        "\n",
        "            elif activation_type == \"multiclass\":\n",
        "                AL, caches, log_loss, dZ = L_model_forward(minibatch_X, parameters,minibatch_Y,Output_classes,keep_prob,predict_result, activation_type)\n",
        "                # Compute cost\n",
        "                cost = compute_cost(AL, minibatch_Y, parameters, lambd, log_loss,reg_type,activation_type)\n",
        "                #print(cost)\n",
        "                # Backward propagation\n",
        "                grads = L_model_backward(AL, minibatch_Y, caches,keep_prob,lambd,reg_type, activation_type)\n",
        "                # Update parameters as per Adam\n",
        "                t += 1\n",
        "                parameters, v, s = update_parameters_with_adam(parameters, grads, v, s, t, learning_rate, beta1, beta2,  epsilon)\n",
        "\n",
        "            \n",
        "        # Print the cost every 10 training example\n",
        "        if print_cost and i % 10 == 0:\n",
        "            print (\"Cost after iteration %i: %f\" % (i, cost))\n",
        "            #print(\"AL and Y\",(AL, minibatch_Y))\n",
        "        if print_cost and i % 10 == 0:\n",
        "            costs.append(cost)\n",
        "            \n",
        "    # plot the cost\n",
        "    plt.plot(np.squeeze(costs))\n",
        "    plt.ylabel('cost')\n",
        "    plt.xlabel('iterations (per tens)')\n",
        "    plt.title((\"Learning rate = {}, Lambda = {} \".format(str(learning_rate),str(lambd))))\n",
        "    plt.show()\n",
        "    \n",
        "    return parameters"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XlP7ejqPw9A4",
        "colab": {}
      },
      "source": [
        "def predict(X, parameters,y,Output_classes, keep_prob,predict_result,activation_type, flags):\n",
        "    \n",
        "    \"\"\"\n",
        "    This function is used to predict the results of a  L-layer neural network\n",
        "    \n",
        "    @ params\n",
        "\n",
        "    X = Data, numpy array of shape (number of examples, number of features)\n",
        "    Y = Ground Truth/ True \"label\" vector (containing classes 0 and 1) \n",
        "        shape = (number of examples, 1)\n",
        "    parameters = Parameters of trained model returned by L_layer_model function\n",
        "    keep_prob = Percentage of neurons to be kept active \n",
        "    predict_result = False while training, True when predicting the ground truth \n",
        "                     values (False only when ground truth values are present)\n",
        "                     Must be kept False if you have ground truth values\n",
        "                     while predicting\n",
        "    activation_type = The activation to be used in this layer, \n",
        "                      stored as a text string: \"bianry\" or \"multiclass\" \n",
        "    flags = During prediction sometime we have grounnd truth values and \n",
        "            sometime we have to predict ground truth values using learned \n",
        "            parameters during training.\n",
        "            so flags is \"y_is_present\" or \"predict_y\"\n",
        "    Returns:\n",
        "\n",
        "    Predictions for the given dataset X\n",
        "    \"\"\"\n",
        "    \n",
        "    m = X.shape[0] # Number of training examples in Dataset\n",
        "    n = len(parameters) // 2 # Number of layers in the neural network\n",
        "    \n",
        "    \"\"\"if activation_type==\"multiclass\":\n",
        "        if flags==\"y_is_present\":\n",
        "            # Forward propagation\n",
        "            AL, _, _, _ = L_model_forward(X, parameters, y,Output_classes,keep_prob, predict_result,activation_type)\n",
        "        elif flags == \"predict_y\":\n",
        "            Z = L_model_forward(X, parameters, y,Output_classes,keep_prob, predict_result,activation_type)\n",
        "            AL = softmax(Z)   # Apply stable Softmax \n",
        "\n",
        "        predicted_class = np.argmax(AL, axis=1) # Prediction\"\"\"\n",
        "        \n",
        "    \n",
        "        #p = np.zeros((m,1))\n",
        "        #Forward Propagation\n",
        "    y_pred, _ = L_model_forward(X, parameters,y,Output_classes,keep_prob, predict_result,activation_type)\n",
        "    \"\"\"for i in range(y_pred.shape[0]):\n",
        "    # As per sigmoid, values greater than 0.5 are categorized as 1\n",
        "    # and values lesser than 0.5 as categorized as 0\n",
        "        if y_pred[i] > 0.5:\n",
        "            p[i] = 1\n",
        "        else:\n",
        "            p[i] = 0\"\"\"\n",
        "    \n",
        "    \"\"\"if flags == \"y_is_present\" and activation_type==\"multiclass\":\n",
        "        acc = np.sum((predicted_class == y)/m)*100\n",
        "        print(\"Accuracy:%.2f%%\" % acc)\n",
        "        #print('Accuracy: {0}%'.format(100*np.mean(predicted_class == y)))\n",
        "        return predicted_class\"\"\"\n",
        "    if flags == \"y_is_present\" and activation_type==\"binary\":\n",
        "        y = y.reshape(y_pred.shape)\n",
        "        acc = np.sum((y_pred == y)/m)*100\n",
        "        print(\"Accuracy\", acc)\n",
        "        return y_pred\n",
        "        \n",
        "    \"\"\"if flags == \"predict_y\" and activation_type==\"multiclass\":\n",
        "        ret = np.column_stack((y, predicted_class)).astype(int)\n",
        "        # Saving the Predictions as Multiclass_Predictions.csv  \n",
        "        pd.DataFrame(ret).to_csv(\"Multiclass_Predictions.csv\", sep = \",\", header = [\"Id\", \"label\"], index = False)\n",
        "        return predicted_class\"\"\"\n",
        "    \n",
        "    if flags == \"predict_y\" and activation_type==\"binary\":\n",
        "        ret = np.column_stack((y, y_pred)).astype(int)\n",
        "        # Saving the Predictions as Binary_Predictions.csv\n",
        "        pd.DataFrame(ret).to_csv(\"Binary_Predictions.csv\", sep = \",\", header = [\"Id\", \"label\"], index = False)\n",
        "        return y_pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wsB4YJ-ZuNSt",
        "colab": {}
      },
      "source": [
        "#Output_classes = np.unique(y).shape[0]\n",
        "#Output_classes"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4xiTp5VBxAP1",
        "scrolled": true,
        "colab": {}
      },
      "source": [
        "#l_parameters = L_layer_model(X, y,Output_classes, layers_dims=[X.shape[1],128,128,Output_classes], predict_result=False,activation_type=\"multiclass\", reg_type=\"l2\",keep_prob=0.7, mini_batch_size=64, n=1, learning_rate = 0.005,lambd=0.01, num_epochs =100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fQVBQ7dcxEbR",
        "outputId": "ddc46bd8-82f0-4549-abde-fd8e31836ba6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\"\"\"pred = predict(X, l_parameters, y,Output_classes,keep_prob=1,predict_result=False, activation_type=\"multiclass\" ,flags=\"y_is_present\")\n",
        "print(np.unique(pred), pred.shape)\n",
        "pred\"\"\""
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'pred = predict(X, l_parameters, y,Output_classes,keep_prob=1,predict_result=False, activation_type=\"multiclass\" ,flags=\"y_is_present\")\\nprint(np.unique(pred), pred.shape)\\npred'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QSBhn1SidTo4",
        "colab": {}
      },
      "source": [
        "#print(classification_report(y, pred))\n",
        "#print(confusion_matrix(y, pred))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WEW3FqKRXu3q",
        "outputId": "98fbe766-211e-4b8d-99c5-36ca0e75bbe8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "\"\"\"pred_test = predict(X_test, l_parameters, ret_X,Output_classes,keep_prob=1, predict_result=True,activation_type=\"multiclass\" ,flags=\"predict_y\")\n",
        "print(np.unique(pred_test), pred_test.shape)\n",
        "pred_test\"\"\""
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'pred_test = predict(X_test, l_parameters, ret_X,Output_classes,keep_prob=1, predict_result=True,activation_type=\"multiclass\" ,flags=\"predict_y\")\\nprint(np.unique(pred_test), pred_test.shape)\\npred_test'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aqLLbWN-a4H0",
        "colab": {}
      },
      "source": [
        "#print(pred_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SLc4Jz0Zb90I",
        "outputId": "60294e7c-e472-4ce9-ba12-02f6489ef97b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\"\"\"KNN_pred = pd.read_csv('Predictions.csv')\n",
        "KNN_pred = KNN_pred.iloc[:,1].values\n",
        "print(KNN_pred.shape)\"\"\""
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"KNN_pred = pd.read_csv('Predictions.csv')\\nKNN_pred = KNN_pred.iloc[:,1].values\\nprint(KNN_pred.shape)\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qhUTwZlLdOL3",
        "outputId": "48b83ff6-8222-4d21-cd26-8be9866269cc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\"\"\"my_pred = pd.read_csv('Multiclass_Predictions.csv')\n",
        "my_pred = my_pred['label'].values\n",
        "print(my_pred.shape)\"\"\""
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"my_pred = pd.read_csv('Multiclass_Predictions.csv')\\nmy_pred = my_pred['label'].values\\nprint(my_pred.shape)\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aSo1a-PRdOUB",
        "colab": {}
      },
      "source": [
        "#print(\"Matching Results\",np.sum((my_pred == KNN_pred)/X_test.shape[0])*100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "1WWNToMR6hsS",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}