LocalisationAndClassification_Loss_functions.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LocalisationAndClassification_Loss_functions.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/srinithish/product-recognition/blob/master/LocalisationAndClassification_Loss_functions.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "5cLd7zC7oI5G",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import sklearn\n",
        "import pandas as pd\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.manifold import TSNE\n",
        "import random\n",
        "import matplotlib.cm as cm\n",
        "import math\n",
        "# import FinalPredictionsProcessing\n",
        "import pickle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0M4ti3alqxU6",
        "colab_type": "code",
        "outputId": "13464efe-0b42-48ec-fb0b-4265cdf7485c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "cell_type": "code",
      "source": [
        "# a = tf.ones([3,2,2],dtype=tf.dtypes.float32)\n",
        "# b = tf.zeros([3,2,2],dtype=tf.dtypes.float32)\n",
        "\n",
        "grows = 3\n",
        "gcols = 3\n",
        "col_id = tf.to_float(tf.reshape(tf.tile(tf.range(grows), [gcols]), (1, grows, gcols, 1)))\n",
        "row_id = tf.to_float(tf.reshape(tf.tile(tf.range(grows), [gcols]), (1, grows, gcols, 1)))\n",
        "row_id = tf.transpose(row_id)\n",
        "coord_id = tf.concat([col_id,row_id],-1)\n",
        "\n",
        "c = tf.multiply(coord_id,[10,10])\n",
        "\n",
        "d = tf.reverse(c,[-1])\n",
        "sess  = tf.InteractiveSession()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-2-925bba77f161>:4: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.cast instead.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "pgO6pKuPlc1l",
        "colab_type": "code",
        "outputId": "26e85521-87f8-4b20-a56b-d644e4175db5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "someTensor = tf.constant([[[1,2],[3,4]]])\n",
        "\n",
        "\n",
        "# tf.reduce\n",
        "\n",
        "tf.reduce_sum(someTensor,axis = 2).eval()\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[3, 7]], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "metadata": {
        "id": "6MGgD7BhrDKu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# https://medium.com/@jonathan_hui/real-time-object-detection-with-yolo-yolov2-28b1b93e2088\n",
        "# https://github.com/thtrieu/darkflow/blob/master/darkflow/net/yolo/train.py\n",
        "# https://hackernoon.com/understanding-yolo-f5a74bbc7967\n",
        "\n",
        "def loss(y_true,y_pred):\n",
        "    '''\n",
        "    y_true.shape => (batch,grows,gcols,params)    \\\\ params = P(obj) + 4 b.b params + each class probabilities\n",
        "    y_pred.shape => y_true.shape\n",
        "        \n",
        "    b.b params => (x,y,h,w)\n",
        "    \n",
        "    There are 3 types of losses:\n",
        "    1. classification loss\n",
        "    2. localization loss\n",
        "    3. confidence loss\n",
        "    \n",
        "    The λ parameters that appear here and also in the first part are used to \n",
        "    differently weight parts of the loss functions. This is necessary to increase \n",
        "    model stability. The highest penalty is for coordinate predictions (λ coord = 5) \n",
        "    and the lowest for confidence predictions when no object is present (λ noobj = 0.5).\n",
        "    '''\n",
        "    \n",
        "    ##### Confidence loss\n",
        "    '''\n",
        "    Most boxes do not contain any objects. This causes a class imbalance problem,\n",
        "    i.e. we train the model to detect background more frequently than detecting \n",
        "    objects. To remedy this, we weight this loss down by a factor λnoobj (default: 0.5).\n",
        "    box_conf_score = P(obj) * (IoU with g.t)\n",
        "    '''\n",
        "    P_obj = y_pred[:,:,:,0]    # Slice probs\n",
        "    \n",
        "    # Calculate iou for all the predicted boxes\n",
        "    # calculate intersection\n",
        "    denorm_coords = Denormalize_coordinates(y_pred[:,:,:,1:3]) # Slice the x,y\n",
        "    true_area,pred_area,interset = Areas(denorm_coords,y_pred[:,:,:,3:5])\n",
        "    \n",
        "    lefttop,rightbot = Corner_coords(denorm_coords,y_pred[:,:,:,3:5]) # Slice the h,w\n",
        "    \n",
        "    \n",
        "    iou = tf.truediv(intersect, true_area + pred_area - intersect)  # intersection/union\n",
        "    \n",
        "    confs = tf.math.multiply(P_obj,iou)\n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6CHjD-A2zgXr",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def Denormalize_coordinates(pred_coords):\n",
        "    '''\n",
        "    Inputs\n",
        "    pred_coords.shape => (batch,grows,gcols,2) # normed x and y of the b.b\n",
        "    \n",
        "    Output:\n",
        "    denormalized_coord.shape => (batch,grows,gcols,2) # denormed x and y of the b.b\n",
        "    \n",
        "    generates center X and center Y denormalised\n",
        "    \n",
        "    \n",
        "    '''\n",
        "    img_height = tf.constant(342)\n",
        "    img_width = tf.constant(342)\n",
        "    \n",
        "    grows = tf.constant(19)\n",
        "    gcols = tf.constant(19)\n",
        "    \n",
        "    cell_height = tf.truediv(img_height,grows)\n",
        "    cell_width = tf.truediv(img_width,gcols)\n",
        "    \n",
        "#     grid_cell_topleft[\"y\"] = grid_row_no*cell_height\n",
        "#     grid_cell_topleft[\"x\"] = grid_col_no*cell_width\n",
        "        \n",
        "    col_id = tf.to_float(tf.reshape(tf.tile(tf.range(grows), [gcols]), (1, grows, gcols, 1)))\n",
        "    row_id = tf.to_float(tf.reshape(tf.tile(tf.range(grows), [gcols]), (1, grows, gcols, 1)))\n",
        "    row_id = tf.transpose(row_id)\n",
        "    coord_id = tf.concat([col_id,row_id],-1) # a 4D tensor that gives the coordinates for every box (col,row)\n",
        "\n",
        "    coords_lefttop = tf.multiply(coord_id,[cell_width,cell_height]) # Gets the leftmost coordinate for every box\n",
        "    cell_denormalized_coords = tf.multiply(pred_coords,[cell_width,cell_height])\n",
        "    \n",
        "    denormalized_coord = tf.add(coords_lefttop ,cell_denormalized_coords)\n",
        "    \n",
        "    return denormalized_coord"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bOrd2d4czhj9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        ""
      ]
    },
    {
      "metadata": {
        "id": "FRXjBuy834t6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_denorm_width_height(normHeightWidth_params):\n",
        "    '''\n",
        "    We have h,w but we want w,h\n",
        "    '''\n",
        "    img_height = tf.constant(342)\n",
        "    img_width = tf.constant(342)\n",
        "    \n",
        "    grows = tf.constant(19)\n",
        "    gcols = tf.constant(19)\n",
        "    \n",
        "    box_params = tf.reverse(normHeightWidth_params,[-1]) # Now w,h\n",
        "    denormHeightWidth_params = tf.multiply(box_params,[img_width,img_height])\n",
        "    \n",
        "    \n",
        "    return denormHeightWidth_params"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wkDBpUjMCiuI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def Areas(denorm_coords,y_true,box_params):\n",
        "    lefttop,rightbot = Corner_coords(denorm_coords,box_params) # Slice the h,w\n",
        "    \n",
        "    \n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "iQlpBu11GjbD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "### will get [[xmin,ymin],[xmin,ymin]]\n",
        "\n",
        "def getLocalisationLoss(yPred,yTrue,lambdaVal):\n",
        "    \n",
        "    objectMask = yTrue[:,:,:,0]\n",
        "    \n",
        "    objectMask2D =  tf.stack([objectMask,objectMask],axis = 3)\n",
        "    \n",
        "    ##for cordinates\n",
        "    pred_denormalized_coord = Denormalize_coordinates(yPred[:,:,:,1:3])\n",
        "    true_denormalized_coord = Denormalize_coordinates(yTrue[:,:,:,1:3])\n",
        "    \n",
        "   \n",
        "    sq_subtract_center = tf.square(true_denormalized_coord-pred_denormalized_coord)\n",
        "    \n",
        "    masked_sq_subtract_center = tf.multiply(sq_subtract_center,objectMask2D)\n",
        "    masked_sq_subtract_center = tf.reduce_sum(masked_sq_subtract_center,axis = 3) ##collapse depth\n",
        "    \n",
        "    flat_center = tf.layers.flatten(masked_sq_subtract_center)\n",
        "    batchwiseLoss_center = tf.reduce_sum(flat_center,axis = 1)\n",
        "    meanLoss_center = tf.reduce_mean(batchwiseLoss_center)\n",
        "    \n",
        "    \n",
        "    \n",
        "    ###for height and width\n",
        "    pred_denorm_heightWidth = get_denorm_width_height(yPred[:,:,:,3:5])\n",
        "    true_denorm_heightWidth = get_denorm_width_height(yTrue[:,:,:,3:5])\n",
        "    \n",
        "    \n",
        "    sq_subtract_sqrt = tf.square(tf.sqrt(pred_denorm_heightWidth) - tf.sqrt(true_denorm_heightWidth))\n",
        "    \n",
        "    maskedWidthHeight  = tf.multiply(sq_subtract_sqrt,objectMask2D)\n",
        "    \n",
        "    flat_widthHeight = tf.layers.flatten(maskedWidthHeight)\n",
        "    batchwiseLoss_widthHeight = tf.reduce_sum(flat_widthHeight,axis = 1)\n",
        "    meanLoss_widthHeight = tf.reduce_mean(batchwiseLoss_widthHeight)\n",
        "    \n",
        "    \n",
        "    \n",
        "    \n",
        "    \n",
        "    \n",
        "    return lambdaVal*(meanLoss_widthHeight+meanLoss_center)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "okOK3kZR-QlV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_classificationLoss(yPred,yTrue,lambdaVal):\n",
        "    \n",
        "    \n",
        "    objectMask = yTrue[:,:,:,0]\n",
        "    \n",
        "    objectMask2D =  tf.stack([objectMask,objectMask],axis = 3)\n",
        "    \n",
        "    trueClassProbabilities = yTrue[:,:,:,5:] ##gets all class probabilities\n",
        "    predClassProbabilities = yTrue[:,:,:,5:] ###\n",
        "    \n",
        "    sq_subtract = tf.square(tf.subtract(trueClassProbabilities,predClassProbabilities))\n",
        "    sq_subtract = tf.reduce_sum(sq_subtract,axis = 3) ##collapse along depth\n",
        "    \n",
        "    masked_sq_subtract = tf.multiply(sq_subtract,objectMask)\n",
        "    \n",
        "    flat = tf.layers.flatten(masked_sq_subtract)\n",
        "    batchwiseLoss = tf.reduce_sum(flat,axis = 1)\n",
        "    meanLoss = tf.reduce_mean(batchwiseLoss)\n",
        "    \n",
        "    return lambdaVal*meanLoss    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BhO_BNdZpnVj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dirpathForTrainingFiles  = \"/content/Drive/My Drive/Deeplearning Project/Reshaped/\"\n",
        "\n",
        "\n",
        "\n",
        "annotationsPath = dirpathForTrainingFiles + 'annotations/'\n",
        "imagesPath = dirpathForTrainingFiles+ 'images/'\n",
        "\n",
        "###for image as numpy array X\n",
        "imagesAsArrayPklPath =  annotationsPath+'resizedAllImgArray.pkl'\n",
        "###for true Y\n",
        "targetVariablePklPath = annotationsPath+'resizedAllTargetArray.pkl'\n",
        "\n",
        "### for visualising\n",
        "\n",
        "imagefilePattern = imagesPath+'*'\n",
        "ImgDictsPath_True_Path = annotationsPath+'resizedImageDictsAllFiles.pkl'\n",
        "ObjLists_True_Path = annotationsPath+'resizedObjectListsAllFiles.pkl'\n",
        "predictionArrayPath = annotationsPath+'PredictionArray.pkl'\n",
        "\n",
        "\n",
        "TrainX = np.array(pickle.load(open(imagesAsArrayPklPath, 'rb')))\n",
        "\n",
        "TrainY = np.array(pickle.load(open(targetVariablePklPath, 'rb')))\n",
        "predictionsY = np.array(pickle.load(open(predictionArrayPath, 'rb')))\n",
        "predictionsY = predictionsY.astype(np.float32)\n",
        "TrainY = TrainY.astype(np.float32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PZlkL2jN1f_y",
        "colab_type": "code",
        "outputId": "3eb997c5-d55e-422e-eba0-437eab8da1bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "sess  = tf.InteractiveSession()\n",
        "predictionsY = tf.convert_to_tensor(predictionsY)\n",
        "TrainY = tf.convert_to_tensor(TrainY)\n",
        "\n",
        "loss = get_classificationLoss(predictionsY[0],TrainY,0.5)\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-8-8b9291575cb0>:16: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.flatten instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py:1702: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
            "  warnings.warn('An interactive session is already active. This can '\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "EKjHqr9D10Q9",
        "colab_type": "code",
        "outputId": "639dc9b7-71fc-4a99-9e2b-e25ab035fbe7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "loss.eval()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "metadata": {
        "id": "TWBDOv0FRP7w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "349607ea-4b28-4a39-c225-da12000c7e2e"
      },
      "cell_type": "code",
      "source": [
        "TrainY[0,3,7,:].eval()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0., 0., 0., 0., 0., 0.], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "metadata": {
        "id": "yrkLtwMn2uFk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "539bff4f-0abe-4c70-ee3b-b48ca908f2e4"
      },
      "cell_type": "code",
      "source": [
        "predictionsY[0,3,7,:].eval()"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.        , 1.        , 0.39699072, 1.        , 1.        ,\n",
              "       1.        ], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "metadata": {
        "id": "irWNIbJaTLmg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}