DataHelper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Helper methods are very specific to p3.\n",
    "Created by Uki D. Lucas on Feb. 4, 2017\n",
    "\"\"\"\n",
    "\n",
    "should_run_tests = False\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def predict_class(predictions, classes):\n",
    "    percentages = predictions[0]\n",
    "\n",
    "    # reverse sort by value, return indexes\n",
    "    sorted_indexes = (-percentages).argsort()\n",
    " \n",
    "    for index in sorted_indexes[:1]:\n",
    "        print(\"predicted class:\", classes[index], \n",
    "              \"\\t\",  np.round(percentages[index]*100,1), \"%\") \n",
    "    \n",
    "    # right now we will return top prediction\n",
    "    predicted_class = classes[sorted_indexes[0]]\n",
    "    return predicted_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# TODO implement batch_from, batch_to - I did not need it for 8037 rows\n",
    "# TODO implement has_header_row\n",
    "def read_csv(file_path):\n",
    "    \"\"\"\n",
    "    Usage:\n",
    "    headers, data = read_csv(file_path)\n",
    "    Parameter: \n",
    "    - file_path: can be relative path \"../../../DATA/stuff.csv\"\n",
    "    Returns:\n",
    "    - headers: array of strings e.g. ['steering', 'throttle', 'brake', 'speed']\n",
    "    - data: array of strings, you have to convert values to int, float yourself\n",
    "   test_read_csv()\n",
    "    \"\"\"\n",
    "    import csv\n",
    "    # Opening spreadsheet to read in TEXT mode: 'rt'\n",
    "    with open(file_path, 'rt') as csvfile:\n",
    "        # Most common format of CSV, TODO improve\n",
    "        payload = csv.reader(csvfile, delimiter=',', quotechar='\"') \n",
    "        row_counter = 0\n",
    "        headers = []\n",
    "        data = []\n",
    "        for row in payload:\n",
    "            row_counter = row_counter + 1\n",
    "            \n",
    "            if row_counter == 1:\n",
    "                headers = row\n",
    "                # print (type(row))\n",
    "                # print ( row)\n",
    "                # print ('\\t '.join(row))\n",
    "            elif 1 < row_counter < 3:\n",
    "                # print (type(row))\n",
    "                # print ( row)\n",
    "                # print ('\\t '.join(row))\n",
    "                data.append(row)\n",
    "            else:\n",
    "                data.append(row)\n",
    "        print(\"Number of imported CSV rows:\", row_counter)\n",
    "        # I am returning data as numpy array instead of list\n",
    "        # because it is handier to use it.\n",
    "        return headers, np.array(data)\n",
    "    \n",
    "    \n",
    "    \n",
    "def test_read_csv():\n",
    "    \"\"\"\n",
    "    This test is specific to Uki's enviroment.\n",
    "    \"\"\"\n",
    "    data_dir = \"../../../DATA/behavioral_cloning_data/\"\n",
    "    headers, data = read_csv(file_path = data_dir + 'driving_log.csv')\n",
    "    print(\"headers \\n\",headers)\n",
    "    print(\"3rd row of data \\n\",data[2:3])\n",
    "\n",
    "if should_run_tests:    \n",
    "    test_read_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_steering_classes(number_of_classes = 41):\n",
    "    steering_classes = np.linspace(-1, 1, num=number_of_classes, endpoint=True, dtype=np.float32) \n",
    "    steering_classes = np.sort(steering_classes)\n",
    "    return steering_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# snapping actual values to given labels\n",
    "\n",
    "def find_nearest(array,value):\n",
    "    idx = (np.abs(array-value)).argmin()\n",
    "    return array[idx]\n",
    "\n",
    "if should_run_tests:  \n",
    "    assert (find_nearest(steering_labels, -0.951) == -1.00),\"method find_nearest() has problem\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "    \n",
    "def split_random(matrix, percent_train=70, percent_test=15):\n",
    "    \"\"\"\n",
    "    Splits matrix data into randomly ordered sets \n",
    "    grouped by provided percentages.\n",
    "    \n",
    "    Usage:\n",
    "    rows = 100\n",
    "    columns = 2\n",
    "    matrix = np.random.rand(rows, columns)\n",
    "    training, testing, validation = \\\n",
    "    split_random(matrix, percent_train=80, percent_test=10)\n",
    "    \n",
    "    percent_validation 10\n",
    "    training (80, 2)\n",
    "    testing (10, 2)\n",
    "    validation (10, 2)\n",
    "    \n",
    "    Returns:\n",
    "    - training_data: percentage_train e.g. 70%\n",
    "    - testing_data: percent_test e.g. 15%\n",
    "    - validation_data: reminder from 100% e.g. 15%\n",
    "    Created by Uki D. Lucas on Feb. 4, 2017\n",
    "    \"\"\"\n",
    "    #print(matrix)  \n",
    "    row_count = matrix.shape[0]\n",
    "    np.random.shuffle(matrix)\n",
    "    \n",
    "    end_training = int(math.ceil(row_count*percent_train/100))    \n",
    "    end_testing = end_training + int(math.ceil((row_count * percent_test/100)))\n",
    "    \n",
    "    percent_validation = 100 - percent_train - percent_test\n",
    "    \n",
    "    training = matrix[:end_training]\n",
    "    testing = []\n",
    "    validation = []\n",
    "    \n",
    "    if percent_validation < 0:\n",
    "        print(\"Make sure that the provided sum of \" + \\\n",
    "        \"training and testing percentages is equal, \" + \\\n",
    "        \"or less than 100%.\")\n",
    "        \n",
    "        testing = matrix[end_training:]\n",
    "    else:\n",
    "        print(\"percent_validation\", percent_validation)\n",
    "        \n",
    "        testing = matrix[end_training:end_testing]\n",
    "        validation = matrix[end_testing:]\n",
    "    \n",
    "    return training, testing, validation\n",
    "\n",
    "# TEST:\n",
    "\n",
    "def test_split_random():\n",
    "    rows = 8037\n",
    "    columns = 2\n",
    "    matrix = np.random.rand(rows, columns)\n",
    "    training, testing, validation = split_random(matrix, percent_train=80, percent_test=20) \n",
    "\n",
    "    print(\"training\",training.shape)\n",
    "    print(\"testing\",testing.shape)\n",
    "    print(\"validation\",validation.shape)\n",
    "    \n",
    "    print(\"sum\",training.shape[0] + testing.shape[0])\n",
    "    \n",
    "    #print(split_random.__doc__)\n",
    "if should_run_tests:  \n",
    "    test_split_random()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_image_center_values(matrix):\n",
    "    data = [row[0] for row in matrix]\n",
    "    return np.array(data)\n",
    "\n",
    "def get_image_left_values(matrix):\n",
    "    data = [row[1] for row in matrix]\n",
    "    return np.array(data)\n",
    "\n",
    "def get_image_right_values(matrix):\n",
    "    data = [row[2] for row in matrix]\n",
    "    return np.array(data)\n",
    "\n",
    "def get_steering_values(matrix):\n",
    "    data = [float(row[3]) for row in matrix]\n",
    "    return np.array(data).astype('float32')\n",
    "\n",
    "def get_throttle_values(matrix):\n",
    "    data = [float(row[4]) for row in matrix]\n",
    "    return np.array(data)\n",
    "\n",
    "def get_brake_values(matrix):\n",
    "    data = [float(row[5]) for row in matrix]\n",
    "    return np.array(data)\n",
    "\n",
    "def get_speed_values(matrix):\n",
    "    data = [float(row[6]) for row in matrix]\n",
    "    return np.array(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# for custom metrics\n",
    "\n",
    "\n",
    "def mean_pred(y_true, y_pred):\n",
    "    return K.mean(y_pred)\n",
    "\n",
    "def false_rates(y_true, y_pred):\n",
    "    false_neg = ...\n",
    "    false_pos = ...\n",
    "    return {\n",
    "        'false_neg': false_neg,\n",
    "        'false_pos': false_pos,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sort_unique_floats(array_x):\n",
    "    # assure that the array is numpy and numerical\n",
    "    #array_x = array_x.astype(np.float)\n",
    "    \n",
    "    # get unique values, a.k.a. set of values\n",
    "    labels_set = set(array_x)\n",
    "    #print(\"labels_set\\n\", labels_set)\n",
    "    \n",
    "    # set is not sorted, so convert it to a numpy array\n",
    "    unique_labels = np.array(list(labels_set))\n",
    "    #print(\"unique_labels\\n\", unique_labels.shape, unique_labels)\n",
    "    \n",
    "    sorted_unique_labels = np.sort(unique_labels)\n",
    "    #print(\"sorted_unique_labels\\n\", sorted_unique_labels.shape, sorted_unique_labels)\n",
    "    return sorted_unique_labels\n",
    "    \n",
    "# TEST   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def locate_one_hot_position(defined_classes, actual_label):\n",
    "    \n",
    "    #print(\"actual_label\", actual_label, type(actual_label))\n",
    "    matched_label = float( find_nearest(defined_classes, actual_label))\n",
    "    \n",
    "    #print(\"matched_label\", matched_label, type(matched_label))\n",
    "    found_at = np.where(defined_classes == matched_label)\n",
    "    \n",
    "    # (array([10]),) <class 'tuple'>\n",
    "    found_at = found_at[0][0] # first element of first vector\n",
    "    #print(\"found_at\", found_at, type(found_at) )\n",
    "    # returns array of locations\n",
    "    return found_at\n",
    "\n",
    "def encode_one_hot(defined_classes, sample_labels):\n",
    "    \"\"\"\n",
    "    Creates one hot encoded vector from a list {1D vector (None,)} containing training labels.\n",
    "    - find all unique labels\n",
    "    - count all unique labels\n",
    "    - create a zero filed array, size equal to count of all unique labels\n",
    "    - order the unique values (small to large)\n",
    "    - create empty output matrix\n",
    "    - for each sample's label create zero vector and set one in position of that label\n",
    "    Created by Uki D. Lucas\n",
    "    \"\"\"\n",
    "    defined_classes = sort_unique_floats(defined_classes)\n",
    "    \n",
    "    # possible float rounding errors\n",
    "    defined_classes = np.round_(defined_classes, decimals=1)\n",
    "    print(\"defined_classes\\n\", defined_classes)\n",
    "    \n",
    "    class_count = len(defined_classes)\n",
    "    print(\"class_count:\", class_count)\n",
    "    \n",
    "    sample_count = len(sample_labels)\n",
    "    print(\"sample_count:\", sample_count)\n",
    "    \n",
    "    one_hot = np.zeros(shape=(sample_count, class_count), dtype=np.int)\n",
    "     \n",
    "    for index in range(sample_count): \n",
    "        actual_label = float(sample_labels[index])\n",
    "        # find first index of actual_label\n",
    "        found_at = locate_one_hot_position(defined_classes, actual_label)\n",
    "        #print(\"found\", index, type(index) )\n",
    "        one_hot[index][found_at] = 1\n",
    "        #print(\"one_hot[index]\", one_hot[index])\n",
    "    print(\"one_hot examples \\n\", one_hot[0:3])\n",
    "    print(\"one_hot shape\", one_hot.shape) \n",
    "    return one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def show_layers(model):\n",
    "    for i in range(len(model.layers)):\n",
    "        layer = model.layers[i]\n",
    "        print(i, \") \",layer.name, \"\\t\\t is trainable: \", layer.trainable)\n",
    "        #layer.trainable = False\n",
    "    return len(model.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "#import random\n",
    "import math\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def margin(value):\n",
    "    return value*(5/100)\n",
    "\n",
    "def plot_histogram(name, values, change_step):\n",
    "    \n",
    "    min_value = min(values)\n",
    "    max_value = max(values)\n",
    "    print(\"min_value\", min_value, \"max_value\", max_value)\n",
    "    \n",
    "    spread = max_value - min_value\n",
    "    recommended_bins = math.ceil(spread/change_step)\n",
    "    print(\"spread\", spread, \"in\", recommended_bins, \"bins\")\n",
    "    \n",
    "    bins = np.linspace(math.floor(min(values)), \n",
    "                       math.ceil(max(values)),\n",
    "                       recommended_bins)\n",
    "\n",
    "    plt.xlim([\n",
    "        min_value - margin(min_value), \n",
    "        max_value + margin(max_value)])\n",
    "\n",
    "    plt.hist(values, bins=bins, alpha=0.5)\n",
    "    plt.title('Distribution of ' + name)\n",
    "    plt.xlabel('values')\n",
    "    plt.ylabel('occurance')\n",
    "    \n",
    "    # RESIZE it nicely for Jupyter Notebook (width = 10)\n",
    "    fig = matplotlib.pyplot.gcf()\n",
    "    fig.set_size_inches(10, 3)\n",
    "    fig.savefig('test2png.png', dpi=72)\n",
    "    plt.margins(0.1)\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_steering_values(values):\n",
    "    \n",
    "    import matplotlib.pyplot as plt\n",
    "    \n",
    "    plt.plot(values, 'b.')\n",
    "\n",
    "    plt.title(\"Distribution of steering value classes.\")\n",
    "    plt.xlabel(\"class number\")\n",
    "    plt.ylabel('steering value')\n",
    "    \n",
    "    # RESIZE it nicely for Jupyter Notebook (width = 10)\n",
    "    fig = matplotlib.pyplot.gcf()\n",
    "    fig.set_size_inches(10, 3)\n",
    "    fig.savefig('test2png.png', dpi=72)\n",
    "    plt.margins(0.1)\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}