Range_Doppler_and_Spectrogram_CNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Range_Doppler_and_Spectrogram_CNN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mertege/FMCW-Data-Classification-/blob/main/Range_Doppler_and_Spectrogram_CNN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kN4ixvH39HW6"
      },
      "source": [
        "# Burada range-doppler ve spectrogram dataları CNN için birlikte kullanıldı.\n",
        "# \"mixup\" yöntemi kullanılarak data 5 kere augmente edildi. Toplam 6 kat data train için kullanıldı.\n",
        "# 5 defa run edildi ve değerler aşağıda paylaşıldı.\n",
        "# Mean test accuracy is 0.95, mean test f1 score is 0.94, max test accuracy is 0.96, max test f1 score is 0.95,\n",
        "# Min test accuracy is 0.94, min test f1 score is 0.93, std of test accuracy is 0.01, std of test f1 score is 0.01"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lzJwN6GfkN8Z"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from keras.models import Model\n",
        "from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization, Normalization, Input, Conv2D, MaxPooling2D, Concatenate\n",
        "import tensorflow as tf\n",
        "import scipy.io\n",
        "import numpy as np\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "from numpy.random import seed\n",
        "from sklearn.model_selection import KFold, StratifiedKFold\n",
        "import time\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras import backend as K \n",
        "import gc"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8FERVnWrkrwE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "86b11a09-79bd-4fb1-da28-3aa3fa3359d0"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o19JHieQ81cS"
      },
      "source": [
        "# Get Range-Doppler data from\n",
        "range_doppler_fast_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_fast_resized.mat')\n",
        "range_doppler_fast_resized = range_doppler_fast_resized['range_doppler_fast_resized']\n",
        "range_doppler_fast_resized = np.transpose(range_doppler_fast_resized, (2, 0, 1))\n",
        "range_doppler_fast_resized = np.delete(range_doppler_fast_resized,(49), axis=0) # 50th row is deleted since there is no 50th row in spectrogram fast data.\n",
        "range_doppler_fast_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_fast_label.mat')\n",
        "range_doppler_fast_label = range_doppler_fast_label['range_doppler_fast_label']  \n",
        "\n",
        "range_doppler_slow_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_resized.mat')\n",
        "range_doppler_slow_resized = range_doppler_slow_resized['range_doppler_slow_resized']\n",
        "range_doppler_slow_resized = np.transpose(range_doppler_slow_resized, (2, 0, 1))\n",
        "range_doppler_slow_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_label.mat')\n",
        "range_doppler_slow_label = range_doppler_slow_label['range_doppler_slow_label']  \n",
        "\n",
        "range_doppler_slow_pocket_resized = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_slow_pocket_resized.mat')\n",
        "range_doppler_slow_pocket_resized = range_doppler_slow_pocket_resized['range_doppler_slow_pocket_resized']\n",
        "range_doppler_slow_pocket_resized = np.transpose(range_doppler_slow_pocket_resized, (2, 0, 1))\n",
        "range_doppler_pocket_label = scipy.io.loadmat('/content/drive/MyDrive/range_doppler_pocket_label.mat')\n",
        "range_doppler_pocket_label = range_doppler_pocket_label['range_doppler_pocket_label']  \n",
        "# Get Range-Doppler data from\n",
        "spectrogram_fast_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_fast_resized.mat')\n",
        "spectrogram_fast_resized = spectrogram_fast_resized['spectrogram_fast_resized']\n",
        "spectrogram_fast_resized = np.transpose(spectrogram_fast_resized, (2, 0, 1))\n",
        "spectrogram_fast_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_fast_label.mat')\n",
        "spectrogram_fast_label = spectrogram_fast_label['spectrogram_fast_label']  \n",
        "\n",
        "spectrogram_slow_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_resized.mat')\n",
        "spectrogram_slow_resized = spectrogram_slow_resized['spectrogram_slow_resized']\n",
        "spectrogram_slow_resized = np.transpose(spectrogram_slow_resized, (2, 0, 1))\n",
        "spectrogram_slow_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_label.mat')\n",
        "spectrogram_slow_label = spectrogram_slow_label['spectrogram_slow_label']  \n",
        "\n",
        "spectrogram_slow_pocket_resized = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_pocket_resized.mat')\n",
        "spectrogram_slow_pocket_resized = spectrogram_slow_pocket_resized['spectrogram_slow_pocket_resized']\n",
        "spectrogram_slow_pocket_resized = np.transpose(spectrogram_slow_pocket_resized, (2, 0, 1))\n",
        "spectrogram_slow_pocket_label = scipy.io.loadmat('/content/drive/MyDrive/spectrogram_slow_pocket_label.mat')\n",
        "spectrogram_slow_pocket_label = spectrogram_slow_pocket_label['spectrogram_slow_pocket_label']  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFp_PErnMLN2"
      },
      "source": [
        "# Concat range-doppler data\n",
        "range_doppler_concat = np.concatenate((range_doppler_fast_resized,range_doppler_slow_resized),axis=0)\n",
        "range_doppler_concat = np.concatenate((range_doppler_concat,range_doppler_slow_pocket_resized),axis=0)\n",
        "range_doppler_concat = range_doppler_concat[:,:,:,np.newaxis] \n",
        "range_doppler_concat_label = np.zeros((range_doppler_concat.shape[0],1))\n",
        "range_doppler_concat_label[:range_doppler_fast_resized.shape[0],:] = 1\n",
        "# Shuffle concat range doppler\n",
        "shuffle_indx = random.sample(range(0, range_doppler_concat.shape[0]), range_doppler_concat.shape[0]) # split validation data\n",
        "range_doppler_concat_shuffle = range_doppler_concat[shuffle_indx,:,:,:]\n",
        "range_doppler_concat_label_shuffle = range_doppler_concat_label[shuffle_indx,:]\n",
        "# Concat range-doppler data\n",
        "spectrogram_concat = np.concatenate((spectrogram_fast_resized,spectrogram_slow_resized),axis=0)\n",
        "spectrogram_concat = np.concatenate((spectrogram_concat,spectrogram_slow_pocket_resized),axis=0)\n",
        "spectrogram_concat = spectrogram_concat[:,:,:,np.newaxis] \n",
        "spectrogram_concat_label = np.zeros((spectrogram_concat.shape[0],1))\n",
        "spectrogram_concat_label[:spectrogram_fast_resized.shape[0],:] = 1\n",
        "# Shuffle concat range doppler\n",
        "# shuffle_indx = random.sample(range(0, spectrogram_concat.shape[0]), spectrogram_concat.shape[0]) # split validation data\n",
        "spectrogram_concat_shuffle = spectrogram_concat[shuffle_indx,:,:,:]\n",
        "spectrogram_concat_label_shuffle = spectrogram_concat_label[shuffle_indx,:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1YbC9MeVko_"
      },
      "source": [
        "del range_doppler_fast_resized\n",
        "del range_doppler_fast_label\n",
        "del range_doppler_slow_resized\n",
        "del range_doppler_slow_label\n",
        "del range_doppler_slow_pocket_resized\n",
        "del range_doppler_pocket_label\n",
        "del spectrogram_fast_resized\n",
        "del spectrogram_fast_label\n",
        "del spectrogram_slow_resized\n",
        "del spectrogram_slow_label\n",
        "del spectrogram_slow_pocket_resized\n",
        "del spectrogram_slow_pocket_label\n",
        "del range_doppler_concat\n",
        "del range_doppler_concat_label\n",
        "del spectrogram_concat\n",
        "del spectrogram_concat_label"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_1aBNkjoLCvI"
      },
      "source": [
        "# ---------------- Augmente and shuffle (train and test) data data ----------------\n",
        "data_augmentation = tf.keras.Sequential([\n",
        "  layers.RandomFlip(\"horizontal_and_vertical\"),\n",
        "  layers.RandomRotation(0.1),\n",
        "])\n",
        "# def sample_beta_distribution(size, concentration_0=0.2, concentration_1=0.2):\n",
        "#     gamma_1_sample = np.random.gamma(shape=[size], alpha=concentration_1)\n",
        "#     gamma_2_sample = np.random.gamma(shape=[size], alpha=concentration_0)\n",
        "#     return gamma_1_sample / (gamma_1_sample + gamma_2_sample)\n",
        "\n",
        "\n",
        "def mixup_augmentation(images, labels, repeat_of_mixup, alpha=0.2):\n",
        "    batch_size = images.shape[0]\n",
        "    concat_images = np.zeros((batch_size*(repeat_of_mixup+1),images.shape[1],images.shape[2],images.shape[3]))\n",
        "    concat_label = np.zeros((batch_size*(repeat_of_mixup+1),labels.shape[1]))\n",
        "    \n",
        "    for ii in range(repeat_of_mixup):\n",
        "      # shuffle train dataset\n",
        "      shuffle_indx_1 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "      images_shuffled_1 = images[shuffle_indx_1,:,:,:]\n",
        "      labels_shuffled_1 = labels[shuffle_indx_1,:]\n",
        "\n",
        "      shuffle_indx_2 = random.sample(range(0, images.shape[0]), images.shape[0]) # split validation data\n",
        "      images_shuffled_2 = images[shuffle_indx_2,:,:,:]\n",
        "      labels_shuffled_2 = labels[shuffle_indx_2,:]\n",
        "\n",
        "\n",
        "      # Sample lambda and reshape it to do the mixup\n",
        "      x_l = np.random.gamma(0.2, 0.2, (batch_size,1,1,1))\n",
        "      y_l = np.random.gamma(0.2, 0.2, (batch_size,1))\n",
        "      \n",
        "      # Perform mixup on both images and labels by combining a pair of images/labels\n",
        "      # (one from each dataset) into one image/label\n",
        "      images_mixup = images_shuffled_1 * x_l + images_shuffled_2 * (1 - x_l)\n",
        "      labels_mixup = labels_shuffled_1 * y_l + labels_shuffled_2 * (1 - y_l)\n",
        "      concat_images[ii*batch_size:(ii+1)*batch_size,:,:,:] = images_mixup\n",
        "      concat_label[ii*batch_size:(ii+1)*batch_size,:] = labels_mixup\n",
        "      # concat_images = np.concatenate((concat_images,images),axis=0)\n",
        "      # concat_label = np.concatenate((concat_label,labels),axis=0)\n",
        "    concat_images[repeat_of_mixup*batch_size:,:,:,:] = images\n",
        "    concat_label[repeat_of_mixup*batch_size:,:] = labels\n",
        "    return (concat_images, concat_label)\n",
        "\n",
        "def split_and_augmentation_of_training(range_doppler_concat_shuffle_train,range_doppler_concat_label_shuffle_train, randomlist_for_validation_indx,repeat_of_mixup, augmentation_enable):\n",
        "  # ---------------- Parameters ----------------\n",
        "  repeat_of_augmentation_for_fast = 1\n",
        "  repeat_of_augmentation_for_slow = np.floor(repeat_of_augmentation_for_fast/2)\n",
        "  repeat_of_augmentation_for_slow = int(repeat_of_augmentation_for_slow)\n",
        "  \n",
        "  # slow_size_of_validation = 20\n",
        "  # fast_size_of_validation = slow_size_of_validation/2\n",
        "  # fast_size_of_validation = int(fast_size_of_validation)\n",
        "  size_of_validation = 30\n",
        "  alpha = 0.2\n",
        "  # randomlist_for_validation_indx spectrogram ve range-doppler aynı kullanılsın diye bu if state eklendi.\n",
        "  if bool(randomlist_for_validation_indx) == False:\n",
        "    randomlist_for_validation_indx = random.sample(range(0, range_doppler_concat_shuffle_train.shape[0]), size_of_validation) # split validation data\n",
        "    randomlist_for_train_indx = np.delete(range(0, range_doppler_concat_shuffle_train.shape[0]), randomlist_for_validation_indx) # split training data\n",
        "  else:\n",
        "    randomlist_for_validation_indx = randomlist_for_validation_indx # split validation data\n",
        "    randomlist_for_train_indx = np.delete(range(0, range_doppler_concat_shuffle_train.shape[0]), randomlist_for_validation_indx) # split training data\n",
        "  # get validation data\n",
        "  validation_spectrograms = range_doppler_concat_shuffle_train[randomlist_for_validation_indx,:,:,:]\n",
        "  validation_labels = range_doppler_concat_label_shuffle_train[randomlist_for_validation_indx,:]\n",
        "  # get training data\n",
        "  training_spectrograms = range_doppler_concat_shuffle_train[randomlist_for_train_indx,:,:,:]\n",
        "  training_labels = range_doppler_concat_label_shuffle_train[randomlist_for_train_indx,:]\n",
        "\n",
        "  # Rotate Augmentation\n",
        "  # get slow and fast indexes of training data\n",
        "  slow_indexes = np.where(training_labels == 0)[0]\n",
        "  fast_indexes = np.delete(range(0, training_labels.shape[0]), slow_indexes)  \n",
        "\n",
        "  slow_spectrograms_train = training_spectrograms[slow_indexes,:,:,:]\n",
        "  size_of_samples_slow = slow_spectrograms_train.shape[0]\n",
        "\n",
        "  fast_spectrograms_train = training_spectrograms[fast_indexes,:,:,:]  \n",
        "  size_of_samples_fast = fast_spectrograms_train.shape[0]\n",
        "\n",
        "\n",
        "\n",
        "  if augmentation_enable == True: \n",
        "    # ---------------- Augmente Train Data for Fast ----------------\n",
        "    augmented_image_fast = np.zeros((size_of_samples_fast*repeat_of_augmentation_for_fast,fast_spectrograms_train.shape[1],fast_spectrograms_train.shape[2],1))\n",
        "    spectrograms_fast_label = np.ones((size_of_samples_fast*(repeat_of_augmentation_for_fast+1),1))\n",
        "    for jj in range(repeat_of_augmentation_for_fast):\n",
        "      for ii in range(size_of_samples_fast):\n",
        "        augmented_image_fast[size_of_samples_fast*jj+ii,:,:,:] = data_augmentation(fast_spectrograms_train[ii,:,:,:])\n",
        "    augmented_image_fast = np.concatenate((augmented_image_fast,fast_spectrograms_train),axis=0)   \n",
        "    # ---------------- Augmente Train Data for Slow ----------------\n",
        "    augmented_image_slow = np.zeros((size_of_samples_slow*repeat_of_augmentation_for_slow,slow_spectrograms_train.shape[1],slow_spectrograms_train.shape[2],1))\n",
        "    spectrograms_slow_label = np.zeros((size_of_samples_slow*(repeat_of_augmentation_for_slow+1),1))\n",
        "    if repeat_of_augmentation_for_slow == 0:\n",
        "      augmented_image_slow = slow_spectrograms_train\n",
        "    else:\n",
        "      for kk in range(repeat_of_augmentation_for_slow):\n",
        "        for ii in range(size_of_samples_slow):\n",
        "          augmented_image_slow[size_of_samples_slow*kk+ii,:,:,:] = data_augmentation(slow_spectrograms_train[ii,:,:,:])\n",
        "      augmented_image_slow = np.concatenate((augmented_image_slow,slow_spectrograms_train),axis=0)    \n",
        "  else:\n",
        "    augmented_image_fast = fast_spectrograms_train\n",
        "    augmented_image_slow = slow_spectrograms_train\n",
        "    spectrograms_fast_label = np.ones((size_of_samples_fast,1))\n",
        "    spectrograms_slow_label = np.zeros((size_of_samples_slow,1))\n",
        "    spectrogram_augmented_image = training_spectrograms\n",
        "    spectrogram_concat_label_shuffle_concat = training_labels\n",
        "\n",
        "  training_spectrograms = np.concatenate((augmented_image_fast,augmented_image_slow),axis=0)\n",
        "  training_labels = np.concatenate((spectrograms_fast_label,spectrograms_slow_label),axis=0)\n",
        "\n",
        "  (augmented_image_mixup,augmented_label_mixup) = mixup_augmentation(training_spectrograms,training_labels,repeat_of_mixup,alpha )\n",
        "\n",
        "  return (augmented_image_mixup,augmented_label_mixup,validation_spectrograms,validation_labels, randomlist_for_validation_indx)\n",
        "\n",
        "\n",
        "def normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable):\n",
        "  # ---------------- Normalize Inputs ----------------\n",
        "  if normalize_inputs_enable == True:\n",
        "    layer = Normalization(axis=None)\n",
        "    layer.adapt(range_doppler_concat_shuffle)\n",
        "    range_doppler_concat_shuffle = layer(range_doppler_concat_shuffle)\n",
        "  else:\n",
        "    range_doppler_concat_shuffle = range_doppler_concat_shuffle\n",
        "  return(range_doppler_concat_shuffle)\n",
        "# def manuel_normalization(images, normalize_inputs_enable):\n",
        "#   if normalize_inputs_enable == True:\n",
        "#     image_normalized  = np.zeros((images.shape))\n",
        "#     for jj in range(images.shape[0]):\n",
        "#       x = images[jj,:,:,:]\n",
        "#       image_normalized[jj,:,:,:] = x / np.linalg.norm(x)\n",
        "#   else:\n",
        "#     image_normalized = images\n",
        "#   return image_normalized"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z0i1pvvIE4A6"
      },
      "source": [
        "normalize_inputs_enable = 1\n",
        "range_doppler_concat_shuffle = normalize_inputs(range_doppler_concat_shuffle, normalize_inputs_enable)\n",
        "spectrogram_concat_shuffle = normalize_inputs(spectrogram_concat_shuffle, normalize_inputs_enable)\n",
        "range_doppler_concat_shuffle = np.float32(range_doppler_concat_shuffle)\n",
        "spectrogram_concat_shuffle = np.float32(spectrogram_concat_shuffle)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNpkBBIadpPo"
      },
      "source": [
        "t = time.time()\n",
        "# ---------- Parameters ----------------\n",
        "augmentation_enable = True\n",
        "normalize_inputs_enable = True\n",
        "num_folds = 5\n",
        "kfold = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state = None) # random_state = 1 ile split run'dan run'a sabit.\n",
        "test_accuracy_per_run = []\n",
        "f1_score_per_run = []\n",
        "epoch_number = 100\n",
        "batch_size = 32\n",
        "dense_size = 32\n",
        "dropout_prob_cnn = 0.1\n",
        "dropout_prob_dense = 0.3\n",
        "repeat_of_mixup = 5\n",
        "number_of_repeat = 5\n",
        "for repeat_run_number in range(number_of_repeat):\n",
        "  test_accuracy_per_fold = []\n",
        "  f1_score_per_fold = []\n",
        "  if repeat_run_number > 0:\n",
        "    del range_doppler_concat_shuffle_test\n",
        "    del spectrogram_concat_shuffle_test\n",
        "    del range_doppler_augmented_image\n",
        "    del range_doppler_concat_shuffle_train\n",
        "    del spectrogram_concat_shuffle_train\n",
        "    del spectrogram_augmented_image\n",
        "    # tf.keras.backend.clear_session\n",
        "    # del model\n",
        "    # K.clear_session()\n",
        "   \n",
        "  for train, test in kfold.split(range_doppler_concat_shuffle,range_doppler_concat_label_shuffle):   \n",
        "    gc.collect()\n",
        "    K.clear_session()\n",
        "    \n",
        "    # ---------------- Range-Doppler Data ----------------\n",
        "    randomlist_for_test_indx = test\n",
        "    randomlist_for_train_indx = train\n",
        "    # test data\n",
        "    range_doppler_concat_shuffle_test = range_doppler_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    range_doppler_concat_label_shuffle_test = range_doppler_concat_label_shuffle[randomlist_for_test_indx,:]\n",
        "    #train data\n",
        "    range_doppler_concat_shuffle_train = range_doppler_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    range_doppler_concat_label_shuffle_train = range_doppler_concat_label_shuffle[randomlist_for_train_indx,:]\n",
        "      # ---------------- Split labels to equal them during augmentation for Validation ----------------\n",
        "    randomlist_for_validation_indx = []\n",
        "    (range_doppler_augmented_image,range_doppler_concat_label_shuffle_concat,validation_range_doppler,range_doppler_validation_labels, randomlist_for_validation_indx)\\\n",
        "    = split_and_augmentation_of_training(range_doppler_concat_shuffle_train,range_doppler_concat_label_shuffle_train,randomlist_for_validation_indx,repeat_of_mixup, augmentation_enable)\n",
        "    # ---------------- Concat Validation and Slow ----------------\n",
        "\n",
        "\n",
        "    \n",
        "    # ---------------- Spectrogram Data ----------------\n",
        "    # test data\n",
        "    spectrogram_concat_shuffle_test = spectrogram_concat_shuffle[randomlist_for_test_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_test = spectrogram_concat_label_shuffle[randomlist_for_validation_indx,:]\n",
        "    #train data\n",
        "    spectrogram_concat_shuffle_train = spectrogram_concat_shuffle[randomlist_for_train_indx,:,:,:]\n",
        "    spectrogram_concat_label_shuffle_train = spectrogram_concat_label_shuffle[randomlist_for_train_indx,:]\n",
        "      # ---------------- Split labels to equal them during augmentation for Validation ----------------\n",
        "    (spectrogram_augmented_image,spectrogram_concat_label_shuffle_concat,validation_spectrogram,spectrogram_validation_labels, randomlist_for_validation_indx)\\\n",
        "     = split_and_augmentation_of_training(spectrogram_concat_shuffle_train,spectrogram_concat_label_shuffle_train,randomlist_for_validation_indx,repeat_of_mixup, augmentation_enable)\n",
        "    # ---------------- Normalization ----------------\n",
        "    # (range_doppler_concat_shuffle_test, validation_range_doppler, range_doppler_augmented_image) = normalize_inputs(range_doppler_concat_shuffle_test,\\\n",
        "    #                                                                                                  validation_range_doppler, range_doppler_augmented_image, normalize_inputs_enable)   \n",
        "    # (spectrogram_concat_shuffle_test, validation_spectrogram, spectrogram_augmented_image) = normalize_inputs(spectrogram_concat_shuffle_test,\\\n",
        "    #                                                                                                  validation_spectrogram, spectrogram_augmented_image, normalize_inputs_enable)\n",
        "\n",
        "\n",
        "    # range_doppler_concat_shuffle_test = manuel_normalization(range_doppler_concat_shuffle_test, normalize_inputs_enable)\n",
        "    # validation_range_doppler = manuel_normalization(validation_range_doppler, normalize_inputs_enable)\n",
        "    # range_doppler_augmented_image = manuel_normalization(range_doppler_augmented_image, normalize_inputs_enable)\n",
        "    # spectrogram_concat_shuffle_test = manuel_normalization(spectrogram_concat_shuffle_test, normalize_inputs_enable)\n",
        "    # validation_spectrogram= manuel_normalization(validation_spectrogram, normalize_inputs_enable)\n",
        "    # spectrogram_augmented_image= manuel_normalization(spectrogram_augmented_image, normalize_inputs_enable)\n",
        "    # ---------------- Neural Network Architecture ----------------\n",
        "\n",
        "\n",
        "    def encoder_for_range_doppler(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Conv2D(8, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Flatten()(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "    def encoder_for_spectrogram(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Conv2D(8, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(input)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Conv2D(16, (3, 3),padding='same',kernel_initializer=\"glorot_normal\")(x)\n",
        "      x = BatchNormalization()(x)\n",
        "      x = Activation('relu')(x)\n",
        "      x = Dropout(dropout_prob_cnn)(x)\n",
        "      x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "      x = Flatten()(x)\n",
        "      return Model(input, x)\n",
        "\n",
        "    def decoder_for_concat(input_shape):\n",
        "      input = Input(shape=input_shape)\n",
        "      x = Dense(128, activation=\"relu\")(input)\n",
        "      x = Dropout(dropout_prob_dense)(x) \n",
        "      x = Dense(dense_size, activation=\"relu\")(x)\n",
        "      x = Dropout(dropout_prob_dense)(x)\n",
        "      x = Dense(1, activation=\"sigmoid\")(x)\n",
        "      return Model(input, x)\n",
        "    input_shape = range_doppler_concat_shuffle.shape[1:]\n",
        "    base_network_range_doppler = encoder_for_range_doppler(input_shape)\n",
        "    range_doppler_input  = Input(shape=input_shape)\n",
        "    processed_range_doppler  = base_network_range_doppler(range_doppler_input)\n",
        "\n",
        "    input_shape = spectrogram_concat_shuffle.shape[1:]\n",
        "    base_network_spectrogram = encoder_for_spectrogram(input_shape) \n",
        "    spectrogram_input  = Input(shape=input_shape)\n",
        "    processed_spectrogram  = base_network_spectrogram(spectrogram_input)\n",
        "\n",
        "    concat_layer = Concatenate()([processed_range_doppler, processed_spectrogram])\n",
        "\n",
        "    base_decoder_network = decoder_for_concat((concat_layer.shape[1]))\n",
        "    out = base_decoder_network(concat_layer)\n",
        "\n",
        "    model = Model(inputs=[range_doppler_input, spectrogram_input], outputs=[out])\n",
        "\n",
        "   \n",
        "    # print(model.summary())\n",
        "    # ---------------- Compile and Fit ----------------\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "                  optimizer='rmsprop',\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    earlyStopping = EarlyStopping(monitor='val_loss', patience=35, verbose=0,restore_best_weights=True, mode='min')\n",
        "    # earlyStopping = EarlyStopping(monitor='val_accuracy', patience=15, verbose=0,restore_best_weights=True, mode='max')\n",
        "    history = model.fit((range_doppler_augmented_image,spectrogram_augmented_image),(range_doppler_concat_label_shuffle_concat),\n",
        "                    epochs=epoch_number,\n",
        "                    batch_size=batch_size,\n",
        "                    shuffle = True,\n",
        "                    callbacks=[earlyStopping],\n",
        "                    validation_data = ((validation_range_doppler,validation_spectrogram) , (range_doppler_validation_labels)))\n",
        "    # tf.keras.models.load_model\n",
        "    test_loss, test_accuracy  = model.evaluate([range_doppler_concat_shuffle_test,spectrogram_concat_shuffle_test],\\\n",
        "                                               [range_doppler_concat_label_shuffle_test],\n",
        "                  batch_size=batch_size)\n",
        "    # ---------------- Get Test Results ----------------\n",
        "    y_test_predicted = model.predict((range_doppler_concat_shuffle_test,spectrogram_concat_shuffle_test), batch_size=batch_size)\n",
        "    # ----- Binarize y_test_predicted values -----\n",
        "    y_test_predicted_binary = np.zeros(y_test_predicted.size)\n",
        "    for ii in range(y_test_predicted.size):\n",
        "      if y_test_predicted[ii] < 0.5:\n",
        "        y_test_predicted_binary[ii] = 0\n",
        "      else:\n",
        "        y_test_predicted_binary[ii] = 1\n",
        "    \n",
        "    test_precision, test_recall, test_f1_score, support = precision_recall_fscore_support(range_doppler_concat_label_shuffle_test, y_test_predicted_binary, average='macro')\n",
        "\n",
        "    test_accuracy_per_fold.append(test_accuracy)\n",
        "    f1_score_per_fold.append(test_f1_score)\n",
        "    del model\n",
        "  test_accuracy_per_run.append(sum(test_accuracy_per_fold)/num_folds)\n",
        "  f1_score_per_run.append(sum(f1_score_per_fold)/num_folds)\n",
        "  print(test_accuracy_per_run)\n",
        "  print(f1_score_per_run)\n",
        "print(f'Mean test accuracy is {\"{:.2f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, mean test f1 score is {\"{:.2f}\".format(sum(f1_score_per_run)/number_of_repeat)}, \\\n",
        "max test accuracy is {\"{:.2f}\".format(max(test_accuracy_per_run))}, max test f1 score is {\"{:.2f}\".format(max(f1_score_per_run))}, \\\n",
        "min test accuracy is {\"{:.2f}\".format(min(test_accuracy_per_run))}, min test f1 score is {\"{:.2f}\".format(min(f1_score_per_run))}, \\\n",
        "std of test accuracy is {\"{:.2f}\".format(np.std(test_accuracy_per_run, axis=0))}, std of test f1 score is {\"{:.2f}\".format(np.std(f1_score_per_run, axis=0))}')\n",
        "elapsed = time.time() - t\n",
        "print(f'Time elapsed through all process: {\"{:.2f}\".format(elapsed)}, sec')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0N7v4PDP4Dv2"
      },
      "source": [
        "print(f'Mean test accuracy is {\"{:.2f}\".format(sum(test_accuracy_per_run)/number_of_repeat)}, mean test f1 score is {\"{:.2f}\".format(sum(f1_score_per_run)/number_of_repeat)}, \\\n",
        "max test accuracy is {\"{:.2f}\".format(max(test_accuracy_per_run))}, max test f1 score is {\"{:.2f}\".format(max(f1_score_per_run))}, \\\n",
        "min test accuracy is {\"{:.2f}\".format(min(test_accuracy_per_run))}, min test f1 score is {\"{:.2f}\".format(min(f1_score_per_run))}, \\\n",
        "std of test accuracy is {\"{:.2f}\".format(np.std(test_accuracy_per_run, axis=0))}, std of test f1 score is {\"{:.2f}\".format(np.std(f1_score_per_run, axis=0))}')\n",
        "print(f'Time elapsed through all process: {\"{:.2f}\".format(elapsed)}, sec')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}