Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sys import byteorder\n",
    "from array import array\n",
    "from struct import pack\n",
    "\n",
    "import pyaudio\n",
    "import wave\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import signal\n",
    "from scipy.io import wavfile\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "from skimage.measure import block_reduce\n",
    "\n",
    "#To find the duration of wave file in seconds\n",
    "import wave\n",
    "import contextlib\n",
    "\n",
    "#Keras imports\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, Dropout, Flatten, MaxPooling2D\n",
    "from keras.models import model_from_json\n",
    "\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']  = 'True'\n",
    "imwidth                             = 50\n",
    "imheight                            = 34\n",
    "total_examples                      = 2000\n",
    "speakers                            = 4\n",
    "examples_per_speaker                = 50\n",
    "tt_split                            = 0.1\n",
    "num_classes                         = 10\n",
    "test_rec_folder                     = \"./testrecs\"\n",
    "log_image_folder                     = \"./logims\"\n",
    "recording_directory                 = \"../SoundCNN/recordings/\"\n",
    "num_test_files                      = 1\n",
    "\n",
    "THRESHOLD                           = 1000\n",
    "CHUNK_SIZE                          = 512\n",
    "FORMAT                              = pyaudio.paInt16\n",
    "RATE                                = 8000#44100\n",
    "WINDOW_SIZE                         = 50\n",
    "CHECK_THRESH                        = 3\n",
    "SLEEP_TIME                          = 0.5 #(seconds)\n",
    "IS_PLOT                             = 1\n",
    "LOG_MODE                            = 0 # 1 for time, 2 for frequency\n",
    "\n",
    "#Check for silence\n",
    "def is_silent(snd_data):\n",
    "    \"Returns 'True' if below the 'silent' threshold\"\n",
    "    return max(snd_data) < THRESHOLD\n",
    "\n",
    "\"\"\"\n",
    "Record a word or words from the microphone and \n",
    "return the data as an array of signed shorts.\n",
    "\"\"\"\n",
    "def record():\n",
    "    p = pyaudio.PyAudio()\n",
    "    stream = p.open(format=FORMAT, channels=1, rate=RATE,\n",
    "        input=True, output=True,\n",
    "        frames_per_buffer=CHUNK_SIZE)\n",
    "\n",
    "    num_silent = 0\n",
    "    snd_started = False\n",
    "\n",
    "    r = array('h')\n",
    "\n",
    "    while 1:\n",
    "        # little endian, signed short\n",
    "        snd_data = array('h', stream.read(CHUNK_SIZE))\n",
    "        if byteorder == 'big':\n",
    "            snd_data.byteswap()\n",
    "        r.extend(snd_data)\n",
    "\n",
    "        silent = is_silent(snd_data)\n",
    "\n",
    "        if silent and snd_started:\n",
    "            num_silent += 1\n",
    "        elif not silent and not snd_started:\n",
    "            snd_started = True\n",
    "\n",
    "        if snd_started and num_silent > 20:\n",
    "            break\n",
    "\n",
    "    sample_width = p.get_sample_size(FORMAT)\n",
    "    stream.stop_stream()\n",
    "    stream.close()\n",
    "    p.terminate()\n",
    "\n",
    "    return sample_width, r\n",
    "\n",
    "#Extract relevant signal from the captured audio\n",
    "def get_bounds(ds):\n",
    "    np.array(ds)\n",
    "    lds = len(ds)\n",
    "    count = 0\n",
    "    ll=-1\n",
    "    ul=-1\n",
    "\n",
    "    #Lower Limit\n",
    "    for i in range(0,lds,WINDOW_SIZE):\n",
    "        sum = 0\n",
    "        for k in range(i,(i+WINDOW_SIZE)%lds):\n",
    "            sum = sum + np.absolute(ds[k])\n",
    "        if(sum>THRESHOLD):\n",
    "            count +=1\n",
    "        if(count>CHECK_THRESH):\n",
    "            ll = i - WINDOW_SIZE * CHECK_THRESH\n",
    "            break\n",
    "        \n",
    "    #Upper Limit\n",
    "    count = 0\n",
    "    for j in range(i,lds,WINDOW_SIZE):\n",
    "        sum = 0\n",
    "        for k in range(j,(j+WINDOW_SIZE)%lds):\n",
    "            sum = sum + np.absolute(ds[k])\n",
    "        if(sum<THRESHOLD):\n",
    "            count +=1\n",
    "        if(count>CHECK_THRESH):\n",
    "            ul = j - WINDOW_SIZE * CHECK_THRESH\n",
    "\n",
    "\n",
    "        if(ul>0 and ll >0):\n",
    "            break\n",
    "    return ll, ul \n",
    "\n",
    "\n",
    "# Records from the microphone and outputs the resulting data to 'path'\n",
    "def record_to_file(path):\n",
    "    \n",
    "    sample_width, data = record()\n",
    "    ll, ul = get_bounds(data)\n",
    "    print(ll,ul)\n",
    "    if(ul-ll<100):\n",
    "        return 0\n",
    "    #nonz  = np.nonzero(data)\n",
    "    ds = data[ll:ul]\n",
    "    if(IS_PLOT):\n",
    "        plt.plot(data)\n",
    "        plt.axvline(x=ll)\n",
    "        #plt.axvline(x=ll+5000)\n",
    "        plt.axvline(x=ul)\n",
    "        plt.show()\n",
    "\n",
    "    #data = pack('<' + ('h'*len(data)), *data)\n",
    "    fname = \"0.wav\"\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    wf = wave.open(os.path.join(path,fname), 'wb')\n",
    "    wf.setnchannels(1)\n",
    "    wf.setsampwidth(sample_width)\n",
    "    wf.setframerate(RATE)\n",
    "    wf.writeframes(ds)\n",
    "    wf.close()\n",
    "    return 1\n",
    "\n",
    "# Function to find the duration of the wave file in seconds\n",
    "def findDuration(fname):\n",
    "    with contextlib.closing(wave.open(fname,'r')) as f:\n",
    "        frames = f.getnframes()\n",
    "        rate = f.getframerate()\n",
    "        sw   = f.getsampwidth()\n",
    "        chan = f.getnchannels()\n",
    "        duration = frames / float(rate)\n",
    "        #print(\"File:\", fname, \"--->\",frames, rate, sw, chan)\n",
    "        return duration\n",
    "\n",
    "#Plot Spectrogram\n",
    "def graph_spectrogram(wav_file, nfft=512, noverlap=511):\n",
    "    findDuration(wav_file)\n",
    "    rate, data = wavfile.read(wav_file)\n",
    "    #print(\"\")\n",
    "    fig,ax = plt.subplots(1)\n",
    "    fig.subplots_adjust(left=0,right=1,bottom=0,top=1)\n",
    "    ax.axis('off')\n",
    "    pxx, freqs, bins, im = ax.specgram(x=data, Fs=rate, noverlap=noverlap, NFFT=nfft)\n",
    "    ax.axis('off')\n",
    "    plt.rcParams['figure.figsize'] = [0.75,0.5]\n",
    "    #fig.savefig('sp_xyz.png', dpi=300, frameon='false')\n",
    "    fig.canvas.draw()\n",
    "    size_inches  = fig.get_size_inches()\n",
    "    dpi          = fig.get_dpi()\n",
    "    width, height = fig.get_size_inches() * fig.get_dpi()\n",
    "\n",
    "    #print(size_inches, dpi, width, height)\n",
    "    mplimage = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)\n",
    "    #print(\"MPLImage Shape: \", np.shape(mplimage))\n",
    "    imarray = np.reshape(mplimage, (int(height), int(width), 3))\n",
    "    plt.close(fig)\n",
    "    return imarray\n",
    "\n",
    "#Convert color image to grayscale\n",
    "def rgb2gray(rgb):\n",
    "    return np.dot(rgb[...,:3], [0.299, 0.587, 0.114])\n",
    "\n",
    "#Normalize Gray colored image\n",
    "def normalize_gray(array):\n",
    "    return (array - array.min())/(array.max() - array.min())\n",
    "\n",
    "#Split the dataset into test and train sets randomly\n",
    "def create_train_test(audio_dir):\n",
    "    file_names = [f for f in os.listdir(audio_dir) if '.wav' in f]\n",
    "    file_names.sort()\n",
    "    test_list = []\n",
    "    train_list = []\n",
    "    \n",
    "    for i in range(int(total_examples/examples_per_speaker)):\n",
    "        test_list.extend(random.sample(file_names[(i*examples_per_speaker+1):(i+1)*examples_per_speaker], int(examples_per_speaker*tt_split)))\n",
    "\n",
    "    train_list = [x for x in file_names if x not in test_list]\n",
    "\n",
    "    y_test = np.zeros(len(test_list))\n",
    "    y_train = np.zeros(len(train_list))\n",
    "    x_train = np.zeros((len(train_list), imheight, imwidth))\n",
    "    x_test = np.zeros((len(test_list), imheight, imwidth))\n",
    "\n",
    "    tuni1   = np.zeros(len(test_list))\n",
    "    tuni2   = np.zeros(len(test_list))\n",
    "\n",
    "    for i, f in enumerate(test_list):\n",
    "        y_test[i]     = int(f[0])\n",
    "        spectrogram   = graph_spectrogram( audio_dir + f )\n",
    "        graygram      = rgb2gray(spectrogram)\n",
    "        normgram      = normalize_gray(graygram)\n",
    "        norm_shape    = normgram.shape\n",
    "        if(norm_shape[0]>150):\n",
    "            continue\n",
    "        redgram       = block_reduce(normgram, block_size = (3,3), func = np.mean)\n",
    "        x_test[i,:,:] = redgram\n",
    "        print(\"Progress Test Data: {:2.1%}\".format(float(i) / len(test_list)), end=\"\\r\")\n",
    "\n",
    "    for i, f in enumerate(train_list):\n",
    "        y_train[i] = int(f[0])\n",
    "        spectrogram   = graph_spectrogram( audio_dir + f )\n",
    "        graygram      = rgb2gray(spectrogram)\n",
    "        normgram      = normalize_gray(graygram)\n",
    "        norm_shape    = normgram.shape\n",
    "        if(norm_shape[0]>150):\n",
    "            continue\n",
    "        redgram       = block_reduce(normgram, block_size = (3,3), func = np.mean)\n",
    "        x_train[i,:,:] = redgram\n",
    "        print(\"Progress Training Data: {:2.1%}\".format(float(i) / len(train_list)), end=\"\\r\")\n",
    "        \n",
    "    return x_train, y_train, x_test, y_test\n",
    "\n",
    "#Create Keras Model\n",
    "def create_model(path):\n",
    "    x_train, y_train, x_test, y_test = create_train_test(path)\n",
    "\n",
    "    print(\"Size of Training Data:\", np.shape(x_train))\n",
    "    print(\"Size of Training Labels:\", np.shape(y_train))\n",
    "    print(\"Size of Test Data:\", np.shape(x_test))\n",
    "    print(\"Size of Test Labels:\", np.shape(y_test))\n",
    "\n",
    "    y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "    y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "    x_train = x_train.reshape(x_train.shape[0], imheight, imwidth, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], imheight, imwidth, 1)\n",
    "    input_shape = (imheight, imwidth, 1)\n",
    "    batch_size = 4\n",
    "    epochs = 1\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))\n",
    "    model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.adam(), metrics=['accuracy'])\n",
    "    print(model.summary())\n",
    "\n",
    "    model.fit(x_train, y_train, batch_size=4, epochs=10, verbose=1, validation_data=(x_test, y_test))\n",
    "    return model\n",
    "\n",
    "#Extract wave data from recorded audio\n",
    "def get_wav_data(path):\n",
    "    input_wav           = path\n",
    "    spectrogram         = graph_spectrogram( input_wav )\n",
    "    graygram            = rgb2gray(spectrogram)\n",
    "    normgram            = normalize_gray(graygram)\n",
    "    norm_shape          = normgram.shape\n",
    "    #print(\"Spec Shape->\", norm_shape)\n",
    "    if(norm_shape[0]>100):\n",
    "        redgram             = block_reduce(normgram, block_size = (26,26), func = np.mean)\n",
    "    else:\n",
    "        redgram             = block_reduce(normgram, block_size = (3,3), func = np.mean)\n",
    "    redgram             = redgram[0:imheight,0:imwidth]\n",
    "    red_data            = redgram.reshape(imheight,imwidth, 1)\n",
    "    empty_data          = np.empty((1,imheight,imwidth,1))\n",
    "    empty_data[0,:,:,:] = red_data\n",
    "    new_data            = empty_data\n",
    "    return new_data\n",
    "\n",
    "#Save created model\n",
    "def save_model_to_disk(model):\n",
    "    # serialize model to JSON\n",
    "    model_json = model.to_json()\n",
    "    with open(\"model.json\", \"w\") as json_file:\n",
    "        json_file.write(model_json)\n",
    "    # serialize weights to HDF5\n",
    "    model.save_weights(\"model.h5\")\n",
    "    print(\"Saved model to disk\")\n",
    "\n",
    "#Load saved model\n",
    "def load_model_from_disk():\n",
    "    # load json and create model\n",
    "    json_file = open('model.json', 'r')\n",
    "    loaded_model_json = json_file.read()\n",
    "    json_file.close()\n",
    "    loaded_model = model_from_json(loaded_model_json)\n",
    "    # load weights into new model\n",
    "    loaded_model.load_weights(\"model.h5\")\n",
    "    print(\"Loaded model from disk\")\n",
    "    return loaded_model\n",
    "\n",
    "#In Loggin mode capture one example of each class. Display it in time and frequency domain.\n",
    "def generate_log(in_dir, num_samps_per_cat):\n",
    "    file_names = [f for f in os.listdir(in_dir) if '.wav' in f]\n",
    "    checklist  = np.zeros(num_samps_per_cat * 10)\n",
    "    final_list = []\n",
    "    iternum = 0\n",
    "    \n",
    "    #Get a random sample for each category\n",
    "    while(1):\n",
    "        print(\"Iteration Number:\", iternum)\n",
    "        sample_names = random.sample(file_names,10)\n",
    "        for name in sample_names:\n",
    "            categ = int(name[0])\n",
    "            if(checklist[categ]<num_samps_per_cat):\n",
    "                checklist[categ]+=1\n",
    "                final_list.append(name)\n",
    "        if(int(checklist.sum())==(num_samps_per_cat * 10)):\n",
    "            break \n",
    "        iternum+=1\n",
    "    print(final_list)\n",
    "\n",
    "    #Generate Images for each sample\n",
    "    lif = os.path.join(log_image_folder,time.strftime(\"%Y-%m-%d_%H-%M-%S\", time.gmtime()))\n",
    "    if not os.path.exists(lif):\n",
    "        os.makedirs(lif)\n",
    "    for name in final_list:      \n",
    "        #Time Domain Signal\n",
    "        rate, data = wavfile.read(os.path.join(in_dir,name))\n",
    "        if(LOG_MODE==1):   \n",
    "            fig, ax = plt.subplots( nrows=1, ncols=1 )  # create figure & 1 axis\n",
    "            ax.set_title('Sound of ' +name[0] + ' - Sampled audio signal in time')\n",
    "            ax.set_xlabel('Sample number')\n",
    "            ax.set_ylabel('Amplitude')\n",
    "            ax.plot(data)\n",
    "            fig.savefig(os.path.join(lif, name[0:5]+'.png'))   # save the figure to file\n",
    "            plt.close(fig)\n",
    "    \n",
    "        #Frequency Domain Signals\n",
    "        if(LOG_MODE==2):\n",
    "            fig,ax = plt.subplots(1)\n",
    "            #fig.subplots_adjust(left=0,right=1,bottom=0,top=1)\n",
    "            #ax.axis('off')\n",
    "            pxx, freqs, bins, im = ax.specgram(x=data, Fs=rate, noverlap=511, NFFT=512)\n",
    "            #ax.axis('off')\n",
    "            #plt.rcParams['figure.figsize'] = [0.75,0.5]\n",
    "            cbar = fig.colorbar(im)\n",
    "            cbar.set_label('Intensity dB')\n",
    "            #ax.axis(\"tight\")\n",
    "\n",
    "            # Prettify\n",
    "            ax.set_title('Spectrogram of spoken ' +name[0] )\n",
    "            ax.set_xlabel('time')\n",
    "            ax.set_ylabel('frequency Hz')\n",
    "            fig.savefig(os.path.join(lif, name[0]+'_spec.png'), dpi=300, frameon='false')\n",
    "            plt.close(fig)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    if(not LOG_MODE):\n",
    "        while(1):\n",
    "            time.sleep(SLEEP_TIME)\n",
    "            if(os.path.isfile('model.json')):\n",
    "                print(\"please speak a word into the microphone\")\n",
    "                success = record_to_file(test_rec_folder)\n",
    "                if(not success):\n",
    "                    print(\" Speak Again Clearly\")\n",
    "                    continue\n",
    "            else:\n",
    "                print(\"********************\\n\\nTraining The Model\\n\")\n",
    "            if(os.path.isfile('model.json')):\n",
    "                model = load_model_from_disk()\n",
    "            else:\n",
    "                model = create_model(recording_directory)\n",
    "                save_model_to_disk(model)\n",
    "            #fname = 'r4.wav'\n",
    "            #new_data = get_wav_data(fname)\n",
    "            for i in range(num_test_files):\n",
    "            #for i in range(1):\n",
    "                fname = str(i)+\".wav\"\n",
    "                new_data    = get_wav_data(os.path.join(test_rec_folder,fname))    \n",
    "                predictions = np.array(model.predict(new_data))\n",
    "                maxpred = predictions.argmax()\n",
    "                normpred = normalize_gray(predictions)*100\n",
    "                predarr = np.array(predictions[0])\n",
    "                sumx = predarr.sum()\n",
    "                print(\"TestFile Name: \", fname, \" The Model Predicts:\", maxpred)\n",
    "                for nc in range(num_classes):\n",
    "                    confidence = np.round(100*(predarr[nc]/sumx))\n",
    "                    print(\"Class \", nc, \" Confidence: \", confidence)\n",
    "                #print(\"TestFile Name: \",fname, \" Values:\", predictions)\n",
    "                print(\"_____________________________\\n\")\n",
    "    else:\n",
    "        generate_log(recording_directory,6)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}