DeepWriter.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DeepWriter.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "R3QKHq7Y-jL5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import cv2\n",
        "from glob import glob\n",
        "import numpy as np\n",
        "from PIL import Image, ImageOps\n",
        "from tqdm import tqdm_notebook\n",
        "\n",
        "%tensorflow_version 2.x\n",
        "import tensorflow\n",
        "import PIL.ImageOps\n",
        "\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras.models import load_model\n",
        "from tensorflow.keras import Sequential, Input, Model\n",
        "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Add\n",
        "from tensorflow.keras.layers import Activation, Dense, Flatten, Dropout\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zSnuTK9s-sZo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def image_resize(image, width = None, height = None, inter = cv2.INTER_AREA):\n",
        "    # initialize the dimensions of the image to be resized and\n",
        "    # grab the image size\n",
        "    dim = None\n",
        "    (h, w) = image.shape[:2]\n",
        "\n",
        "    # if both the width and height are None, then return the\n",
        "    # original image\n",
        "    if width is None and height is None:\n",
        "        return image\n",
        "\n",
        "    # check to see if the width is None\n",
        "    if width is None:\n",
        "        # calculate the ratio of the height and construct the\n",
        "        # dimensions\n",
        "        r = height / float(h)\n",
        "        dim = (int(w * r), height)\n",
        "\n",
        "    # otherwise, the height is None\n",
        "    else:\n",
        "        # calculate the ratio of the width and construct the\n",
        "        # dimensions\n",
        "        r = width / float(w)\n",
        "        dim = (width, int(h * r))\n",
        "    \n",
        "    # resize the image\n",
        "    resized = cv2.resize(image, dim, interpolation = inter)\n",
        "\n",
        "    # return the resized image\n",
        "    return resized\n",
        "\n",
        "def slicer(images, names, segment = 80, timeAxis=False):\n",
        "    half = segment//4\n",
        "    \n",
        "    retImg = []\n",
        "    retClass = []\n",
        "    totbit = segment*segment\n",
        "    \n",
        "    for img, name in zip(images, names):\n",
        "        if timeAxis == True:\n",
        "            timeImage = []\n",
        "\n",
        "        for i in range(0, img.shape[1], half):\n",
        "            if i+half*3 > img.shape[1]:\n",
        "                continue\n",
        "            \n",
        "            tmp = img[:, i:i+segment]\n",
        "            tmp = np.pad(tmp, ((0, 0), (0, segment-tmp.shape[1])), 'constant', \n",
        "                         constant_values=0)\n",
        "\n",
        "            if timeAxis == True:\n",
        "                timeImage.append(tmp)\n",
        "            else:\n",
        "                retImg.append(tmp)\n",
        "                retClass.append(name)\n",
        "\n",
        "        if timeAxis == True:\n",
        "            retImg.append(np.array(timeImage))\n",
        "            retClass.append(name)\n",
        "    \n",
        "    X = np.array(retImg)\n",
        "    y = np.array(retClass)\n",
        "    \n",
        "    return X, y\n",
        "\n",
        "# Load data\n",
        "def loadData(perClassData=None, h=80):\n",
        "    '''\n",
        "    Give the directory of dataset in the glob function\n",
        "    Generate target name/identity in name variable\n",
        "    '''\n",
        "    imgFiles = glob(\"./dataset_60.54/**/*.png\")\n",
        "    print(len(imgFiles), 'images found.')\n",
        "\n",
        "    ImageArray = []\n",
        "    Names = []\n",
        "\n",
        "    for imgFile in tqdm_notebook(imgFiles):\n",
        "        fileName = (imgFile.split('/')[-1]).split('.')[0]\n",
        "        name = fileName.split('_')[0]                           # Target Class\n",
        "\n",
        "        img = Image.open(imgFile)\n",
        "        img = PIL.ImageOps.invert(img)\n",
        "        image = image_resize(np.array(img, dtype=np.uint8), height=h)\n",
        "        \n",
        "        if image.ndim > 2:\n",
        "            continue\n",
        "\n",
        "        image = image / 255\n",
        "        ImageArray.append(image)\n",
        "        Names.append(name)\n",
        "    \n",
        "    print('Total Unique Classes', len(np.unique(Names)))\n",
        "    return ImageArray, Names    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FiR26Uio-v4_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X, y = loadData(60, h=113)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kYJkPHbs6oCH",
        "colab_type": "text"
      },
      "source": [
        "X is the black and white word image of shape (row, cols, 1).\n",
        "\n",
        "row and col doesn't have to be same.\n",
        "\n",
        "y is the target class."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POi877L3SuPa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, \n",
        "                                                    random_state=40)\n",
        "\n",
        "# XX_test, yy_test is the multiple (113, 113) segmented images of word line\n",
        "XX_test, yy_test = slicer(X_test, y_test, segment=113, timeAxis=True)\n",
        "\n",
        "# Set timeAxis=False for a lower dimention"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEV4tbWtS1Cr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.preprocessing import OneHotEncoder\n",
        "OHE = OneHotEncoder().fit(np.array(y).reshape(-1, 1))\n",
        "\n",
        "y_train = np.array(y_train)\n",
        "y_test = np.array(y_test)\n",
        "\n",
        "y_train = y_train.reshape(-1, 1)\n",
        "y_test = y_test.reshape(-1, 1)\n",
        "\n",
        "y_train_OHE = OHE.fit_transform(y_train).toarray()\n",
        "y_test_OHE = OHE.transform(y_test).toarray()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7tnuGR1-6VO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def deepWriter(input_shape, classes):\n",
        "    # Two different input patches\n",
        "    patch_1 = Input(shape=input_shape)\n",
        "    patch_2 = Input(shape=input_shape)\n",
        "\n",
        "    # Convolution_1 shares the same weight\n",
        "    conv1 = Conv2D(96, kernel_size=5, strides=2, activation='relu')\n",
        "    out1 = conv1(patch_1)\n",
        "    out2 = conv1(patch_2)\n",
        "\n",
        "    # MaxPooling\n",
        "    MP = MaxPooling2D(3, strides=2)\n",
        "    out1 = MP(out1)\n",
        "    out2 = MP(out2)\n",
        "\n",
        "    # Convolution_2 shares the same weight\n",
        "    conv2 = Conv2D(256, kernel_size=3, activation='relu')\n",
        "    out1 = conv2(out1)\n",
        "    out2 = conv2(out2)\n",
        "\n",
        "    # MaxPooling\n",
        "    out1 = MP(out1)\n",
        "    out2 = MP(out2)\n",
        "\n",
        "    # Convolution_3 shares the same weight\n",
        "    conv3 = Conv2D(384, kernel_size=3, activation='relu')\n",
        "    out1 = conv3(out1)\n",
        "    out2 = conv3(out2)\n",
        "\n",
        "    # Convolution_4 shares the same weight\n",
        "    conv4 = Conv2D(384, kernel_size=3, activation='relu')\n",
        "    out1 = conv4(out1)\n",
        "    out2 = conv4(out2)\n",
        "\n",
        "    # Convolution_5 shares the same weight\n",
        "    conv5 = Conv2D(256, kernel_size=3, activation='relu')\n",
        "    out1 = conv5(out1)\n",
        "    out2 = conv5(out2)\n",
        "\n",
        "    # MaxPooling\n",
        "    out1 = MP(out1)\n",
        "    out2 = MP(out2)\n",
        "\n",
        "    # Flatten\n",
        "    flat = Flatten()\n",
        "    out1 = flat(out1)\n",
        "    out2 = flat(out2)\n",
        "\n",
        "    # Fully Connected Layer (FC6)\n",
        "    FC6 = Dense(1024)\n",
        "    out1 = FC6(out1)\n",
        "    out2 = FC6(out2)\n",
        "\n",
        "    # Dropout of 0.5\n",
        "    out1 = Dropout(0.5)(out1)\n",
        "    out2 = Dropout(0.5)(out2)\n",
        "\n",
        "    # Fully Conneted Layer (FC7)\n",
        "    FC7 = Dense(1024)\n",
        "    out1 = FC7(out1)\n",
        "    out2 = FC7(out2)\n",
        "\n",
        "    # Dropout of 0.5\n",
        "    out1 = Dropout(0.5)(out1)\n",
        "    out2 = Dropout(0.5)(out2)\n",
        "\n",
        "    # Summation of two outputs\n",
        "    out = Add()([out1, out2])\n",
        "\n",
        "    # Softmax layer\n",
        "    out = Dense(classes, activation='softmax')(out)\n",
        "\n",
        "    # Make model and compile\n",
        "    model = Model(inputs=[patch_1, patch_2], outputs=out)\n",
        "    model.compile(optimizer='adam', loss='categorical_crossentropy',\n",
        "                  metrics=['acc'])\n",
        "\n",
        "    return model\n",
        "\n",
        "\n",
        "def halfDeepWriter(input_shape, classes, frac=1):\n",
        "    patch_1 = Input(shape=input_shape)\n",
        "\n",
        "    out1 = Conv2D(int(96*frac), kernel_size=5, strides=2, activation='relu')(patch_1)\n",
        "    out1 = MaxPooling2D(3, strides=2)(out1)\n",
        "\n",
        "    out1 = Conv2D(int(256*frac), kernel_size=3, activation='relu')(out1)\n",
        "    out1 = MaxPooling2D(3, strides=2)(out1)\n",
        "\n",
        "    out1 = Conv2D(int(384*frac), kernel_size=3, activation='relu')(out1)\n",
        "    out1 = Conv2D(int(384*frac), kernel_size=3, activation='relu')(out1)\n",
        "    out1 = Conv2D(int(256*frac), kernel_size=3, activation='relu')(out1)\n",
        "    out1 = MaxPooling2D(3, strides=2)(out1)\n",
        "\n",
        "    out1 = Flatten()(out1)\n",
        "    out1 = Dense(int(1024*frac), activation='relu')(out1)\n",
        "    out1 = Dropout(0.5)(out1)\n",
        "\n",
        "    out1 = Dense(int(1024*frac), activation='relu')(out1)\n",
        "    out1 = Dropout(0.5)(out1)\n",
        "\n",
        "    out1 = Dense(classes, activation='softmax')(out1)\n",
        "\n",
        "    model = Model(inputs=patch_1, outputs=out1)\n",
        "    model.compile(optimizer='adam', loss='categorical_crossentropy',\n",
        "                  metrics=['acc'])\n",
        "\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ojO2tQAtEftb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Random image strip image generator of DeepWriter's image stripping strategy\n",
        "\n",
        "class dataGeneratorDeepWriter(tensorflow.keras.utils.Sequence):\n",
        "    def __init__(self, X, y, batch_size=32, shuffle=True, w=80):\n",
        "        self.batch_size = batch_size\n",
        "        self.shuffle = shuffle\n",
        "        self.inputX = X\n",
        "        self.inputY = y\n",
        "        self.w = w\n",
        "        self.h = self.inputX[0].shape[0]\n",
        "        self.total = len(X)\n",
        "        self.indexes = np.arange(self.total)\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        'Denotes the number of batches per epoch'\n",
        "        return int(np.floor(self.total / self.batch_size))\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        'Generate one batch of data'\n",
        "        # Generate indexes of the batch\n",
        "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
        "\n",
        "        # Generate data\n",
        "        return self.__data_generation(indexes)\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        'Updates indexes after each epoch'\n",
        "        if self.shuffle == True:\n",
        "            np.random.shuffle(self.indexes)\n",
        "\n",
        "    def __data_generation(self, batchIndexes):\n",
        "        'Generates data containing batch_size samples' # X : (2, n_samples, *dim, n_channels)\n",
        "        # Initialization\n",
        "        X = np.zeros((self.batch_size, 2, self.h, self.w))\n",
        "        y = np.empty((self.batch_size, self.inputY.shape[-1]), dtype=int)\n",
        "        \n",
        "        # Generate data\n",
        "        for i, ID in enumerate(batchIndexes):\n",
        "            # Black Image\n",
        "            tmpImg = np.zeros((self.h, self.w))\n",
        "            \n",
        "            # Starting column position\n",
        "            y_pos1, y_pos2 = map(int, (np.random.randint(low=0, \n",
        "                        high=max(self.inputX[ID].shape[1]-self.w//3, 1),\n",
        "                        size=2)))\n",
        "            \n",
        "            # Placing Image in black image\n",
        "            tmpImg1 = (self.inputX[ID])[:, y_pos1:y_pos1+self.w]\n",
        "            tmpImg2 = (self.inputX[ID])[:, y_pos2:y_pos2+self.w]\n",
        "\n",
        "            # Placing Image in output\n",
        "            X[i, 0, 0:tmpImg1.shape[0], 0:tmpImg1.shape[1]] = tmpImg1\n",
        "            X[i, 1, 0:tmpImg2.shape[0], 0:tmpImg2.shape[1]] = tmpImg2\n",
        "            \n",
        "            # Store class\n",
        "            y[i] = self.inputY[ID]\n",
        "\n",
        "        X = X[:, :, :, :, np.newaxis]\n",
        "        return [X[:, 0, :, :], X[:, 1, :, :]], y\n",
        "\n",
        "\n",
        "class dataGeneratorHalfDeepWriter(tensorflow.keras.utils.Sequence):\n",
        "    def __init__(self, X, y, batch_size=32, shuffle=True, w=80):\n",
        "        self.batch_size = batch_size\n",
        "        self.shuffle = shuffle\n",
        "        self.inputX = X\n",
        "        self.inputY = y\n",
        "        self.w = w\n",
        "        self.h = self.inputX[0].shape[0]\n",
        "        self.total = len(X)\n",
        "        self.indexes = np.arange(self.total)\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        'Denotes the number of batches per epoch'\n",
        "        return int(np.floor(self.total / self.batch_size))\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        'Generate one batch of data'\n",
        "        # Generate indexes of the batch\n",
        "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
        "\n",
        "        # Generate data\n",
        "        return self.__data_generation(indexes)\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        'Updates indexes after each epoch'\n",
        "        if self.shuffle == True:\n",
        "            np.random.shuffle(self.indexes)\n",
        "\n",
        "    def __data_generation(self, batchIndexes):\n",
        "        'Generates data containing batch_size samples' # X : (2, n_samples, *dim, n_channels)\n",
        "        # Initialization\n",
        "        X = np.zeros((self.batch_size, self.h, self.w))\n",
        "        y = np.empty((self.batch_size, self.inputY.shape[-1]), dtype=int)\n",
        "        \n",
        "        # Generate data\n",
        "        for i, ID in enumerate(batchIndexes):\n",
        "            # Black Image\n",
        "            tmpImg = np.zeros((self.h, self.w))\n",
        "            \n",
        "            # Starting column position\n",
        "            y_pos1 = int(np.random.randint(low=0, \n",
        "                        high=max(self.inputX[ID].shape[1]-self.w//3, 1),\n",
        "                        size=1))\n",
        "            \n",
        "            # Placing Image in black image\n",
        "            tmpImg1 = (self.inputX[ID])[:, y_pos1:y_pos1+self.w]\n",
        "\n",
        "            # Placing Image in output\n",
        "            X[i, 0:tmpImg1.shape[0], 0:tmpImg1.shape[1]] = tmpImg1\n",
        "            \n",
        "            # Store class\n",
        "            y[i] = self.inputY[ID]\n",
        "\n",
        "        X = X[:, :, :, np.newaxis]\n",
        "        return X, y\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wHHI9yWlInfu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = halfDeepWriter((113, 113, 1), 54, )\n",
        "model.summary()\n",
        "#model.load_weights('/content/best.hdf5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Fn0A6pYSpeS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_gen = dataGeneratorHalfDeepWriter(X_train, y_train_OHE, batch_size=128, w=113)\n",
        "test_gen = dataGeneratorHalfDeepWriter(X_test, y_test_OHE, batch_size=128, w=113)\n",
        "\n",
        "hist = model.fit(train_gen, validation_data=test_gen, epochs=3000, \n",
        "                 callbacks=[ ModelCheckpoint(filepath='/content/best.hdf5',\n",
        "                             save_best_only=True, monitor='acc', mode='max',\n",
        "                            ), ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TptGmiCKzR-J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Calculating word-level accuracy\n",
        "\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "y_pred = []\n",
        "y_true = []\n",
        "\n",
        "for batch, tar in zip(XX_test, yy_test_OHE):\n",
        "    if batch.shape[0] <= 0:\n",
        "        continue\n",
        "    batch = batch[:, :, :, np.newaxis]\n",
        "    y_pred.append(np.argmax(np.sum(model.predict(batch), axis=0), axis=0))\n",
        "    y_true.append(np.argmax(tar))\n",
        "\n",
        "accuracy_score(y_true, y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2-UNfAoS_tc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}