load_split_xray_folders.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nezamtrm/Extracting-contents-of-a-table-in-pdf-file-by-pdfplumber/blob/main/load_split_xray_folders.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QvDvcCwkUYkh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b44bd30c-f0b0-4f40-a9d3-609c1960413f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting opendatasets\n",
            "  Downloading opendatasets-0.1.22-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from opendatasets) (4.65.0)\n",
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.10/dist-packages (from opendatasets) (1.5.16)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from opendatasets) (8.1.6)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (1.16.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (2023.7.22)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (2.8.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (2.27.1)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (8.0.1)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (1.26.16)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.10/dist-packages (from kaggle->opendatasets) (6.0.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from bleach->kaggle->opendatasets) (0.5.1)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.10/dist-packages (from python-slugify->kaggle->opendatasets) (1.3)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->kaggle->opendatasets) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->kaggle->opendatasets) (3.4)\n",
            "Installing collected packages: opendatasets\n",
            "Successfully installed opendatasets-0.1.22\n"
          ]
        }
      ],
      "source": [
        "!pip install opendatasets"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import opendatasets as od"
      ],
      "metadata": {
        "id": "L97AOcGHUjh7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "od.download('https://www.kaggle.com/datasets/artyomkolas/3-kinds-of-pneumonia')"
      ],
      "metadata": {
        "id": "MCqVCi3mUng6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80474196-0c89-40c6-8b8d-297f28a804eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Please provide your Kaggle credentials to download this dataset. Learn more: http://bit.ly/kaggle-creds\n",
            "Your Kaggle username:"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "metadata": {
        "id": "Ewthl1CeUrIO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "colab_dataset_path = '/content/3-kinds-of-pneumonia/'\n",
        "\n",
        "drive_dataset_path = '/content/gdrive/My Drive/Colab Notebooks/dataset'"
      ],
      "metadata": {
        "id": "YgW_IJpHUv8a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import shutil\n",
        "\n",
        "shutil.copytree(colab_dataset_path, drive_dataset_path)"
      ],
      "metadata": {
        "id": "kb8vVBikUyEK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_dir = '/content/gdrive/MyDrive/Colab Notebooks/dataset/Curated X-Ray Dataset'\n",
        "class_names = ['COVID-19', 'Normal', 'Pneumonia-Bacterial', 'Pneumonia-Viral']"
      ],
      "metadata": {
        "id": "Ag7KKjPPUy0S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_image_fps = []\n",
        "all_labels = []"
      ],
      "metadata": {
        "id": "kHH6Bo6XUy21"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for class_index, class_name in enumerate(class_names):\n",
        "    class_dir = os.path.join(data_dir, class_name)\n",
        "    image_fps = [os.path.join(class_dir, file) for file in os.listdir(class_dir)]\n",
        "    all_image_fps.extend(image_fps)\n",
        "    all_labels.extend([class_index] * len(image_fps))"
      ],
      "metadata": {
        "id": "uKld8oDQU5d6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_size = 0.7  # Percentage of data for training\n",
        "valid_size = 0.15  # Percentage of data for validation\n",
        "test_size = 0.15  # Percentage of data for testing\n",
        "\n",
        "# Split data into training and temp sets\n",
        "train_image_fps, temp_image_fps, train_labels, temp_labels = train_test_split(\n",
        "    all_image_fps, all_labels, train_size=train_size, stratify=all_labels, random_state=42\n",
        ")\n",
        "\n",
        "# Split temp set into validation and test sets\n",
        "valid_image_fps, test_image_fps, valid_labels, test_labels = train_test_split(\n",
        "    temp_image_fps, temp_labels, test_size=test_size / (test_size + valid_size),\n",
        "    stratify=temp_labels, random_state=42\n",
        ")\n"
      ],
      "metadata": {
        "id": "p0wsTiyzU5kr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data_dir0 = os.path.join(data_dir, 'test_set')\n",
        "if not os.path.exists(test_data_dir0):\n",
        "    os.makedirs(test_data_dir0)\n",
        "\n",
        "valid_data_dir0 = os.path.join(data_dir, 'valid_set')\n",
        "if not os.path.exists(valid_data_dir0):\n",
        "    os.makedirs(valid_data_dir0)\n",
        "\n",
        "train_data_dir0 = os.path.join(data_dir, 'train_set')\n",
        "if not os.path.exists(train_data_dir0):\n",
        "    os.makedirs(train_data_dir0)\n"
      ],
      "metadata": {
        "id": "Mc03mJNUU5m5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for image_fp in valid_image_fps:\n",
        "    class_name = class_names[valid_labels[valid_image_fps.index(image_fp)]]\n",
        "    vaild_data_dir = os.path.join(valid_data_dir0, class_name)\n",
        "    if not os.path.exists(vaild_data_dir):\n",
        "        os.makedirs(vaild_data_dir)\n",
        "    shutil.copy(image_fp, vaild_data_dir)\n"
      ],
      "metadata": {
        "id": "QscYlYM-U5sq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for image_fp in test_image_fps:\n",
        "    class_name = class_names[test_labels[test_image_fps.index(image_fp)]]\n",
        "    test_data_dir = os.path.join(test_data_dir0, class_name)\n",
        "    if not os.path.exists(test_data_dir):\n",
        "        os.makedirs(test_data_dir)\n",
        "    shutil.copy(image_fp, test_data_dir)\n"
      ],
      "metadata": {
        "id": "YE4VKuhdVB-6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for image_fp in train_image_fps:\n",
        "    class_name = class_names[train_labels[train_image_fps.index(image_fp)]]\n",
        "    train_data_dir = os.path.join(train_data_dir0, class_name)\n",
        "    if not os.path.exists(train_data_dir):\n",
        "        os.makedirs(train_data_dir)\n",
        "    shutil.copy(image_fp, train_data_dir)"
      ],
      "metadata": {
        "id": "i4ZlNDV3VCBy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data_dir0 = '/content/gdrive/MyDrive/Colab Notebooks/dataset/Curated X-Ray Dataset/test_set'\n",
        "train_data_dir0 = '/content/gdrive/MyDrive/Colab Notebooks/dataset/Curated X-Ray Dataset/train_set'\n",
        "valid_data_dir0 = '/content/gdrive/MyDrive/Colab Notebooks/dataset/Curated X-Ray Dataset/valid_set'"
      ],
      "metadata": {
        "id": "fSwPbnE4VCHF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "datagen = ImageDataGenerator(rescale=1.0 / 255)\n"
      ],
      "metadata": {
        "id": "l3_BP2dZVI0K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set batch size and image size\n",
        "batch_size = 32\n",
        "image_size = (224, 224)\n",
        "\n",
        "train_dataset = datagen.flow_from_directory(\n",
        "    train_data_dir0,\n",
        "    target_size=image_size,\n",
        "    batch_size=batch_size,\n",
        "    class_mode='categorical',\n",
        "    classes=class_names,\n",
        "    shuffle=True,\n",
        "    subset='training'  # Use this if you have separate train and validation folders\n",
        ")\n",
        "\n",
        "validation_dataset = datagen.flow_from_directory(\n",
        "    valid_data_dir0,\n",
        "    target_size=image_size,\n",
        "    batch_size=batch_size,\n",
        "    class_mode='categorical',\n",
        "    classes=class_names,\n",
        "    shuffle=False,\n",
        ")\n",
        "\n",
        "\n",
        "test_dataset = datagen.flow_from_directory(\n",
        "    test_data_dir0,\n",
        "    target_size=image_size,\n",
        "    batch_size=batch_size,\n",
        "    class_mode='categorical',\n",
        "    classes=class_names,\n",
        "    shuffle=False  # Keep the original order of test images\n",
        ")\n"
      ],
      "metadata": {
        "id": "TH1CwrOnVI_Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ankJUgugVJBx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "W6twqlkUVJG6"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}